fix(translate): three audit fixes — dict hash, SQL chunking, duplicate logic
1. dict_snapshot_hash now includes entry count + max(updated_at) per dictionary. Previously only hashed dictionary IDs, meaning edits to dictionary entries did NOT invalidate the translation cache. Stale cached translations could be served after editing dictionary entries. (HIGH severity) 2. _batch_insert.py now uses SQLGenerator.generate_batch() with 500-row chunking instead of a single massive INSERT statement. Prevents potential SQL size limit issues with large batches or many target languages. (LOW severity) 3. Fixed same bug in preview_response_parser.py — compute_dict_snapshot_hash had identical ID-only hash flaw. Tests: 69/69 translate tests pass.
This commit is contained in:
@@ -49,11 +49,25 @@ def insert_batch_to_target(
|
||||
if dialect is None:
|
||||
return
|
||||
|
||||
sql, row_count = _generate_insert_sql(dialect, job, columns, rows_for_sql, batch_id)
|
||||
if sql is None:
|
||||
return
|
||||
|
||||
_execute_insert_sql(executor, sql, batch_id, row_count)
|
||||
# Use chunked INSERT to avoid SQL size limits (max 500 VALUES rows per statement)
|
||||
statements = SQLGenerator.generate_batch(
|
||||
dialect=dialect,
|
||||
target_schema=job.target_schema,
|
||||
target_table=job.target_table or "translated_data",
|
||||
columns=columns,
|
||||
rows=rows_for_sql,
|
||||
key_columns=job.target_key_cols,
|
||||
upsert_strategy=job.upsert_strategy or "MERGE",
|
||||
max_rows_per_statement=500,
|
||||
)
|
||||
total_inserted = 0
|
||||
for sql, chunk_count in statements:
|
||||
if sql is None:
|
||||
continue
|
||||
_execute_insert_sql(executor, sql, batch_id, chunk_count)
|
||||
total_inserted += chunk_count
|
||||
logger.reason(f"Batch {batch_id[:12]} inserted {total_inserted} rows",
|
||||
{"batch_id": batch_id, "rows": total_inserted, "chunks": len(statements)})
|
||||
# #endregion insert_batch_to_target
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user