fix(llm): add fetch-models endpoint, fix SQL Lab INSERT (client_id truncation, sync mode, target_column, timestamp normalization)
- Add POST /api/llm/providers/fetch-models route with LLMClient.fetch_models() - Add target_column to TranslationJob model/schema/service/orchestrator - Fix SQL Lab execute: truncate client_id to 11 chars (varchar(11)) - Switch SQL Lab to sync mode (runAsync: false) — no Celery workers - Fix polling: unwrap nested result from Superset query API - Fix ClickHouse timestamp: normalize float timestamps to YYYY-MM-DD
This commit is contained in:
@@ -63,7 +63,6 @@ class TranslationOrchestrator:
|
||||
self._job: Optional[TranslationJob] = None
|
||||
|
||||
# region start_run [TYPE Function]
|
||||
# @COMPLEXITY: 5
|
||||
# @PURPOSE: Start a new translation run for a job.
|
||||
# @PRE: job_id exists. For manual runs, there must be an accepted preview session.
|
||||
# @POST: TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.
|
||||
@@ -105,6 +104,7 @@ class TranslationOrchestrator:
|
||||
"source_key_cols": job.source_key_cols,
|
||||
"target_key_cols": job.target_key_cols,
|
||||
"translation_column": job.translation_column,
|
||||
"target_column": job.target_column,
|
||||
"context_columns": job.context_columns,
|
||||
"target_language": job.target_language,
|
||||
"provider_id": job.provider_id,
|
||||
@@ -263,7 +263,11 @@ class TranslationOrchestrator:
|
||||
run.insert_status = insert_result.get("status")
|
||||
run.superset_execution_id = str(insert_result.get("query_id") or "")
|
||||
run.superset_execution_log = insert_result
|
||||
run.status = "COMPLETED" if insert_result.get("status") == "success" else "COMPLETED"
|
||||
# Preserve the translation-phase status. If the executor already
|
||||
# marked the run FAILED (all LLM calls failed) we do NOT upgrade it
|
||||
# to COMPLETED just because the insert phase ran.
|
||||
if run.status != "FAILED":
|
||||
run.status = "COMPLETED"
|
||||
if insert_result.get("error_message"):
|
||||
run.error_message = insert_result["error_message"]
|
||||
run.completed_at = datetime.now(timezone.utc)
|
||||
@@ -288,7 +292,9 @@ class TranslationOrchestrator:
|
||||
)
|
||||
|
||||
self.db.commit()
|
||||
self.db.refresh(run)
|
||||
# Re-query run after commit — refresh may fail if the object was
|
||||
# created in a different session or became detached during commit.
|
||||
run = self.db.query(TranslationRun).filter(TranslationRun.id == run.id).first()
|
||||
|
||||
logger.reflect("Run execution complete", {
|
||||
"run_id": run.id,
|
||||
@@ -329,11 +335,20 @@ class TranslationOrchestrator:
|
||||
"dialect": job.database_dialect or job.target_dialect,
|
||||
})
|
||||
|
||||
# Build rows for SQL generation
|
||||
# Determine effective target column for INSERT (defaults to translation_column)
|
||||
effective_target = job.target_column or job.translation_column
|
||||
|
||||
# Build columns for SQL generation
|
||||
columns = job.context_columns or []
|
||||
|
||||
# Always include translation_column (original text) if it's different from target
|
||||
if job.translation_column and job.translation_column not in columns:
|
||||
columns.append(job.translation_column)
|
||||
|
||||
# Add target_column separately if it differs from translation_column
|
||||
if effective_target and effective_target != job.translation_column and effective_target not in columns:
|
||||
columns.append(effective_target)
|
||||
|
||||
# Also include key columns if used for upsert
|
||||
if job.target_key_cols:
|
||||
for k in job.target_key_cols:
|
||||
@@ -343,27 +358,57 @@ class TranslationOrchestrator:
|
||||
rows_for_sql = []
|
||||
for rec in records:
|
||||
row_data = {}
|
||||
source_data = rec.source_data or {}
|
||||
|
||||
# Context columns from source data
|
||||
if job.context_columns:
|
||||
for col in job.context_columns:
|
||||
row_data[col] = ""
|
||||
if job.translation_column:
|
||||
row_data[job.translation_column] = rec.target_sql or ""
|
||||
row_data[col] = source_data.get(col, "")
|
||||
|
||||
# Original text column (translation_column)
|
||||
if job.translation_column and job.translation_column not in (job.target_key_cols or []):
|
||||
# If target_column differs, keep the original value from source_data;
|
||||
# otherwise use the translated value
|
||||
if effective_target and effective_target != job.translation_column:
|
||||
row_data[job.translation_column] = source_data.get(job.translation_column, "")
|
||||
else:
|
||||
row_data[job.translation_column] = rec.target_sql or ""
|
||||
|
||||
# Translated text goes into target_column (may be same as translation_column)
|
||||
if effective_target:
|
||||
row_data[effective_target] = rec.target_sql or ""
|
||||
|
||||
# Key columns from source data
|
||||
if job.target_key_cols:
|
||||
source_data = rec.source_data or {}
|
||||
for k in job.target_key_cols:
|
||||
# Use source_data value if available, fall back to empty string
|
||||
row_data[k] = source_data.get(k, "")
|
||||
rows_for_sql.append(row_data)
|
||||
|
||||
if not columns:
|
||||
# Use target_sql as the sole column
|
||||
columns = [job.translation_column or "translated_text"]
|
||||
columns = [effective_target or "translated_text"]
|
||||
rows_for_sql = [{columns[0]: rec.target_sql or ""} for rec in records]
|
||||
|
||||
# Resolve the real database backend engine from Superset
|
||||
try:
|
||||
env_id = job.environment_id or job.source_dialect or ""
|
||||
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
|
||||
executor.resolve_database_id(
|
||||
target_database_id=job.target_database_id,
|
||||
)
|
||||
real_backend = executor.get_database_backend()
|
||||
except Exception as e:
|
||||
logger.explore("Failed to resolve database backend, falling back to job dialet", {
|
||||
"error": str(e),
|
||||
})
|
||||
real_backend = None
|
||||
|
||||
dialect = real_backend or job.database_dialect or job.target_dialect or "postgresql"
|
||||
|
||||
# Generate SQL
|
||||
try:
|
||||
sql, row_count = SQLGenerator.generate(
|
||||
dialect=job.database_dialect or job.target_dialect or "postgresql",
|
||||
dialect=dialect,
|
||||
target_schema=job.target_schema,
|
||||
target_table=job.target_table or "translated_data",
|
||||
columns=columns,
|
||||
@@ -375,19 +420,24 @@ class TranslationOrchestrator:
|
||||
logger.explore("SQL generation failed", {"error": str(e)})
|
||||
return {"status": "failed", "error_message": str(e), "query_id": None}
|
||||
|
||||
logger.reason("SQL generated with dialect", {
|
||||
"dialect": dialect,
|
||||
"real_backend": real_backend,
|
||||
"job_database_dialect": job.database_dialect,
|
||||
"job_target_dialect": job.target_dialect,
|
||||
})
|
||||
|
||||
# Log insert phase start
|
||||
self.event_log.log_event(
|
||||
job_id=job.id,
|
||||
run_id=run.id,
|
||||
event_type="INSERT_PHASE_STARTED",
|
||||
payload={"sql_length": len(sql), "row_count": row_count},
|
||||
payload={"sql_length": len(sql), "row_count": row_count, "dialect": dialect},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
# Submit to Superset
|
||||
try:
|
||||
env_id = job.environment_id or job.source_dialect or ""
|
||||
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
|
||||
result = executor.execute_and_poll(
|
||||
sql=sql,
|
||||
max_polls=30,
|
||||
|
||||
Reference in New Issue
Block a user