busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(llm): add fetch-models endpoint, fix SQL Lab INSERT (client_id truncation, sync mode, target_column, timestamp normalization)

Browse Source 
- Add POST /api/llm/providers/fetch-models route with LLMClient.fetch_models()
- Add target_column to TranslationJob model/schema/service/orchestrator
- Fix SQL Lab execute: truncate client_id to 11 chars (varchar(11))
- Switch SQL Lab to sync mode (runAsync: false) — no Celery workers
- Fix polling: unwrap nested result from Superset query API
- Fix ClickHouse timestamp: normalize float timestamps to YYYY-MM-DD
This commit is contained in:
busya
2026-05-13 20:06:15 +03:00
parent 39ab647851
commit a3c7c402b7
276 changed files with 1923 additions and 1822 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
backend/src/plugins/translate/executor.py
View File

@@ -164,12 +164,102 @@ class TranslationExecutor:
# endregion execute_run
# region _fetch_source_rows [TYPE Function]
# @PURPOSE: Fetch source rows from the accepted preview session for this job.
# @PRE: job_id exists.
# @POST: Returns list of dicts with source data.
# @PURPOSE: Fetch full source dataset from Superset (via datasource) for full translation.
# @PRE: job_id exists. Job may have source_datasource_id for full fetch.
# @POST: Returns list of dicts with source data (all rows from the source datasource).
# @SIDE_EFFECT: Makes HTTP call to Superset chart data API when datasource is configured.
def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]:
with belief_scope("TranslationExecutor._fetch_source_rows"):
# Get the latest APPLIED preview session
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
# If source_datasource_id is configured, fetch ALL rows from the Superset chart data API
if job and job.source_datasource_id:
try:
logger.reason("Fetching full dataset from Superset datasource", {
"run_id": run_id,
"datasource_id": job.source_datasource_id,
"environment_id": job.environment_id,
})
# Determine environment
environments = self.config_manager.get_environments()
target_env_id = job.environment_id or job.source_dialect or ""
env_config = next(
(e for e in environments if e.id == target_env_id),
None,
)
if not env_config and environments:
env_config = environments[0]
if env_config:
from ...core.superset_client import SupersetClient
client = SupersetClient(env_config)
# Fetch dataset detail to build proper query context
dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))
# Build query context (same approach as preview but without row_limit)
query_context = client.build_dataset_preview_query_context(
dataset_id=int(job.source_datasource_id),
dataset_record=dataset_detail,
template_params={},
effective_filters=[],
)
# Remove row_limit to get ALL rows; use result_type="samples"
queries = query_context.get("queries", [])
if queries:
queries[0].pop("row_limit", None)
queries[0].pop("result_type", None)
queries[0]["metrics"] = []
query_context["result_type"] = "samples"
form_data = query_context.get("form_data", {})
form_data.pop("query_mode", None)
response = client.network.request(
method="POST",
endpoint="/api/v1/chart/data",
data=json.dumps(query_context),
headers={"Content-Type": "application/json"},
)
# Extract rows
rows = self._extract_chart_data_rows(response)
if rows:
logger.reason(f"Fetched {len(rows)} rows from Superset datasource", {
"run_id": run_id,
})
# Map rows to source_rows format
source_rows = []
for idx, row in enumerate(rows):
source_data_dict = dict(row) if row else None
source_text = str(row.get(job.translation_column, "")) if job.translation_column else json.dumps(row)
source_rows.append({
"row_index": str(idx),
"source_text": source_text,
"approved_translation": None,
"source_object_name": f"Row {idx}",
"source_data": source_data_dict,
})
return source_rows
else:
logger.explore("Superset datasource returned no rows", {
"run_id": run_id,
"datasource_id": job.source_datasource_id,
})
else:
logger.explore("No environment config found for datasource fetch", {
"env_id": target_env_id,
})
except Exception as e:
logger.explore("Failed to fetch full dataset from Superset, falling back to preview", {
"run_id": run_id,
"error": str(e),
})
# Fall through to preview-based fetch
# Fallback: get the latest APPLIED preview session
session = (
self.db.query(TranslationPreviewSession)
.filter(
@@ -195,20 +285,48 @@ class TranslationExecutor:
source_rows = []
for rec in records:
source_data_dict = None
if hasattr(rec, "source_data") and rec.source_data:
source_data_dict = dict(rec.source_data)
source_rows.append({
"row_index": rec.source_object_id or "0",
"source_text": rec.source_sql or "",
"approved_translation": rec.target_sql if rec.status == "APPROVED" else None,
"source_object_name": rec.source_object_name or "",
"source_data": source_data_dict,
})
logger.reason(f"Fetched {len(source_rows)} source rows from preview", {
logger.reason(f"Fetched {len(source_rows)} source rows from preview fallback", {
"run_id": run_id,
"session_id": session.id,
})
return source_rows
# endregion _fetch_source_rows
# region _extract_chart_data_rows [TYPE Function]
# @PURPOSE: Extract data rows from Superset chart data API response.
# @POST: Returns list of dicts with column-value pairs.
@staticmethod
def _extract_chart_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]:
result = response.get("result")
if isinstance(result, list):
for item in result:
if isinstance(item, dict):
data = item.get("data")
if isinstance(data, list) and data:
return data
if isinstance(result, dict):
data = result.get("data")
if isinstance(data, list) and data:
return data
data = response.get("data")
if isinstance(data, list) and data:
return data
if isinstance(result, list):
return result
return []
# endregion _extract_chart_data_rows
# region _process_batch [TYPE Function]
# @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records.
# @PRE: job and batch_rows are valid.
@@ -272,6 +390,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data"),
status="SUCCESS",
)
self.db.add(record)
@@ -398,6 +517,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data"),
status="FAILED",
error_message=f"LLM call failed after {retries} retries: {last_error}",
)
@@ -425,6 +545,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data"),
status="SKIPPED",
error_message=f"LLM parse failure: {e}",
)
@@ -456,6 +577,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data"),
status="SKIPPED",
error_message="NULL translation returned by LLM",
)
@@ -474,6 +596,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data"),
status="SKIPPED",
error_message="Empty translation returned by LLM",
)
@@ -490,6 +613,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data"),
status="SUCCESS",
)
self.db.add(record)