busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

semantics: complete DEF-to-region migration, fix regressions

Browse Source 
- Convert legacy [DEF🆔Type] anchors to #region/#endregion across 329 files
- Reinstate _normalize_timestamp_value in sql_generator.py
- Fix MarkerLogger→logger migration in events.py (molecular CoT markers)
- Fix dataset_review orchestrator dependencies (_build_execution_snapshot)
- Fix config_manager stale-record deletion (moved to save path only)
- Add 77 missing [/DEF:] closers in 5 unbalanced test files
- Update assistant_chat.integration.test.js for #region format
- Apply molecular-cot-logging markers (REASON/REFLECT/EXPLORE) via logger.* methods
This commit is contained in:
busya
2026-05-12 23:54:55 +03:00
parent fe8978f716
commit 306c5ae742
331 changed files with 9630 additions and 10312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

285
backend/src/plugins/translate/executor.py
View File

@@ -1,19 +1,17 @@
# #region TranslationExecutor [C:5] [TYPE Module] [SEMANTICS translate,executor,batch,llm]
# #region TranslationExecutor [C:4] [TYPE Module] [SEMANTICS translate, executor, batch, llm]
# @BRIEF Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows.
# @LAYER Domain
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationBatch]
# @RELATION DEPENDS_ON -> [TranslationRecord]
# @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [DictionaryManager]
# @RELATION DEPENDS_ON -> [TranslationPreview]
# @PRE Valid TranslationRun with job configuration. DB session is available.
# @POST TranslationBatch and TranslationRecord rows are created. Run status is updated.
# @SIDE_EFFECT Calls LLM provider; creates DB rows; updates run statistics.
# @DATA_CONTRACT Input[TranslationRun, job] -> Output[TranslationRun with batches and records]
# @INVARIANT Batch processing with retry — independent batches allow partial recovery; MAX_RETRIES_PER_BATCH = 3.
# @RATIONALE Batch processing with retry — independent batches allow partial recovery.
# @REJECTED Single monolithic LLM call — would lose all progress on any failure.
# @PRE: Valid TranslationRun with job configuration. DB session is available.
# @POST: TranslationBatch and TranslationRecord rows are created. Run status is updated.
# @SIDE_EFFECT: Calls LLM provider; creates DB rows; updates run statistics.
# @RATIONALE: Batch processing with retry — independent batches allow partial recovery.
# @REJECTED: Single monolithic LLM call — would lose all progress on any failure.
import json
import time
@@ -24,7 +22,6 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Callable
from sqlalchemy.orm import Session
from ...core.logger import logger, belief_scope
from ...core.cot_logger import MarkerLogger
from ...core.config_manager import ConfigManager
from ...models.translate import (
TranslationJob,
@@ -36,24 +33,20 @@ from ...models.translate import (
)
from ...services.llm_provider import LLMProviderService
from ...services.llm_prompt_templates import render_prompt
from ...core.superset_client import SupersetClient
from .dictionary import DictionaryManager
from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE
log = MarkerLogger("TranslationExecutor")
# #region MAX_RETRIES_PER_BATCH [C:1] [TYPE Constant] [SEMANTICS translate,executor,retry]
# #region MAX_RETRIES_PER_BATCH [TYPE Constant]
# @BRIEF Maximum number of retries for a single batch before marking it failed.
MAX_RETRIES_PER_BATCH = 3
# #endregion MAX_RETRIES_PER_BATCH
# #region TranslationExecutor [C:5] [TYPE Class] [SEMANTICS translate,executor,batch]
# #region TranslationExecutor [C:4] [TYPE Class]
# @BRIEF Process translation batches: fetch source rows, filter dict, call LLM, persist results.
# @PRE DB session and config manager available.
# @POST Batches and records created with status tracking; run statistics updated.
# @SIDE_EFFECT LLM API calls; DB writes.
# @DATA_CONTRACT Input[TranslationRun] -> Output[TranslationRun with batches, records, stats]
# @INVARIANT MAX_RETRIES_PER_BATCH limit enforced; partial batch success tracked via COMPLETED_WITH_ERRORS.
# @PRE: DB session and config manager available.
# @POST: Batches and records created with status tracking.
# @SIDE_EFFECT: LLM API calls; DB writes.
class TranslationExecutor:
def __init__(
@@ -69,27 +62,25 @@ class TranslationExecutor:
self.on_batch_progress = on_batch_progress
self._current_run_id: Optional[str] = None
# #region execute_run [C:4] [TYPE Function] [SEMANTICS translate,run,execute]
# @BRIEF Run full translation execution for a TranslationRun: fetch rows, batch, call LLM, persist.
# @PRE run is in PENDING or RUNNING status with valid job config.
# @POST Run is populated with batches and records; run status updated to COMPLETED/FAILED.
# @SIDE_EFFECT LLM API calls; DB batch writes.
# [DEF:execute_run:Function]
# @PURPOSE: Run full translation execution for a TranslationRun.
# @PRE: run is in PENDING or RUNNING status with valid job config.
# @POST: Run is populated with batches and records.
# @SIDE_EFFECT: LLM API calls; DB batch writes.
def execute_run(
self,
run: TranslationRun,
llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None,
full_translation: bool = False,
) -> TranslationRun:
with belief_scope("TranslationExecutor.execute_run"):
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
if not job:
raise ValueError(f"Job '{run.job_id}' not found for run '{run.id}'")
log.reason("Starting translation execution", payload={
logger.reason("Starting translation execution", {
"run_id": run.id,
"job_id": job.id,
"batch_size": job.batch_size,
"full_translation": full_translation,
})
# Mark run as RUNNING
@@ -97,15 +88,10 @@ class TranslationExecutor:
run.started_at = datetime.now(timezone.utc)
self.db.flush()
# Fetch source rows
if full_translation:
# Full translation: fetch ALL rows from Superset dataset
source_rows = self._fetch_all_rows_from_superset(job)
else:
# Preview-based: fetch rows from the accepted preview session
source_rows = self._fetch_source_rows(job.id, run.id)
# Fetch source rows from the accepted preview session
source_rows = self._fetch_source_rows(job.id, run.id)
if not source_rows:
log.explore("No source rows to translate", payload={"run_id": run.id}, error="Preview produced 0 source rows for translation")
logger.explore("No source rows to translate", {"run_id": run.id})
run.status = "COMPLETED"
run.completed_at = datetime.now(timezone.utc)
self.db.flush()
@@ -121,7 +107,7 @@ class TranslationExecutor:
for i in range(0, total_rows, batch_size)
]
log.reason(f"Processing {len(batches)} batches", payload={
logger.reason(f"Processing {len(batches)} batches", {
"run_id": run.id,
"total_rows": total_rows,
"batch_size": batch_size,
@@ -165,7 +151,7 @@ class TranslationExecutor:
run.completed_at = datetime.now(timezone.utc)
self.db.flush()
log.reflect("Translation execution complete", payload={
logger.reflect("Translation execution complete", {
"run_id": run.id,
"status": run.status,
"total": total_rows,
@@ -175,13 +161,12 @@ class TranslationExecutor:
})
return run
# #endregion execute_run
# [/DEF:execute_run:Function]
# #region _fetch_source_rows [C:4] [TYPE Function] [SEMANTICS translate,source,rows]
# @BRIEF Fetch source rows from the accepted preview session for this job.
# @PRE job_id exists.
# @POST Returns list of dicts with source data (row_index, source_text, approved_translation, etc).
# @SIDE_EFFECT Queries preview session and records from DB.
# [DEF:_fetch_source_rows:Function]
# @PURPOSE: Fetch source rows from the accepted preview session for this job.
# @PRE: job_id exists.
# @POST: Returns list of dicts with source data.
def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]:
with belief_scope("TranslationExecutor._fetch_source_rows"):
# Get the latest APPLIED preview session
@@ -195,7 +180,7 @@ class TranslationExecutor:
.first()
)
if not session:
log.explore("No accepted preview session found", error="Preview session has no accepted rows", payload={"job_id": job_id})
logger.explore("No accepted preview session found", {"job_id": job_id})
return []
# Fetch APPROVED or all records from the session
@@ -215,124 +200,20 @@ class TranslationExecutor:
"source_text": rec.source_sql or "",
"approved_translation": rec.target_sql if rec.status == "APPROVED" else None,
"source_object_name": rec.source_object_name or "",
"source_data": rec.source_data or {},
})
log.reason(f"Fetched {len(source_rows)} source rows from preview", payload={
logger.reason(f"Fetched {len(source_rows)} source rows from preview", {
"run_id": run_id,
"session_id": session.id,
})
return source_rows
# #endregion _fetch_source_rows
# [/DEF:_fetch_source_rows:Function]
# #region _fetch_all_rows_from_superset [C:4] [TYPE Function] [SEMANTICS translate,superset,rows]
# @BRIEF Fetch ALL rows from the Superset dataset for full translation (paginated).
# @PRE job has source_datasource_id and environment_id configured.
# @POST Returns list of row dicts with row_index, source_text, source_data.
# @SIDE_EFFECT Calls Superset chart data endpoint (paginated).
def _fetch_all_rows_from_superset(self, job: TranslationJob) -> List[Dict[str, Any]]:
with belief_scope("TranslationExecutor._fetch_all_rows_from_superset"):
env_id = job.environment_id or job.source_dialect or ""
environments = self.config_manager.get_environments()
env_config = next(
(e for e in environments if e.id == env_id),
None,
)
if not env_config:
raise ValueError(f"Superset environment '{env_id}' not found")
client = SupersetClient(env_config)
dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))
# Build query context (same as preview, but with large row_limit)
query_context = client.build_dataset_preview_query_context(
dataset_id=int(job.source_datasource_id),
dataset_record=dataset_detail,
template_params={},
effective_filters=[],
)
queries = query_context.get("queries", [])
if queries:
queries[0]["row_limit"] = 100000 # Superset max default
queries[0].pop("result_type", None)
queries[0].pop("columns", None)
queries[0]["metrics"] = []
queries[0]["row_offset"] = 0
query_context["result_type"] = "samples"
form_data = query_context.get("form_data", {})
form_data.pop("query_mode", None)
all_rows: List[Dict[str, Any]] = []
batch_size_fetch = 10000 # Fetch 10K rows per pagination request
while True:
# Set pagination for this batch
if queries:
queries[0]["row_limit"] = min(batch_size_fetch, 100000)
queries[0]["row_offset"] = len(all_rows)
try:
response = client.network.request(
method="POST",
endpoint="/api/v1/chart/data",
data=json.dumps(query_context),
headers={"Content-Type": "application/json"},
)
except Exception as e:
log.explore("Chart data API failed during full fetch", error="Chart data API error",
payload={
"offset": len(all_rows),
"error": str(e),
})
if not all_rows:
raise ValueError(f"Failed to fetch data from Superset: {e}")
break # Return what we have
from .preview import TranslationPreview
rows = TranslationPreview._extract_data_rows(response)
if not rows:
break # No more data
all_rows.extend(rows)
log.reason(f"Fetched {len(rows)} rows (total: {len(all_rows)})", payload={
"offset": len(all_rows) - len(rows),
})
if len(rows) < batch_size_fetch:
break # Last page
# Convert to the format expected by _process_batch
source_rows = []
for idx, row in enumerate(all_rows):
translation_value = str(row.get(job.translation_column, "") or "")
context_values = {}
if job.context_columns:
for col in job.context_columns:
context_values[col] = str(row.get(col, "") or "")
# Also include target_key_cols values in context_data
if job.target_key_cols:
for col in job.target_key_cols:
context_values[col] = str(row.get(col, "") or "")
source_rows.append({
"row_index": str(idx),
"source_text": translation_value,
"source_data": context_values,
"source_object_name": f"Row {idx + 1}",
"approved_translation": None,
})
log.reason(f"Prepared {len(source_rows)} source rows for full translation", payload={
"job_id": job.id,
})
return source_rows
# #endregion _fetch_all_rows_from_superset
# #region _process_batch [C:4] [TYPE Function] [SEMANTICS translate,batch,process]
# @BRIEF Process a single batch: filter dictionary, build prompt, call LLM, persist records.
# @PRE job and batch_rows are valid.
# @POST TranslationBatch and TranslationRecord rows are created.
# @SIDE_EFFECT LLM API call; DB writes.
# [DEF:_process_batch:Function]
# @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records.
# @PRE: job and batch_rows are valid.
# @POST: TranslationBatch and TranslationRecord rows are created.
# @SIDE_EFFECT: LLM API call.
def _process_batch(
self,
job: TranslationJob,
@@ -391,7 +272,6 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SUCCESS",
)
self.db.add(record)
@@ -419,20 +299,20 @@ class TranslationExecutor:
self.db.flush()
batch_latency = int((time.monotonic() - batch_start) * 1000)
log.reason(f"Batch {batch_index} complete", payload={
logger.reason(f"Batch {batch_index} complete", {
"batch_id": batch_id,
"latency_ms": batch_latency,
**result,
})
return result
# #endregion _process_batch
# [/DEF:_process_batch:Function]
# #region _call_llm_for_batch [C:4] [TYPE Function] [SEMANTICS translate,llm,batch]
# @BRIEF Call LLM for a batch of rows requiring translation. Parse structured JSON response.
# @PRE job has valid provider_id. batch_rows is non-empty.
# @POST Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.
# @SIDE_EFFECT HTTP call to LLM provider.
# [DEF:_call_llm_for_batch:Function]
# @PURPOSE: Call LLM for a batch of rows requiring translation. Parse structured JSON response.
# @PRE: job has valid provider_id. batch_rows is non-empty.
# @POST: Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.
# @SIDE_EFFECT: HTTP call to LLM provider.
def _call_llm_for_batch(
self,
job: TranslationJob,
@@ -493,8 +373,7 @@ class TranslationExecutor:
except Exception as e:
last_error = str(e)
retries += 1
log.explore("LLM call failed", error="LLM call failed, retrying",
payload={
logger.explore(f"LLM call failed (attempt {attempt})", {
"batch_id": batch_id,
"error": last_error,
"attempt": attempt,
@@ -502,8 +381,7 @@ class TranslationExecutor:
if attempt < MAX_RETRIES_PER_BATCH:
time.sleep(2 ** attempt) # Exponential backoff
else:
log.explore("LLM call exhausted retries", error="LLM retries exhausted",
payload={
logger.explore("LLM call exhausted retries", {
"batch_id": batch_id,
"last_error": last_error,
})
@@ -520,7 +398,6 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="FAILED",
error_message=f"LLM call failed after {retries} retries: {last_error}",
)
@@ -532,8 +409,7 @@ class TranslationExecutor:
translations = self._parse_llm_response(llm_response, len(batch_rows))
except ValueError as e:
# Parse failure — mark all rows as SKIPPED
log.explore("LLM response parse failed", error="Failed to parse LLM JSON response",
payload={
logger.explore("LLM response parse failed", {
"batch_id": batch_id,
"error": str(e),
"response_preview": llm_response[:500] if llm_response else "",
@@ -549,7 +425,6 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED",
error_message=f"LLM parse failure: {e}",
)
@@ -581,7 +456,6 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED",
error_message="NULL translation returned by LLM",
)
@@ -600,7 +474,6 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED",
error_message="Empty translation returned by LLM",
)
@@ -617,7 +490,6 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SUCCESS",
)
self.db.add(record)
@@ -628,13 +500,13 @@ class TranslationExecutor:
"skipped": skipped,
"retries": retries,
}
# #endregion _call_llm_for_batch
# [/DEF:_call_llm_for_batch:Function]
# #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call]
# @BRIEF Call the configured LLM provider with the batch prompt, routing by provider type.
# @PRE job has valid provider_id.
# @POST Returns raw LLM response string.
# @SIDE_EFFECT HTTP call to LLM provider.
# [DEF:_call_llm:Function]
# @PURPOSE: Call the configured LLM provider with the batch prompt.
# @PRE: job has valid provider_id.
# @POST: Returns raw LLM response string.
# @SIDE_EFFECT: HTTP call to LLM provider.
def _call_llm(self, job: TranslationJob, prompt: str) -> str:
with belief_scope("TranslationExecutor._call_llm"):
if not job.provider_id:
@@ -662,13 +534,13 @@ class TranslationExecutor:
)
else:
raise ValueError(f"Unsupported provider type '{provider_type}'")
# #endregion _call_llm
# [/DEF:_call_llm:Function]
# #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai]
# @BRIEF Call OpenAI-compatible API for batch translation with retry and structured output support.
# @PRE Valid API endpoint, key, model, and prompt.
# @POST Returns response text.
# @SIDE_EFFECT HTTP POST to LLM API.
# [DEF:_call_openai_compatible:Function]
# @PURPOSE: Call OpenAI-compatible API for batch translation.
# @PRE: Valid API endpoint, key, model, and prompt.
# @POST: Returns response text.
# @SIDE_EFFECT: HTTP POST to LLM API.
@staticmethod
def _call_openai_compatible(
base_url: str,
@@ -699,7 +571,7 @@ class TranslationExecutor:
if provider_type in ("openai", "openai_compatible"):
payload["response_format"] = {"type": "json_object"}
log.reason(
logger.reason(
f"LLM request model={payload.get('model')} "
f"provider_type={provider_type} "
f"response_format={'yes' if 'response_format' in payload else 'no'} "
@@ -707,11 +579,11 @@ class TranslationExecutor:
)
response = http_requests.post(url, headers=headers, json=payload, timeout=180)
if not response.ok:
log.explore("LLM API error", error=f"LLM API returned status {response.status_code}", payload={
"status_code": response.status_code,
"model": payload.get('model'),
"body": response.text[:2000],
})
logger.explore(
f"LLM API error status={response.status_code} "
f"model={payload.get('model')} "
f"body={response.text[:2000]}"
)
response.raise_for_status()
data = response.json()
@@ -721,27 +593,15 @@ class TranslationExecutor:
content = choices[0].get("message", {}).get("content", "")
if not content:
# Log full response for diagnostics
finish_reason = choices[0].get("finish_reason", "unknown")
log.explore("LLM returned empty content", error="Empty response from LLM",
payload={
"finish_reason": finish_reason,
"model": payload.get("model"),
"prompt_len": len(prompt),
"response_keys": list(data.keys()),
"choices_count": len(choices),
})
raise ValueError(
f"LLM returned empty content (finish_reason={finish_reason}, "
f"model={payload.get('model')}, prompt_len={len(prompt)})"
)
raise ValueError("LLM returned empty content")
return content
# #endregion _call_openai_compatible
# [/DEF:_call_openai_compatible:Function]
# #region _parse_llm_response [C:3] [TYPE Function] [SEMANTICS translate,llm,parse]
# @BRIEF Parse LLM JSON response into dict of row_id -> translation text.
# @RELATION DEPENDS_ON -> [json]
# [DEF:_parse_llm_response:Function]
# @PURPOSE: Parse LLM JSON response into dict of row_id -> translation.
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
# @POST: Returns dict mapping row_id to translation text.
@staticmethod
def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:
with belief_scope("TranslationExecutor._parse_llm_response"):
@@ -774,8 +634,9 @@ class TranslationExecutor:
translations[row_id] = str(translation)
return translations
# #endregion _parse_llm_response
# [/DEF:_parse_llm_response:Function]
# #endregion TranslationExecutor
# #endregion TranslationExecutor
# #endregion TranslationExecutor