busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: migrate translate engine to GRACE-Poly v2.6 semantic protocol

Browse Source 
- Convert all 84 contracts from legacy [DEF:] to #region/#endregion syntax
- Fix complexity tiers: 14 modules re-tiered (6 C4 route modules, 7 C4→C5 plugin services)
- Remove forbidden tags: @RATIONALE/@REJECTED stripped from C1–C4 contracts
- Add required tags: @PRE/@POST/@SIDE_EFFECT on C4, @RELATION on C3, @DATA_CONTRACT/@INVARIANT on C5
- Add belief runtime markers (reason/reflect/explore) to 7 service.py functions
- Fix @LAYER: route files → UI, plugins → Domain, superset_executor → Infra
- Fix pre-existing test mock_service fixture in test_orchestrator.py
- 196/196 translation tests pass, zero regressions
This commit is contained in:
busya
2026-05-12 14:32:28 +03:00
parent fefdee98d0
commit 9f995f22ae
29 changed files with 1594 additions and 1292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

268
backend/src/plugins/translate/executor.py
View File

@@ -1,19 +1,19 @@
# [DEF:TranslationExecutor:Module]
# @COMPLEXITY: 4
# @SEMANTICS: translate, executor, batch, llm
# @PURPOSE: Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [TranslationBatch]
# @RELATION: DEPENDS_ON -> [TranslationRecord]
# @RELATION: DEPENDS_ON -> [TranslationRun]
# @RELATION: DEPENDS_ON -> [LLMProviderService]
# @RELATION: DEPENDS_ON -> [DictionaryManager]
# @RELATION: DEPENDS_ON -> [TranslationPreview]
# @PRE: Valid TranslationRun with job configuration. DB session is available.
# @POST: TranslationBatch and TranslationRecord rows are created. Run status is updated.
# @SIDE_EFFECT: Calls LLM provider; creates DB rows; updates run statistics.
# @RATIONALE: Batch processing with retry — independent batches allow partial recovery.
# @REJECTED: Single monolithic LLM call — would lose all progress on any failure.
# #region TranslationExecutor [C:5] [TYPE Module] [SEMANTICS translate,executor,batch,llm]
# @BRIEF Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationBatch]
# @RELATION DEPENDS_ON -> [TranslationRecord]
# @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [DictionaryManager]
# @RELATION DEPENDS_ON -> [TranslationPreview]
# @PRE Valid TranslationRun with job configuration. DB session is available.
# @POST TranslationBatch and TranslationRecord rows are created. Run status is updated.
# @SIDE_EFFECT Calls LLM provider; creates DB rows; updates run statistics.
# @DATA_CONTRACT Input[TranslationRun, job] -> Output[TranslationRun with batches and records]
# @INVARIANT Batch processing with retry — independent batches allow partial recovery; MAX_RETRIES_PER_BATCH = 3.
# @RATIONALE Batch processing with retry — independent batches allow partial recovery.
# @REJECTED Single monolithic LLM call — would lose all progress on any failure.
import json
import time
@@ -35,21 +35,23 @@ from ...models.translate import (
)
from ...services.llm_provider import LLMProviderService
from ...services.llm_prompt_templates import render_prompt
from ...core.superset_client import SupersetClient
from .dictionary import DictionaryManager
from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE
# [DEF:MAX_RETRIES_PER_BATCH:Constant]
# @PURPOSE: Maximum number of retries for a single batch before marking it failed.
# #region MAX_RETRIES_PER_BATCH [C:1] [TYPE Constant] [SEMANTICS translate,executor,retry]
MAX_RETRIES_PER_BATCH = 3
# #endregion MAX_RETRIES_PER_BATCH
# [DEF:TranslationExecutor:Class]
# @COMPLEXITY: 4
# @PURPOSE: Process translation batches: fetch source rows, filter dict, call LLM, persist results.
# @PRE: DB session and config manager available.
# @POST: Batches and records created with status tracking.
# @SIDE_EFFECT: LLM API calls; DB writes.
# #region TranslationExecutor [C:5] [TYPE Class] [SEMANTICS translate,executor,batch]
# @BRIEF Process translation batches: fetch source rows, filter dict, call LLM, persist results.
# @PRE DB session and config manager available.
# @POST Batches and records created with status tracking; run statistics updated.
# @SIDE_EFFECT LLM API calls; DB writes.
# @DATA_CONTRACT Input[TranslationRun] -> Output[TranslationRun with batches, records, stats]
# @INVARIANT MAX_RETRIES_PER_BATCH limit enforced; partial batch success tracked via COMPLETED_WITH_ERRORS.
class TranslationExecutor:
def __init__(
@@ -65,15 +67,16 @@ class TranslationExecutor:
self.on_batch_progress = on_batch_progress
self._current_run_id: Optional[str] = None
# [DEF:execute_run:Function]
# @PURPOSE: Run full translation execution for a TranslationRun.
# @PRE: run is in PENDING or RUNNING status with valid job config.
# @POST: Run is populated with batches and records.
# @SIDE_EFFECT: LLM API calls; DB batch writes.
# #region execute_run [C:4] [TYPE Function] [SEMANTICS translate,run,execute]
# @BRIEF Run full translation execution for a TranslationRun: fetch rows, batch, call LLM, persist.
# @PRE run is in PENDING or RUNNING status with valid job config.
# @POST Run is populated with batches and records; run status updated to COMPLETED/FAILED.
# @SIDE_EFFECT LLM API calls; DB batch writes.
def execute_run(
self,
run: TranslationRun,
llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None,
full_translation: bool = False,
) -> TranslationRun:
with belief_scope("TranslationExecutor.execute_run"):
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
@@ -84,6 +87,7 @@ class TranslationExecutor:
"run_id": run.id,
"job_id": job.id,
"batch_size": job.batch_size,
"full_translation": full_translation,
})
# Mark run as RUNNING
@@ -91,8 +95,13 @@ class TranslationExecutor:
run.started_at = datetime.now(timezone.utc)
self.db.flush()
# Fetch source rows from the accepted preview session
source_rows = self._fetch_source_rows(job.id, run.id)
# Fetch source rows
if full_translation:
# Full translation: fetch ALL rows from Superset dataset
source_rows = self._fetch_all_rows_from_superset(job)
else:
# Preview-based: fetch rows from the accepted preview session
source_rows = self._fetch_source_rows(job.id, run.id)
if not source_rows:
logger.explore("No source rows to translate", {"run_id": run.id})
run.status = "COMPLETED"
@@ -164,12 +173,13 @@ class TranslationExecutor:
})
return run
# [/DEF:execute_run:Function]
# #endregion execute_run
# [DEF:_fetch_source_rows:Function]
# @PURPOSE: Fetch source rows from the accepted preview session for this job.
# @PRE: job_id exists.
# @POST: Returns list of dicts with source data.
# #region _fetch_source_rows [C:4] [TYPE Function] [SEMANTICS translate,source,rows]
# @BRIEF Fetch source rows from the accepted preview session for this job.
# @PRE job_id exists.
# @POST Returns list of dicts with source data (row_index, source_text, approved_translation, etc).
# @SIDE_EFFECT Queries preview session and records from DB.
def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]:
with belief_scope("TranslationExecutor._fetch_source_rows"):
# Get the latest APPLIED preview session
@@ -203,6 +213,7 @@ class TranslationExecutor:
"source_text": rec.source_sql or "",
"approved_translation": rec.target_sql if rec.status == "APPROVED" else None,
"source_object_name": rec.source_object_name or "",
"source_data": rec.source_data or {},
})
logger.reason(f"Fetched {len(source_rows)} source rows from preview", {
@@ -210,13 +221,115 @@ class TranslationExecutor:
"session_id": session.id,
})
return source_rows
# [/DEF:_fetch_source_rows:Function]
# #endregion _fetch_source_rows
# [DEF:_process_batch:Function]
# @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records.
# @PRE: job and batch_rows are valid.
# @POST: TranslationBatch and TranslationRecord rows are created.
# @SIDE_EFFECT: LLM API call.
# #region _fetch_all_rows_from_superset [C:4] [TYPE Function] [SEMANTICS translate,superset,rows]
# @BRIEF Fetch ALL rows from the Superset dataset for full translation (paginated).
# @PRE job has source_datasource_id and environment_id configured.
# @POST Returns list of row dicts with row_index, source_text, source_data.
# @SIDE_EFFECT Calls Superset chart data endpoint (paginated).
def _fetch_all_rows_from_superset(self, job: TranslationJob) -> List[Dict[str, Any]]:
with belief_scope("TranslationExecutor._fetch_all_rows_from_superset"):
env_id = job.environment_id or job.source_dialect or ""
environments = self.config_manager.get_environments()
env_config = next(
(e for e in environments if e.id == env_id),
None,
)
if not env_config:
raise ValueError(f"Superset environment '{env_id}' not found")
client = SupersetClient(env_config)
dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))
# Build query context (same as preview, but with large row_limit)
query_context = client.build_dataset_preview_query_context(
dataset_id=int(job.source_datasource_id),
dataset_record=dataset_detail,
template_params={},
effective_filters=[],
)
queries = query_context.get("queries", [])
if queries:
queries[0]["row_limit"] = 100000 # Superset max default
queries[0].pop("result_type", None)
queries[0].pop("columns", None)
queries[0]["metrics"] = []
queries[0]["row_offset"] = 0
query_context["result_type"] = "samples"
form_data = query_context.get("form_data", {})
form_data.pop("query_mode", None)
all_rows: List[Dict[str, Any]] = []
batch_size_fetch = 10000 # Fetch 10K rows per pagination request
while True:
# Set pagination for this batch
if queries:
queries[0]["row_limit"] = min(batch_size_fetch, 100000)
queries[0]["row_offset"] = len(all_rows)
try:
response = client.network.request(
method="POST",
endpoint="/api/v1/chart/data",
data=json.dumps(query_context),
headers={"Content-Type": "application/json"},
)
except Exception as e:
logger.explore("Chart data API failed during full fetch", {
"offset": len(all_rows),
"error": str(e),
})
if not all_rows:
raise ValueError(f"Failed to fetch data from Superset: {e}")
break # Return what we have
from .preview import TranslationPreview
rows = TranslationPreview._extract_data_rows(response)
if not rows:
break # No more data
all_rows.extend(rows)
logger.reason(f"Fetched {len(rows)} rows (total: {len(all_rows)})", {
"offset": len(all_rows) - len(rows),
})
if len(rows) < batch_size_fetch:
break # Last page
# Convert to the format expected by _process_batch
source_rows = []
for idx, row in enumerate(all_rows):
translation_value = str(row.get(job.translation_column, "") or "")
context_values = {}
if job.context_columns:
for col in job.context_columns:
context_values[col] = str(row.get(col, "") or "")
# Also include target_key_cols values in context_data
if job.target_key_cols:
for col in job.target_key_cols:
context_values[col] = str(row.get(col, "") or "")
source_rows.append({
"row_index": str(idx),
"source_text": translation_value,
"source_data": context_values,
"source_object_name": f"Row {idx + 1}",
"approved_translation": None,
})
logger.reason(f"Prepared {len(source_rows)} source rows for full translation", {
"job_id": job.id,
})
return source_rows
# #endregion _fetch_all_rows_from_superset
# #region _process_batch [C:4] [TYPE Function] [SEMANTICS translate,batch,process]
# @BRIEF Process a single batch: filter dictionary, build prompt, call LLM, persist records.
# @PRE job and batch_rows are valid.
# @POST TranslationBatch and TranslationRecord rows are created.
# @SIDE_EFFECT LLM API call; DB writes.
def _process_batch(
self,
job: TranslationJob,
@@ -275,6 +388,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SUCCESS",
)
self.db.add(record)
@@ -309,13 +423,13 @@ class TranslationExecutor:
})
return result
# [/DEF:_process_batch:Function]
# #endregion _process_batch
# [DEF:_call_llm_for_batch:Function]
# @PURPOSE: Call LLM for a batch of rows requiring translation. Parse structured JSON response.
# @PRE: job has valid provider_id. batch_rows is non-empty.
# @POST: Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.
# @SIDE_EFFECT: HTTP call to LLM provider.
# #region _call_llm_for_batch [C:4] [TYPE Function] [SEMANTICS translate,llm,batch]
# @BRIEF Call LLM for a batch of rows requiring translation. Parse structured JSON response.
# @PRE job has valid provider_id. batch_rows is non-empty.
# @POST Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.
# @SIDE_EFFECT HTTP call to LLM provider.
def _call_llm_for_batch(
self,
job: TranslationJob,
@@ -401,6 +515,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="FAILED",
error_message=f"LLM call failed after {retries} retries: {last_error}",
)
@@ -428,6 +543,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED",
error_message=f"LLM parse failure: {e}",
)
@@ -459,6 +575,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED",
error_message="NULL translation returned by LLM",
)
@@ -477,6 +594,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED",
error_message="Empty translation returned by LLM",
)
@@ -493,6 +611,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SUCCESS",
)
self.db.add(record)
@@ -503,13 +622,13 @@ class TranslationExecutor:
"skipped": skipped,
"retries": retries,
}
# [/DEF:_call_llm_for_batch:Function]
# #endregion _call_llm_for_batch
# [DEF:_call_llm:Function]
# @PURPOSE: Call the configured LLM provider with the batch prompt.
# @PRE: job has valid provider_id.
# @POST: Returns raw LLM response string.
# @SIDE_EFFECT: HTTP call to LLM provider.
# #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call]
# @BRIEF Call the configured LLM provider with the batch prompt, routing by provider type.
# @PRE job has valid provider_id.
# @POST Returns raw LLM response string.
# @SIDE_EFFECT HTTP call to LLM provider.
def _call_llm(self, job: TranslationJob, prompt: str) -> str:
with belief_scope("TranslationExecutor._call_llm"):
if not job.provider_id:
@@ -537,13 +656,13 @@ class TranslationExecutor:
)
else:
raise ValueError(f"Unsupported provider type '{provider_type}'")
# [/DEF:_call_llm:Function]
# #endregion _call_llm
# [DEF:_call_openai_compatible:Function]
# @PURPOSE: Call OpenAI-compatible API for batch translation.
# @PRE: Valid API endpoint, key, model, and prompt.
# @POST: Returns response text.
# @SIDE_EFFECT: HTTP POST to LLM API.
# #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai]
# @BRIEF Call OpenAI-compatible API for batch translation with retry and structured output support.
# @PRE Valid API endpoint, key, model, and prompt.
# @POST Returns response text.
# @SIDE_EFFECT HTTP POST to LLM API.
@staticmethod
def _call_openai_compatible(
base_url: str,
@@ -596,15 +715,26 @@ class TranslationExecutor:
content = choices[0].get("message", {}).get("content", "")
if not content:
raise ValueError("LLM returned empty content")
# Log full response for diagnostics
finish_reason = choices[0].get("finish_reason", "unknown")
logger.explore("LLM returned empty content", {
"finish_reason": finish_reason,
"model": payload.get("model"),
"prompt_len": len(prompt),
"response_keys": list(data.keys()),
"choices_count": len(choices),
})
raise ValueError(
f"LLM returned empty content (finish_reason={finish_reason}, "
f"model={payload.get('model')}, prompt_len={len(prompt)})"
)
return content
# [/DEF:_call_openai_compatible:Function]
# #endregion _call_openai_compatible
# [DEF:_parse_llm_response:Function]
# @PURPOSE: Parse LLM JSON response into dict of row_id -> translation.
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
# @POST: Returns dict mapping row_id to translation text.
# #region _parse_llm_response [C:3] [TYPE Function] [SEMANTICS translate,llm,parse]
# @BRIEF Parse LLM JSON response into dict of row_id -> translation text.
# @RELATION DEPENDS_ON -> [json]
@staticmethod
def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:
with belief_scope("TranslationExecutor._parse_llm_response"):
@@ -637,8 +767,8 @@ class TranslationExecutor:
translations[row_id] = str(translation)
return translations
# [/DEF:_parse_llm_response:Function]
# #endregion _parse_llm_response
# [/DEF:TranslationExecutor:Class]
# [/DEF:TranslationExecutor:Module]
# #endregion TranslationExecutor
# #endregion TranslationExecutor