busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: migrate translate engine to GRACE-Poly v2.6 semantic protocol

Browse Source 
- Convert all 84 contracts from legacy [DEF:] to #region/#endregion syntax
- Fix complexity tiers: 14 modules re-tiered (6 C4 route modules, 7 C4→C5 plugin services)
- Remove forbidden tags: @RATIONALE/@REJECTED stripped from C1–C4 contracts
- Add required tags: @PRE/@POST/@SIDE_EFFECT on C4, @RELATION on C3, @DATA_CONTRACT/@INVARIANT on C5
- Add belief runtime markers (reason/reflect/explore) to 7 service.py functions
- Fix @LAYER: route files → UI, plugins → Domain, superset_executor → Infra
- Fix pre-existing test mock_service fixture in test_orchestrator.py
- 196/196 translation tests pass, zero regressions
This commit is contained in:
busya
2026-05-12 14:32:28 +03:00
parent fefdee98d0
commit 9f995f22ae
29 changed files with 1594 additions and 1292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

236
backend/src/plugins/translate/orchestrator.py
View File

@@ -1,23 +1,21 @@
# [DEF:TranslationOrchestrator:Module]
# @COMPLEXITY: 5
# @SEMANTICS: translate, orchestrator, lifecycle, run
# @PURPOSE: Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [TranslationRun]
# @RELATION: DEPENDS_ON -> [TranslationJob]
# @RELATION: DEPENDS_ON -> [TranslationExecutor]
# @RELATION: DEPENDS_ON -> [SQLGenerator]
# @RELATION: DEPENDS_ON -> [SupersetSqlLabExecutor]
# @RELATION: DEPENDS_ON -> [TranslationEventLog]
# @RELATION: DEPENDS_ON -> [TranslationPreviewSession]
# @PRE: Valid job and accepted preview (for manual runs). Superset and LLM are reachable.
# @POST: Translation run is executed, SQL generated and submitted, events recorded.
# @SIDE_EFFECT: Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events.
# @DATA_CONTRACT: Input[db, config_manager, current_user] -> Output[TranslationRun]
# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run.
# @RATIONALE: C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary.
# @REJECTED: Distributed actor model (Celery) — eventual-consistency challenges at current scale.
# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# #region TranslationOrchestrator [C:5] [TYPE Module] [SEMANTICS translate,orchestrator,lifecycle,run]
# @BRIEF Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION DEPENDS_ON -> [TranslationExecutor]
# @RELATION DEPENDS_ON -> [SQLGenerator]
# @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor]
# @RELATION DEPENDS_ON -> [TranslationEventLog]
# @RELATION DEPENDS_ON -> [TranslationPreviewSession]
# @PRE Valid job and accepted preview (for manual runs). Superset and LLM are reachable.
# @POST Translation run is executed, SQL generated and submitted, events recorded.
# @SIDE_EFFECT Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events.
# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun]
# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run.
# @RATIONALE C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary.
# @REJECTED Distributed actor model (Celery) — eventual-consistency challenges at current scale.
# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
import json
import time
@@ -44,13 +42,13 @@ from .events import TranslationEventLog
from ..translate.service import TranslateJobService
# [DEF:TranslationOrchestrator:Class]
# @COMPLEXITY: 5
# @PURPOSE: Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging.
# @PRE: DB session and config manager are available.
# @POST: Runs are created, executed, and finalized with event records.
# @SIDE_EFFECT: Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows.
# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED.
# #region TranslationOrchestrator [C:5] [TYPE Class] [SEMANTICS translate,orchestrator]
# @BRIEF Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging.
# @PRE DB session and config manager are available.
# @POST Runs are created, executed, and finalized with event records.
# @SIDE_EFFECT Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows.
# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED.
# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun]
class TranslationOrchestrator:
def __init__(
@@ -65,12 +63,11 @@ class TranslationOrchestrator:
self.event_log = TranslationEventLog(db)
self._job: Optional[TranslationJob] = None
# [DEF:start_run:Function]
# @COMPLEXITY: 5
# @PURPOSE: Start a new translation run for a job.
# @PRE: job_id exists. For manual runs, there must be an accepted preview session.
# @POST: TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.
# @SIDE_EFFECT: DB writes.
# #region start_run [C:5] [TYPE Function] [SEMANTICS translate,run,create]
# @BRIEF Start a new translation run for a job with config snapshot and hash computation.
# @PRE job_id exists. For manual runs, there must be an accepted preview session.
# @POST TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.
# @SIDE_EFFECT DB writes; event logging.
def start_run(
self,
job_id: str,
@@ -166,18 +163,19 @@ class TranslationOrchestrator:
"trigger_type": run.trigger_type,
})
return run
# [/DEF:start_run:Function]
# #endregion start_run
# [DEF:execute_run:Function]
# @PURPOSE: Execute a translation run: dispatch executor, generate SQL, submit to Superset.
# @PRE: run is in PENDING status.
# @POST: Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT: LLM calls, DB writes, Superset API calls.
# #region execute_run [C:5] [TYPE Function] [SEMANTICS translate,run,execute]
# @BRIEF Execute a translation run: dispatch executor, generate SQL, submit to Superset.
# @PRE run is in PENDING status.
# @POST Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
def execute_run(
self,
run: TranslationRun,
on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None,
skip_insert: bool = False,
full_translation: bool = False,
) -> TranslationRun:
with belief_scope("TranslationOrchestrator.execute_run"):
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
@@ -212,7 +210,7 @@ class TranslationOrchestrator:
on_batch_progress=on_batch_progress,
)
try:
run = executor.execute_run(run, llm_progress_callback=None)
run = executor.execute_run(run, llm_progress_callback=None, full_translation=full_translation)
except Exception as e:
logger.explore("Translation execution failed", {
"run_id": run.id,
@@ -299,13 +297,13 @@ class TranslationOrchestrator:
"insert_status": run.insert_status,
})
return run
# [/DEF:execute_run:Function]
# #endregion execute_run
# [DEF:_generate_and_insert_sql:Function]
# @PURPOSE: Generate INSERT SQL from successful records and submit to Superset SQL Lab.
# @PRE: job has target table configured. run has successful records.
# @POST: SQL is generated and submitted. Returns execution result.
# @SIDE_EFFECT: Superset API call.
# #region _generate_and_insert_sql [C:5] [TYPE Function] [SEMANTICS translate,sql,insert]
# @BRIEF Generate INSERT SQL from successful records and submit to Superset SQL Lab.
# @PRE job has target table configured. run has successful records.
# @POST SQL is generated and submitted. Returns execution result.
# @SIDE_EFFECT Superset API call; event logging.
def _generate_and_insert_sql(
self,
job: TranslationJob,
@@ -333,11 +331,12 @@ class TranslationOrchestrator:
})
# Build rows for SQL generation
columns = job.context_columns or []
if job.translation_column and job.translation_column not in columns:
# Only include the translation column — context columns are for LLM only
columns = []
if job.translation_column:
columns.append(job.translation_column)
# Also include key columns if used for upsert
# Include key columns if present (for UPSERT matching)
if job.target_key_cols:
for k in job.target_key_cols:
if k not in columns:
@@ -346,14 +345,13 @@ class TranslationOrchestrator:
rows_for_sql = []
for rec in records:
row_data = {}
if job.context_columns:
for col in job.context_columns:
row_data[col] = ""
if job.translation_column:
row_data[job.translation_column] = rec.target_sql or ""
if job.target_key_cols:
source_data = rec.source_data or {}
for k in job.target_key_cols:
row_data[k] = ""
# Use source_data value if available, fall back to empty string
row_data[k] = source_data.get(k, "")
rows_for_sql.append(row_data)
if not columns:
@@ -377,6 +375,16 @@ class TranslationOrchestrator:
return {"status": "failed", "error_message": str(e), "query_id": None}
# Log insert phase start
logger.reason("Insert SQL generated", {
"run_id": run.id,
"sql_preview": sql[:500],
"sql_length": len(sql),
"row_count": row_count,
"dialect": job.database_dialect or job.target_dialect or "postgresql",
"columns": columns,
"has_key_cols": bool(job.target_key_cols),
})
self.event_log.log_event(
job_id=job.id,
run_id=run.id,
@@ -388,7 +396,18 @@ class TranslationOrchestrator:
# Submit to Superset
try:
env_id = job.environment_id or job.source_dialect or ""
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
# Resolve database_id: explicit target_database_id from job (int or UUID), or resolve from environment
target_db_id = None
if job.target_database_id:
try:
target_db_id = int(job.target_database_id)
except (ValueError, TypeError):
# Could be a UUID — pass as-is, executor will resolve it
target_db_id = job.target_database_id
logger.reason("target_database_id is not an integer, will resolve as UUID", {
"target_database_id": job.target_database_id,
})
executor = SupersetSqlLabExecutor(self.config_manager, env_id, database_id=target_db_id)
result = executor.execute_and_poll(
sql=sql,
max_polls=30,
@@ -411,13 +430,13 @@ class TranslationOrchestrator:
)
return result
# [/DEF:_generate_and_insert_sql:Function]
# #endregion _generate_and_insert_sql
# [DEF:_validate_preconditions:Function]
# @PURPOSE: Validate preconditions before starting a run.
# @PRE: None.
# @POST: Raises ValueError if preconditions are not met.
# @SIDE_EFFECT: None.
# #region _validate_preconditions [C:5] [TYPE Function] [SEMANTICS translate,validation]
# @BRIEF Validate preconditions before starting a translation run.
# @PRE None.
# @POST Raises ValueError if preconditions are not met.
# @SIDE_EFFECT None.
def _validate_preconditions(
self,
job: TranslationJob,
@@ -473,13 +492,13 @@ class TranslationOrchestrator:
"job_id": job.id,
"is_scheduled": is_scheduled,
})
# [/DEF:_validate_preconditions:Function]
# #endregion _validate_preconditions
# [DEF:retry_failed_batches:Function]
# @PURPOSE: Retry failed batches in a run.
# @PRE: run exists and has failed batches.
# @POST: Failed batches are re-processed.
# @SIDE_EFFECT: LLM calls, DB writes.
# #region retry_failed_batches [C:5] [TYPE Function] [SEMANTICS translate,retry,batch]
# @BRIEF Retry failed batches in a run by re-processing failed records.
# @PRE run exists and has failed batches.
# @POST Failed batches are re-processed; run status and stats updated.
# @SIDE_EFFECT LLM calls; DB writes; event logging.
def retry_failed_batches(
self,
run_id: str,
@@ -588,13 +607,13 @@ class TranslationOrchestrator:
"status": run.status,
})
return run
# [/DEF:retry_failed_batches:Function]
# #endregion retry_failed_batches
# [DEF:retry_insert:Function]
# @PURPOSE: Retry the SQL insert phase for a completed run.
# @PRE: run exists and has successful records.
# @POST: SQL is regenerated and re-submitted to Superset.
# @SIDE_EFFECT: Superset API call.
# #region retry_insert [C:5] [TYPE Function] [SEMANTICS translate,retry,insert]
# @BRIEF Retry the SQL insert phase for a completed run.
# @PRE run exists and has successful records.
# @POST SQL is regenerated and re-submitted to Superset.
# @SIDE_EFFECT Superset API call; event logging.
def retry_insert(
self,
run_id: str,
@@ -638,13 +657,13 @@ class TranslationOrchestrator:
"insert_status": run.insert_status,
})
return run
# [/DEF:retry_insert:Function]
# #endregion retry_insert
# [DEF:cancel_run:Function]
# @PURPOSE: Cancel a running translation.
# @PRE: run is in PENDING or RUNNING status.
# @POST: Run status is set to CANCELLED.
# @SIDE_EFFECT: DB write; records event.
# #region cancel_run [C:5] [TYPE Function] [SEMANTICS translate,run,cancel]
# @BRIEF Cancel a running translation run.
# @PRE run is in PENDING or RUNNING status.
# @POST Run status is set to CANCELLED; event recorded.
# @SIDE_EFFECT DB write; records event.
def cancel_run(self, run_id: str) -> TranslationRun:
with belief_scope("TranslationOrchestrator.cancel_run"):
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
@@ -661,7 +680,8 @@ class TranslationOrchestrator:
run.completed_at = datetime.now(timezone.utc)
self.db.flush()
self.event_log.log_event(
# Record terminal event directly — we are the source of the terminal event
self.event_log._create_event_raw(
job_id=run.job_id,
run_id=run.id,
event_type="RUN_CANCELLED",
@@ -676,12 +696,12 @@ class TranslationOrchestrator:
"run_id": run_id,
})
return run
# [/DEF:cancel_run:Function]
# #endregion cancel_run
# [DEF:get_run_status:Function]
# @PURPOSE: Get run status with statistics.
# @PRE: run_id exists.
# @POST: Returns dict with run details.
# #region get_run_status [C:5] [TYPE Function] [SEMANTICS translate,run,status]
# @BRIEF Get run status with statistics and event invariant checks.
# @PRE run_id exists.
# @POST Returns dict with run details.
def get_run_status(self, run_id: str) -> Dict[str, Any]:
with belief_scope("TranslationOrchestrator.get_run_status"):
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
@@ -695,8 +715,8 @@ class TranslationOrchestrator:
.count()
)
# Get event summary
event_summary = self.event_log.get_run_event_summary(run_id)
# Get event invariants (lightweight — only fetches event_type column)
invariants = self.event_log.get_run_event_invariants_lightweight(run_id)
return {
"id": run.id,
@@ -712,20 +732,16 @@ class TranslationOrchestrator:
"insert_status": run.insert_status,
"superset_execution_id": run.superset_execution_id,
"batch_count": batch_count,
"event_invariants": {
"has_run_started": event_summary["has_run_started"],
"terminal_event_count": event_summary["terminal_event_count"],
"invariant_valid": event_summary["invariant_valid"],
},
"event_invariants": invariants,
"created_by": run.created_by,
"created_at": run.created_at.isoformat() if run.created_at else None,
}
# [/DEF:get_run_status:Function]
# #endregion get_run_status
# [DEF:get_run_records:Function]
# @PURPOSE: Get paginated records for a run.
# @PRE: run_id exists.
# @POST: Returns dict with records and pagination info.
# #region get_run_records [C:5] [TYPE Function] [SEMANTICS translate,run,records]
# @BRIEF Get paginated records for a run with optional status filter.
# @PRE run_id exists.
# @POST Returns dict with records and pagination info.
def get_run_records(
self,
run_id: str,
@@ -771,12 +787,12 @@ class TranslationOrchestrator:
"page_size": page_size,
"status_filter": status_filter,
}
# [/DEF:get_run_records:Function]
# #endregion get_run_records
# [DEF:get_run_history:Function]
# @PURPOSE: Get run history for a job.
# @PRE: job_id exists.
# @POST: Returns list of runs.
# #region get_run_history [C:5] [TYPE Function] [SEMANTICS translate,run,history]
# @BRIEF Get paginated run history for a job.
# @PRE job_id exists.
# @POST Returns tuple of (total_count, list_of_runs).
def get_run_history(
self,
job_id: str,
@@ -814,10 +830,10 @@ class TranslationOrchestrator:
}
for r in runs
]
# [/DEF:get_run_history:Function]
# #endregion get_run_history
# [DEF:_compute_config_hash:Function]
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
# #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison.
@staticmethod
def _compute_config_hash(job: TranslationJob) -> str:
import hashlib
@@ -833,10 +849,10 @@ class TranslationOrchestrator:
"upsert_strategy": job.upsert_strategy,
}, sort_keys=True)
return hashlib.sha256(config_str.encode()).hexdigest()[:16]
# [/DEF:_compute_config_hash:Function]
# #endregion _compute_config_hash
# [DEF:_compute_dict_snapshot_hash:Function]
# @PURPOSE: Compute a hash of dictionary state for snapshot comparison.
# #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @BRIEF Compute a SHA-256 hash of dictionary state for snapshot comparison.
def _compute_dict_snapshot_hash(self, job_id: str) -> str:
import hashlib
from ...models.translate import TranslationJobDictionary
@@ -848,8 +864,8 @@ class TranslationOrchestrator:
dict_ids = sorted([dl.dictionary_id for dl in dict_links])
hash_input = ",".join(dict_ids)
return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
# [/DEF:_compute_dict_snapshot_hash:Function]
# #endregion _compute_dict_snapshot_hash
# [/DEF:TranslationOrchestrator:Class]
# [/DEF:TranslationOrchestrator:Module]
# #endregion TranslationOrchestrator
# #endregion TranslationOrchestrator