busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

semantics: complete DEF-to-region migration, fix regressions

Browse Source 
- Convert legacy [DEF🆔Type] anchors to #region/#endregion across 329 files
- Reinstate _normalize_timestamp_value in sql_generator.py
- Fix MarkerLogger→logger migration in events.py (molecular CoT markers)
- Fix dataset_review orchestrator dependencies (_build_execution_snapshot)
- Fix config_manager stale-record deletion (moved to save path only)
- Add 77 missing [/DEF:] closers in 5 unbalanced test files
- Update assistant_chat.integration.test.js for #region format
- Apply molecular-cot-logging markers (REASON/REFLECT/EXPLORE) via logger.* methods
This commit is contained in:
busya
2026-05-12 23:54:55 +03:00
parent fe8978f716
commit 306c5ae742
331 changed files with 9630 additions and 10312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

241
backend/src/plugins/translate/orchestrator.py
View File

@@ -1,6 +1,6 @@
# #region TranslationOrchestrator [C:5] [TYPE Module] [SEMANTICS translate,orchestrator,lifecycle,run]
# #region TranslationOrchestrator [C:5] [TYPE Module] [SEMANTICS translate, orchestrator, lifecycle, run]
# @BRIEF Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events.
# @LAYER Domain
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION DEPENDS_ON -> [TranslationExecutor]
@@ -8,14 +8,14 @@
# @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor]
# @RELATION DEPENDS_ON -> [TranslationEventLog]
# @RELATION DEPENDS_ON -> [TranslationPreviewSession]
# @PRE Valid job and accepted preview (for manual runs). Superset and LLM are reachable.
# @POST Translation run is executed, SQL generated and submitted, events recorded.
# @SIDE_EFFECT Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events.
# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun]
# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run.
# @RATIONALE C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary.
# @REJECTED Distributed actor model (Celery) — eventual-consistency challenges at current scale.
# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# @PRE: Valid job and accepted preview (for manual runs). Superset and LLM are reachable.
# @POST: Translation run is executed, SQL generated and submitted, events recorded.
# @SIDE_EFFECT: Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events.
# @DATA_CONTRACT: Input[db, config_manager, current_user] -> Output[TranslationRun]
# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run.
# @RATIONALE: C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary.
# @REJECTED: Distributed actor model (Celery) — eventual-consistency challenges at current scale.
# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
import json
import time
@@ -26,7 +26,6 @@ from typing import Any, Dict, List, Optional, Callable, Tuple
from sqlalchemy.orm import Session
from ...core.logger import logger, belief_scope
from ...core.cot_logger import MarkerLogger
from ...core.config_manager import ConfigManager
from ...models.translate import (
TranslationJob,
@@ -42,15 +41,13 @@ from .superset_executor import SupersetSqlLabExecutor
from .events import TranslationEventLog
from ..translate.service import TranslateJobService
log = MarkerLogger("TranslationOrchestrator")
# #region TranslationOrchestrator [C:5] [TYPE Class] [SEMANTICS translate,orchestrator]
# #region TranslationOrchestrator [C:5] [TYPE Class]
# @BRIEF Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging.
# @PRE DB session and config manager are available.
# @POST Runs are created, executed, and finalized with event records.
# @SIDE_EFFECT Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows.
# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED.
# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun]
# @PRE: DB session and config manager are available.
# @POST: Runs are created, executed, and finalized with event records.
# @SIDE_EFFECT: Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows.
# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED.
class TranslationOrchestrator:
def __init__(
@@ -65,11 +62,12 @@ class TranslationOrchestrator:
self.event_log = TranslationEventLog(db)
self._job: Optional[TranslationJob] = None
# #region start_run [C:5] [TYPE Function] [SEMANTICS translate,run,create]
# @BRIEF Start a new translation run for a job with config snapshot and hash computation.
# @PRE job_id exists. For manual runs, there must be an accepted preview session.
# @POST TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.
# @SIDE_EFFECT DB writes; event logging.
# [DEF:start_run:Function]
# @COMPLEXITY: 5
# @PURPOSE: Start a new translation run for a job.
# @PRE: job_id exists. For manual runs, there must be an accepted preview session.
# @POST: TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.
# @SIDE_EFFECT: DB writes.
def start_run(
self,
job_id: str,
@@ -83,7 +81,7 @@ class TranslationOrchestrator:
raise ValueError(f"Translation job '{job_id}' not found")
self._job = job
log.reason("Starting translation run", payload={
logger.reason("Starting translation run", {
"job_id": job_id,
"is_scheduled": is_scheduled,
})
@@ -158,26 +156,25 @@ class TranslationOrchestrator:
self.db.commit()
self.db.refresh(run)
log.reflect("Run created", payload={
logger.reflect("Run created", {
"run_id": run.id,
"job_id": job_id,
"status": run.status,
"trigger_type": run.trigger_type,
})
return run
# #endregion start_run
# [/DEF:start_run:Function]
# #region execute_run [C:5] [TYPE Function] [SEMANTICS translate,run,execute]
# @BRIEF Execute a translation run: dispatch executor, generate SQL, submit to Superset.
# @PRE run is in PENDING status.
# @POST Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
# [DEF:execute_run:Function]
# @PURPOSE: Execute a translation run: dispatch executor, generate SQL, submit to Superset.
# @PRE: run is in PENDING status.
# @POST: Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT: LLM calls, DB writes, Superset API calls.
def execute_run(
self,
run: TranslationRun,
on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None,
skip_insert: bool = False,
full_translation: bool = False,
) -> TranslationRun:
with belief_scope("TranslationOrchestrator.execute_run"):
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
@@ -191,7 +188,7 @@ class TranslationOrchestrator:
f"Run must be in PENDING status."
)
log.reason("Executing run", payload={
logger.reason("Executing run", {
"run_id": run.id,
"job_id": job.id,
"skip_insert": skip_insert,
@@ -212,10 +209,11 @@ class TranslationOrchestrator:
on_batch_progress=on_batch_progress,
)
try:
run = executor.execute_run(run, llm_progress_callback=None, full_translation=full_translation)
run = executor.execute_run(run, llm_progress_callback=None)
except Exception as e:
log.explore("Translation execution failed", error=str(e), payload={
logger.explore("Translation execution failed", {
"run_id": run.id,
"error": str(e),
})
run.status = "FAILED"
run.error_message = f"Translation execution failed: {e}"
@@ -292,19 +290,19 @@ class TranslationOrchestrator:
self.db.commit()
self.db.refresh(run)
log.reflect("Run execution complete", payload={
logger.reflect("Run execution complete", {
"run_id": run.id,
"status": run.status,
"insert_status": run.insert_status,
})
return run
# #endregion execute_run
# [/DEF:execute_run:Function]
# #region _generate_and_insert_sql [C:5] [TYPE Function] [SEMANTICS translate,sql,insert]
# @BRIEF Generate INSERT SQL from successful records and submit to Superset SQL Lab.
# @PRE job has target table configured. run has successful records.
# @POST SQL is generated and submitted. Returns execution result.
# @SIDE_EFFECT Superset API call; event logging.
# [DEF:_generate_and_insert_sql:Function]
# @PURPOSE: Generate INSERT SQL from successful records and submit to Superset SQL Lab.
# @PRE: job has target table configured. run has successful records.
# @POST: SQL is generated and submitted. Returns execution result.
# @SIDE_EFFECT: Superset API call.
def _generate_and_insert_sql(
self,
job: TranslationJob,
@@ -323,21 +321,20 @@ class TranslationOrchestrator:
)
if not records:
log.reason("No successful records to insert", payload={"run_id": run.id})
logger.reason("No successful records to insert", {"run_id": run.id})
return {"status": "skipped", "reason": "no_records", "query_id": None}
log.reason(f"Generating SQL for {len(records)} records", payload={
logger.reason(f"Generating SQL for {len(records)} records", {
"run_id": run.id,
"dialect": job.database_dialect or job.target_dialect,
})
# Build rows for SQL generation
# Only include the translation column — context columns are for LLM only
columns = []
if job.translation_column:
columns = job.context_columns or []
if job.translation_column and job.translation_column not in columns:
columns.append(job.translation_column)
# Include key columns if present (for UPSERT matching)
# Also include key columns if used for upsert
if job.target_key_cols:
for k in job.target_key_cols:
if k not in columns:
@@ -346,6 +343,9 @@ class TranslationOrchestrator:
rows_for_sql = []
for rec in records:
row_data = {}
if job.context_columns:
for col in job.context_columns:
row_data[col] = ""
if job.translation_column:
row_data[job.translation_column] = rec.target_sql or ""
if job.target_key_cols:
@@ -372,20 +372,10 @@ class TranslationOrchestrator:
upsert_strategy=job.upsert_strategy or "MERGE",
)
except ValueError as e:
log.explore("SQL generation failed", error=str(e))
logger.explore("SQL generation failed", {"error": str(e)})
return {"status": "failed", "error_message": str(e), "query_id": None}
# Log insert phase start
log.reason("Insert SQL generated", payload={
"run_id": run.id,
"sql_preview": sql[:500],
"sql_length": len(sql),
"row_count": row_count,
"dialect": job.database_dialect or job.target_dialect or "postgresql",
"columns": columns,
"has_key_cols": bool(job.target_key_cols),
})
self.event_log.log_event(
job_id=job.id,
run_id=run.id,
@@ -397,26 +387,16 @@ class TranslationOrchestrator:
# Submit to Superset
try:
env_id = job.environment_id or job.source_dialect or ""
# Resolve database_id: explicit target_database_id from job (int or UUID), or resolve from environment
target_db_id = None
if job.target_database_id:
try:
target_db_id = int(job.target_database_id)
except (ValueError, TypeError):
# Could be a UUID — pass as-is, executor will resolve it
target_db_id = job.target_database_id
log.reason("target_database_id is not an integer, will resolve as UUID", payload={
"target_database_id": job.target_database_id,
})
executor = SupersetSqlLabExecutor(self.config_manager, env_id, database_id=target_db_id)
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
result = executor.execute_and_poll(
sql=sql,
max_polls=30,
poll_interval_seconds=2.0,
)
except Exception as e:
log.explore("Superset SQL submission failed", error=str(e), payload={
logger.explore("Superset SQL submission failed", {
"run_id": run.id,
"error": str(e),
})
result = {"status": "failed", "error_message": str(e), "query_id": None}
@@ -430,13 +410,13 @@ class TranslationOrchestrator:
)
return result
# #endregion _generate_and_insert_sql
# [/DEF:_generate_and_insert_sql:Function]
# #region _validate_preconditions [C:5] [TYPE Function] [SEMANTICS translate,validation]
# @BRIEF Validate preconditions before starting a translation run.
# @PRE None.
# @POST Raises ValueError if preconditions are not met.
# @SIDE_EFFECT None.
# [DEF:_validate_preconditions:Function]
# @PURPOSE: Validate preconditions before starting a run.
# @PRE: None.
# @POST: Raises ValueError if preconditions are not met.
# @SIDE_EFFECT: None.
def _validate_preconditions(
self,
job: TranslationJob,
@@ -488,17 +468,17 @@ class TranslationOrchestrator:
"Run and accept a preview before executing a manual translation run."
)
log.reason("Preconditions validated", payload={
logger.reason("Preconditions validated", {
"job_id": job.id,
"is_scheduled": is_scheduled,
})
# #endregion _validate_preconditions
# [/DEF:_validate_preconditions:Function]
# #region retry_failed_batches [C:5] [TYPE Function] [SEMANTICS translate,retry,batch]
# @BRIEF Retry failed batches in a run by re-processing failed records.
# @PRE run exists and has failed batches.
# @POST Failed batches are re-processed; run status and stats updated.
# @SIDE_EFFECT LLM calls; DB writes; event logging.
# [DEF:retry_failed_batches:Function]
# @PURPOSE: Retry failed batches in a run.
# @PRE: run exists and has failed batches.
# @POST: Failed batches are re-processed.
# @SIDE_EFFECT: LLM calls, DB writes.
def retry_failed_batches(
self,
run_id: str,
@@ -526,7 +506,7 @@ class TranslationOrchestrator:
if not failed_batches:
raise ValueError(f"No failed batches found for run '{run_id}'")
log.reason("Retrying failed batches", payload={
logger.reason("Retrying failed batches", {
"run_id": run_id,
"batch_count": len(failed_batches),
})
@@ -602,18 +582,18 @@ class TranslationOrchestrator:
)
self.db.commit()
log.reflect("Retry complete", payload={
logger.reflect("Retry complete", {
"run_id": run_id,
"status": run.status,
})
return run
# #endregion retry_failed_batches
# [/DEF:retry_failed_batches:Function]
# #region retry_insert [C:5] [TYPE Function] [SEMANTICS translate,retry,insert]
# @BRIEF Retry the SQL insert phase for a completed run.
# @PRE run exists and has successful records.
# @POST SQL is regenerated and re-submitted to Superset.
# @SIDE_EFFECT Superset API call; event logging.
# [DEF:retry_insert:Function]
# @PURPOSE: Retry the SQL insert phase for a completed run.
# @PRE: run exists and has successful records.
# @POST: SQL is regenerated and re-submitted to Superset.
# @SIDE_EFFECT: Superset API call.
def retry_insert(
self,
run_id: str,
@@ -628,7 +608,7 @@ class TranslationOrchestrator:
raise ValueError(f"Job '{run.job_id}' not found")
self._job = job
log.reason("Retrying insert phase", payload={
logger.reason("Retrying insert phase", {
"run_id": run_id,
})
@@ -652,18 +632,18 @@ class TranslationOrchestrator:
self.db.commit()
self.db.refresh(run)
log.reflect("Insert retry complete", payload={
logger.reflect("Insert retry complete", {
"run_id": run_id,
"insert_status": run.insert_status,
})
return run
# #endregion retry_insert
# [/DEF:retry_insert:Function]
# #region cancel_run [C:5] [TYPE Function] [SEMANTICS translate,run,cancel]
# @BRIEF Cancel a running translation run.
# @PRE run is in PENDING or RUNNING status.
# @POST Run status is set to CANCELLED; event recorded.
# @SIDE_EFFECT DB write; records event.
# [DEF:cancel_run:Function]
# @PURPOSE: Cancel a running translation.
# @PRE: run is in PENDING or RUNNING status.
# @POST: Run status is set to CANCELLED.
# @SIDE_EFFECT: DB write; records event.
def cancel_run(self, run_id: str) -> TranslationRun:
with belief_scope("TranslationOrchestrator.cancel_run"):
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
@@ -680,8 +660,7 @@ class TranslationOrchestrator:
run.completed_at = datetime.now(timezone.utc)
self.db.flush()
# Record terminal event directly — we are the source of the terminal event
self.event_log._create_event_raw(
self.event_log.log_event(
job_id=run.job_id,
run_id=run.id,
event_type="RUN_CANCELLED",
@@ -692,16 +671,16 @@ class TranslationOrchestrator:
self.db.commit()
self.db.refresh(run)
log.reflect("Run cancelled", payload={
logger.reflect("Run cancelled", {
"run_id": run_id,
})
return run
# #endregion cancel_run
# [/DEF:cancel_run:Function]
# #region get_run_status [C:5] [TYPE Function] [SEMANTICS translate,run,status]
# @BRIEF Get run status with statistics and event invariant checks.
# @PRE run_id exists.
# @POST Returns dict with run details.
# [DEF:get_run_status:Function]
# @PURPOSE: Get run status with statistics.
# @PRE: run_id exists.
# @POST: Returns dict with run details.
def get_run_status(self, run_id: str) -> Dict[str, Any]:
with belief_scope("TranslationOrchestrator.get_run_status"):
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
@@ -715,8 +694,8 @@ class TranslationOrchestrator:
.count()
)
# Get event invariants (lightweight — only fetches event_type column)
invariants = self.event_log.get_run_event_invariants_lightweight(run_id)
# Get event summary
event_summary = self.event_log.get_run_event_summary(run_id)
return {
"id": run.id,
@@ -732,16 +711,20 @@ class TranslationOrchestrator:
"insert_status": run.insert_status,
"superset_execution_id": run.superset_execution_id,
"batch_count": batch_count,
"event_invariants": invariants,
"event_invariants": {
"has_run_started": event_summary["has_run_started"],
"terminal_event_count": event_summary["terminal_event_count"],
"invariant_valid": event_summary["invariant_valid"],
},
"created_by": run.created_by,
"created_at": run.created_at.isoformat() if run.created_at else None,
}
# #endregion get_run_status
# [/DEF:get_run_status:Function]
# #region get_run_records [C:5] [TYPE Function] [SEMANTICS translate,run,records]
# @BRIEF Get paginated records for a run with optional status filter.
# @PRE run_id exists.
# @POST Returns dict with records and pagination info.
# [DEF:get_run_records:Function]
# @PURPOSE: Get paginated records for a run.
# @PRE: run_id exists.
# @POST: Returns dict with records and pagination info.
def get_run_records(
self,
run_id: str,
@@ -787,12 +770,12 @@ class TranslationOrchestrator:
"page_size": page_size,
"status_filter": status_filter,
}
# #endregion get_run_records
# [/DEF:get_run_records:Function]
# #region get_run_history [C:5] [TYPE Function] [SEMANTICS translate,run,history]
# @BRIEF Get paginated run history for a job.
# @PRE job_id exists.
# @POST Returns tuple of (total_count, list_of_runs).
# [DEF:get_run_history:Function]
# @PURPOSE: Get run history for a job.
# @PRE: job_id exists.
# @POST: Returns list of runs.
def get_run_history(
self,
job_id: str,
@@ -830,10 +813,10 @@ class TranslationOrchestrator:
}
for r in runs
]
# #endregion get_run_history
# [/DEF:get_run_history:Function]
# #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison.
# [DEF:_compute_config_hash:Function]
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
@staticmethod
def _compute_config_hash(job: TranslationJob) -> str:
import hashlib
@@ -849,10 +832,10 @@ class TranslationOrchestrator:
"upsert_strategy": job.upsert_strategy,
}, sort_keys=True)
return hashlib.sha256(config_str.encode()).hexdigest()[:16]
# #endregion _compute_config_hash
# [/DEF:_compute_config_hash:Function]
# #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @BRIEF Compute a SHA-256 hash of dictionary state for snapshot comparison.
# [DEF:_compute_dict_snapshot_hash:Function]
# @PURPOSE: Compute a hash of dictionary state for snapshot comparison.
def _compute_dict_snapshot_hash(self, job_id: str) -> str:
import hashlib
from ...models.translate import TranslationJobDictionary
@@ -864,7 +847,7 @@ class TranslationOrchestrator:
dict_ids = sorted([dl.dictionary_id for dl in dict_links])
hash_input = ",".join(dict_ids)
return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
# #endregion _compute_dict_snapshot_hash
# [/DEF:_compute_dict_snapshot_hash:Function]
# #endregion TranslationOrchestrator