# #region TranslationExecutor [C:4] [TYPE Module] [SEMANTICS sqlalchemy, tenacity, translate, insert, llm-retry] # @BRIEF Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows. # @LAYER: Domain # @RELATION DEPENDS_ON -> [TranslationBatch] # @RELATION DEPENDS_ON -> [TranslationRecord] # @RELATION DEPENDS_ON -> [TranslationRun] # @RELATION DEPENDS_ON -> [LLMProviderService] # @RELATION DEPENDS_ON -> [DictionaryManager] # @RELATION DEPENDS_ON -> [TranslationPreview] # @PRE: Valid TranslationRun with job configuration. DB session is available. # @POST: TranslationBatch and TranslationRecord rows are created. Run status is updated. # @SIDE_EFFECT: Calls LLM provider; creates DB rows; updates run statistics. # @RATIONALE: Batch processing with retry — independent batches allow partial recovery. # @REJECTED: Single monolithic LLM call — would lose all progress on any failure. import json import time import uuid from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Set, Tuple, Callable from sqlalchemy.orm import Session from ...core.logger import logger, belief_scope from ...core.config_manager import ConfigManager from ...models.translate import ( TranslationJob, TranslationRun, TranslationBatch, TranslationRecord, TranslationPreviewSession, TranslationPreviewRecord, ) from ...services.llm_provider import LLMProviderService from ...services.llm_prompt_templates import render_prompt from .dictionary import DictionaryManager from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE # #region MAX_RETRIES_PER_BATCH [TYPE Constant] # @BRIEF Maximum number of retries for a single batch before marking it failed. MAX_RETRIES_PER_BATCH = 3 # #endregion MAX_RETRIES_PER_BATCH # #region TranslationExecutor [C:4] [TYPE Class] # @BRIEF Process translation batches: fetch source rows, filter dict, call LLM, persist results. # @PRE: DB session and config manager available. # @POST: Batches and records created with status tracking. # @SIDE_EFFECT: LLM API calls; DB writes. class TranslationExecutor: def __init__( self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None, on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None, ): self.db = db self.config_manager = config_manager self.current_user = current_user self.on_batch_progress = on_batch_progress self._current_run_id: Optional[str] = None # region execute_run [TYPE Function] # @PURPOSE: Run full translation execution for a TranslationRun. # @PRE: run is in PENDING or RUNNING status with valid job config. # @POST: Run is populated with batches and records. # @SIDE_EFFECT: LLM API calls; DB batch writes. def execute_run( self, run: TranslationRun, llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None, ) -> TranslationRun: with belief_scope("TranslationExecutor.execute_run"): job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first() if not job: raise ValueError(f"Job '{run.job_id}' not found for run '{run.id}'") logger.reason("Starting translation execution", { "run_id": run.id, "job_id": job.id, "batch_size": job.batch_size, }) # Mark run as RUNNING run.status = "RUNNING" run.started_at = datetime.now(timezone.utc) self.db.flush() # Fetch source rows from the accepted preview session source_rows = self._fetch_source_rows(job.id, run.id) if not source_rows: logger.explore("No source rows to translate", {"run_id": run.id}) run.status = "COMPLETED" run.completed_at = datetime.now(timezone.utc) self.db.flush() return run # Apply new-key-only filtering for scheduled runs if run.trigger_type == "new_key_only": source_rows = self._filter_new_keys(job, run.id, source_rows) if not source_rows: logger.reason("run_noop — no new rows to translate", { "job_id": job.id, "run_id": run.id, }) run.status = "COMPLETED" run.insert_status = "skipped" run.total_records = 0 run.completed_at = datetime.now(timezone.utc) self.db.commit() return total_rows = len(source_rows) run.total_records = total_rows # Split into batches batch_size = job.batch_size or 50 batches = [ source_rows[i:i + batch_size] for i in range(0, total_rows, batch_size) ] logger.reason(f"Processing {len(batches)} batches", { "run_id": run.id, "total_rows": total_rows, "batch_size": batch_size, }) successful_records = 0 failed_records = 0 skipped_records = 0 for batch_idx, batch_rows in enumerate(batches): batch_result = self._process_batch( job=job, run_id=run.id, batch_index=batch_idx, batch_rows=batch_rows, ) successful_records += batch_result["successful"] failed_records += batch_result["failed"] skipped_records += batch_result["skipped"] # Update run stats incrementally run.successful_records = successful_records run.failed_records = failed_records run.skipped_records = skipped_records self.db.flush() if self.on_batch_progress: self.on_batch_progress( run.id, batch_idx + 1, len(batches), successful_records, total_rows, ) # Update final run status if failed_records == 0 and skipped_records == 0: run.status = "COMPLETED" elif successful_records == 0: run.status = "FAILED" else: run.status = "COMPLETED" # Partial success run.completed_at = datetime.now(timezone.utc) self.db.flush() logger.reflect("Translation execution complete", { "run_id": run.id, "status": run.status, "total": total_rows, "successful": successful_records, "failed": failed_records, "skipped": skipped_records, }) return run # endregion execute_run # region _fetch_source_rows [TYPE Function] # @PURPOSE: Fetch full source dataset from Superset (via datasource) for full translation. # @PRE: job_id exists. Job may have source_datasource_id for full fetch. # @POST: Returns list of dicts with source data (all rows from the source datasource). # @SIDE_EFFECT: Makes HTTP call to Superset chart data API when datasource is configured. def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]: with belief_scope("TranslationExecutor._fetch_source_rows"): job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() # If source_datasource_id is configured, fetch ALL rows from the Superset chart data API if job and job.source_datasource_id: try: logger.reason("Fetching full dataset from Superset datasource", { "run_id": run_id, "datasource_id": job.source_datasource_id, "environment_id": job.environment_id, }) # Determine environment environments = self.config_manager.get_environments() target_env_id = job.environment_id or job.source_dialect or "" env_config = next( (e for e in environments if e.id == target_env_id), None, ) if not env_config and environments: env_config = environments[0] if env_config: from ...core.superset_client import SupersetClient client = SupersetClient(env_config) # Fetch dataset detail to build proper query context dataset_detail = client.get_dataset_detail(int(job.source_datasource_id)) # Build query context (same approach as preview but without row_limit) query_context = client.build_dataset_preview_query_context( dataset_id=int(job.source_datasource_id), dataset_record=dataset_detail, template_params={}, effective_filters=[], ) # Remove row_limit to get ALL rows; use result_type="samples" queries = query_context.get("queries", []) if queries: queries[0].pop("row_limit", None) queries[0].pop("result_type", None) queries[0]["metrics"] = [] query_context["result_type"] = "samples" form_data = query_context.get("form_data", {}) form_data.pop("query_mode", None) response = client.network.request( method="POST", endpoint="/api/v1/chart/data", data=json.dumps(query_context), headers={"Content-Type": "application/json"}, ) # Extract rows rows = self._extract_chart_data_rows(response) if rows: logger.reason(f"Fetched {len(rows)} rows from Superset datasource", { "run_id": run_id, }) # Map rows to source_rows format source_rows = [] for idx, row in enumerate(rows): source_data_dict = dict(row) if row else None source_text = str(row.get(job.translation_column, "")) if job.translation_column else json.dumps(row) source_rows.append({ "row_index": str(idx), "source_text": source_text, "approved_translation": None, "source_object_name": f"Row {idx}", "source_data": source_data_dict, }) return source_rows else: logger.explore("Superset datasource returned no rows", { "run_id": run_id, "datasource_id": job.source_datasource_id, }) else: logger.explore("No environment config found for datasource fetch", { "env_id": target_env_id, }) except Exception as e: logger.explore("Failed to fetch full dataset from Superset, falling back to preview", { "run_id": run_id, "error": str(e), }) # Fall through to preview-based fetch # Fallback: get the latest APPLIED preview session session = ( self.db.query(TranslationPreviewSession) .filter( TranslationPreviewSession.job_id == job_id, TranslationPreviewSession.status == "APPLIED", ) .order_by(TranslationPreviewSession.created_at.desc()) .first() ) if not session: logger.explore("No accepted preview session found", {"job_id": job_id}) return [] # Fetch APPROVED or all records from the session records = ( self.db.query(TranslationPreviewRecord) .filter( TranslationPreviewRecord.session_id == session.id, TranslationPreviewRecord.status.in_(["APPROVED", "PENDING"]), ) .all() ) source_rows = [] for rec in records: source_data_dict = None if hasattr(rec, "source_data") and rec.source_data: source_data_dict = dict(rec.source_data) source_rows.append({ "row_index": rec.source_object_id or "0", "source_text": rec.source_sql or "", "approved_translation": rec.target_sql if rec.status == "APPROVED" else None, "source_object_name": rec.source_object_name or "", "source_data": source_data_dict, }) logger.reason(f"Fetched {len(source_rows)} source rows from preview fallback", { "run_id": run_id, "session_id": session.id, }) return source_rows # endregion _fetch_source_rows # region _filter_new_keys [TYPE Function] # @PURPOSE: Filter source rows to only include those with keys absent from the last successful run. # @PRE: job and run are persisted; source_rows is a list of dicts with source_data containing key columns. # @POST: Returns filtered list of source rows with only new keys. Logs skip count. # @SIDE_EFFECT: Queries TranslationRecord from previous runs; no writes. def _filter_new_keys(self, job, run_id: str, source_rows: list) -> list: with belief_scope("TranslationExecutor._filter_new_keys"): # Find most recent COMPLETED run with successful insert for this job prev_run = ( self.db.query(TranslationRun) .filter( TranslationRun.job_id == job.id, TranslationRun.status == "COMPLETED", TranslationRun.insert_status == "succeeded", TranslationRun.id != run_id, ) .order_by(TranslationRun.created_at.desc()) .first() ) if not prev_run: logger.reason("No prior successful run — all keys treated as new", { "job_id": job.id, }) return source_rows # Get successful records from that run prev_records = ( self.db.query(TranslationRecord) .filter( TranslationRecord.run_id == prev_run.id, TranslationRecord.status == "SUCCESS", ) .all() ) if not prev_records: return source_rows # Build set of already-translated composite keys key_cols = job.target_key_cols or job.source_key_cols or [] if not key_cols: logger.explore("No key columns configured — skipping new-key-only filter", {"job_id": job.id}) return source_rows existing_keys = set() for rec in prev_records: sd = rec.source_data or {} key_tuple = tuple(str(sd.get(k, "")) for k in key_cols) existing_keys.add(key_tuple) # Filter: keep only rows whose keys are NOT in the existing set filtered = [] skipped = 0 for row in source_rows: sd = row.get("source_data", {}) or {} key_tuple = tuple(str(sd.get(k, "")) for k in key_cols) if key_tuple not in existing_keys: filtered.append(row) else: skipped += 1 logger.reason(f"New-key-only filter: {len(source_rows)} total → {len(filtered)} new, {skipped} skipped", { "job_id": job.id, "prev_run_id": prev_run.id, "key_cols": key_cols, }) return filtered # endregion _filter_new_keys # region _extract_chart_data_rows [TYPE Function] # @PURPOSE: Extract data rows from Superset chart data API response. # @POST: Returns list of dicts with column-value pairs. @staticmethod def _extract_chart_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]: result = response.get("result") if isinstance(result, list): for item in result: if isinstance(item, dict): data = item.get("data") if isinstance(data, list) and data: return data if isinstance(result, dict): data = result.get("data") if isinstance(data, list) and data: return data data = response.get("data") if isinstance(data, list) and data: return data if isinstance(result, list): return result return [] # endregion _extract_chart_data_rows # region _process_batch [TYPE Function] # @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records. # @PRE: job and batch_rows are valid. # @POST: TranslationBatch and TranslationRecord rows are created. # @SIDE_EFFECT: LLM API call. def _process_batch( self, job: TranslationJob, run_id: str, batch_index: int, batch_rows: List[Dict[str, Any]], ) -> Dict[str, int]: with belief_scope("TranslationExecutor._process_batch"): batch_start = time.monotonic() # Create batch record batch = TranslationBatch( id=str(uuid.uuid4()), run_id=run_id, batch_index=batch_index, status="RUNNING", total_records=len(batch_rows), started_at=datetime.now(timezone.utc), ) self.db.add(batch) self.db.flush() batch_id = batch.id result = {"successful": 0, "failed": 0, "skipped": 0, "retries": 0} # Extract source texts for dict filtering source_texts = [ row.get("source_text", "") for row in batch_rows if row.get("source_text") ] # Filter dictionary entries dict_matches = DictionaryManager.filter_for_batch( self.db, source_texts, job.id ) # For each row, determine if we need LLM translation or can use approved translation rows_for_llm = [] pre_translated = [] for row in batch_rows: if row.get("approved_translation"): pre_translated.append(row) else: rows_for_llm.append(row) # Handle pre-translated (approved) rows for row in pre_translated: record = TranslationRecord( id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id, source_sql=row.get("source_text", ""), target_sql=row.get("approved_translation"), source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), source_data=row.get("source_data"), status="SUCCESS", ) self.db.add(record) result["successful"] += 1 # Process rows needing LLM translation if rows_for_llm: llm_result = self._call_llm_for_batch( job=job, run_id=run_id, batch_rows=rows_for_llm, dict_matches=dict_matches, batch_id=batch_id, ) result["successful"] += llm_result["successful"] result["failed"] += llm_result["failed"] result["skipped"] += llm_result["skipped"] result["retries"] += llm_result.get("retries", 0) # Update batch status batch.successful_records = result["successful"] batch.failed_records = result["failed"] batch.completed_at = datetime.now(timezone.utc) batch.status = "COMPLETED" if result["failed"] == 0 else "COMPLETED_WITH_ERRORS" self.db.flush() batch_latency = int((time.monotonic() - batch_start) * 1000) logger.reason(f"Batch {batch_index} complete", { "batch_id": batch_id, "latency_ms": batch_latency, **result, }) return result # endregion _process_batch # region _call_llm_for_batch [TYPE Function] # @PURPOSE: Call LLM for a batch of rows requiring translation. Parse structured JSON response. # @PRE: job has valid provider_id. batch_rows is non-empty. # @POST: Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows. # @SIDE_EFFECT: HTTP call to LLM provider. def _call_llm_for_batch( self, job: TranslationJob, run_id: str, batch_rows: List[Dict[str, Any]], dict_matches: List[Dict[str, Any]], batch_id: str, ) -> Dict[str, int]: with belief_scope("TranslationExecutor._call_llm_for_batch"): # Build dictionary section dictionary_section = "" if dict_matches: glossary_lines = [] for m in dict_matches: glossary_lines.append( f"- '{m['source_term']}' -> '{m['target_term']}'" f"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}" ) dictionary_section = ( "Terminology dictionary (use these translations when applicable):\n" + "\n".join(glossary_lines) + "\n\n" ) # Build rows JSON for LLM rows_json = json.dumps([ { "row_id": str(row.get("row_index", idx)), "text": row.get("source_text", ""), } for idx, row in enumerate(batch_rows) ], indent=2) # Build prompt prompt = render_prompt( DEFAULT_EXECUTION_PROMPT_TEMPLATE, { "source_language": job.source_dialect or "SQL", "target_language": job.target_language or job.target_dialect or "en", "source_dialect": job.source_dialect or "", "target_dialect": job.target_dialect or "", "translation_column": job.translation_column or "", "dictionary_section": dictionary_section, "rows_json": rows_json, "row_count": str(len(batch_rows)), }, ) # Call LLM with retry llm_response = None last_error = None retries = 0 for attempt in range(1, MAX_RETRIES_PER_BATCH + 1): try: llm_response = self._call_llm(job, prompt) break except Exception as e: last_error = str(e) retries += 1 logger.explore(f"LLM call failed (attempt {attempt})", { "batch_id": batch_id, "error": last_error, "attempt": attempt, }) if attempt < MAX_RETRIES_PER_BATCH: time.sleep(2 ** attempt) # Exponential backoff else: logger.explore("LLM call exhausted retries", { "batch_id": batch_id, "last_error": last_error, }) if llm_response is None: # All retries failed — mark all rows as failed for row in batch_rows: record = TranslationRecord( id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id, source_sql=row.get("source_text", ""), target_sql=None, source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), source_data=row.get("source_data"), status="FAILED", error_message=f"LLM call failed after {retries} retries: {last_error}", ) self.db.add(record) return {"successful": 0, "failed": len(batch_rows), "skipped": 0, "retries": retries} # Parse LLM response try: translations = self._parse_llm_response(llm_response, len(batch_rows)) except ValueError as e: # Parse failure — mark all rows as SKIPPED logger.explore("LLM response parse failed", { "batch_id": batch_id, "error": str(e), "response_preview": llm_response[:500] if llm_response else "", }) skipped = len(batch_rows) for row in batch_rows: record = TranslationRecord( id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id, source_sql=row.get("source_text", ""), target_sql=None, source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), source_data=row.get("source_data"), status="SKIPPED", error_message=f"LLM parse failure: {e}", ) self.db.add(record) return { "successful": 0, "failed": 0, "skipped": skipped, "retries": retries, } successful = 0 failed = 0 skipped = 0 for row in batch_rows: row_id = str(row.get("row_index", "")) translation = translations.get(row_id) if translation is None: # NULL translation — skip skipped += 1 record = TranslationRecord( id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id, source_sql=row.get("source_text", ""), target_sql="", source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), source_data=row.get("source_data"), status="SKIPPED", error_message="NULL translation returned by LLM", ) self.db.add(record) continue if translation.strip() == "": # Empty translation — skip skipped += 1 record = TranslationRecord( id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id, source_sql=row.get("source_text", ""), target_sql="", source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), source_data=row.get("source_data"), status="SKIPPED", error_message="Empty translation returned by LLM", ) self.db.add(record) continue successful += 1 record = TranslationRecord( id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id, source_sql=row.get("source_text", ""), target_sql=translation, source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), source_data=row.get("source_data"), status="SUCCESS", ) self.db.add(record) return { "successful": successful, "failed": failed, "skipped": skipped, "retries": retries, } # endregion _call_llm_for_batch # region _call_llm [TYPE Function] # @PURPOSE: Call the configured LLM provider with the batch prompt. # @PRE: job has valid provider_id. # @POST: Returns raw LLM response string. # @SIDE_EFFECT: HTTP call to LLM provider. def _call_llm(self, job: TranslationJob, prompt: str) -> str: with belief_scope("TranslationExecutor._call_llm"): if not job.provider_id: raise ValueError("Job has no LLM provider configured") provider_svc = LLMProviderService(self.db) provider = provider_svc.get_provider(job.provider_id) if not provider: raise ValueError(f"LLM provider '{job.provider_id}' not found") api_key = provider_svc.get_decrypted_api_key(job.provider_id) if not api_key: raise ValueError(f"Could not decrypt API key for provider '{job.provider_id}'") model = provider.default_model or "gpt-4o-mini" provider_type = provider.provider_type.lower() if provider.provider_type else "openai" if provider_type in ("openai", "openai_compatible", "openrouter", "kilo"): return self._call_openai_compatible( base_url=provider.base_url, api_key=api_key, model=model, prompt=prompt, provider_type=provider_type, ) else: raise ValueError(f"Unsupported provider type '{provider_type}'") # endregion _call_llm # region _call_openai_compatible [TYPE Function] # @PURPOSE: Call OpenAI-compatible API for batch translation. # @PRE: Valid API endpoint, key, model, and prompt. # @POST: Returns response text. # @SIDE_EFFECT: HTTP POST to LLM API. @staticmethod def _call_openai_compatible( base_url: str, api_key: str, model: str, prompt: str, provider_type: str = "openai", ) -> str: with belief_scope("TranslationExecutor._call_openai_compatible"): import requests as http_requests url = f"{base_url.rstrip('/')}/chat/completions" headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", } payload = { "model": model, "messages": [ {"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics."}, {"role": "user", "content": prompt}, ], "temperature": 0.1, "max_tokens": 4096, } # Structured output (response_format) only for native OpenAI — upstream providers routed via # Kilo/OpenRouter may not support it (e.g. StepFun returns "structured_outputs is not supported") if provider_type in ("openai", "openai_compatible"): payload["response_format"] = {"type": "json_object"} logger.reason( f"LLM request model={payload.get('model')} " f"provider_type={provider_type} " f"response_format={'yes' if 'response_format' in payload else 'no'} " f"prompt_len={len(prompt)}" ) response = http_requests.post(url, headers=headers, json=payload, timeout=180) if not response.ok: logger.explore( f"LLM API error status={response.status_code} " f"model={payload.get('model')} " f"body={response.text[:2000]}" ) response.raise_for_status() data = response.json() choices = data.get("choices", []) if not choices: raise ValueError("LLM returned no choices") content = choices[0].get("message", {}).get("content", "") if not content: raise ValueError("LLM returned empty content") return content # endregion _call_openai_compatible # region _parse_llm_response [TYPE Function] # @PURPOSE: Parse LLM JSON response into dict of row_id -> translation. # @PRE: response_text is valid JSON with {"rows": [...]} structure. # @POST: Returns dict mapping row_id to translation text. @staticmethod def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]: with belief_scope("TranslationExecutor._parse_llm_response"): try: data = json.loads(response_text) except json.JSONDecodeError: # Try to extract from markdown code block import re match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', response_text, re.DOTALL) if match: try: data = json.loads(match.group(1)) except json.JSONDecodeError: raise ValueError("LLM response was not valid JSON") else: raise ValueError("LLM response was not valid JSON") rows = data.get("rows", []) if not isinstance(rows, list): raise ValueError("LLM response missing 'rows' array") translations: Dict[str, str] = {} for item in rows: row_id = str(item.get("row_id", "")) translation = item.get("translation") if translation is None: # Skip NULL translations — they'll be handled by caller continue if row_id: translations[row_id] = str(translation) return translations # endregion _parse_llm_response # #endregion TranslationExecutor # #endregion TranslationExecutor