refactor: migrate translate engine to GRACE-Poly v2.6 semantic protocol
- Convert all 84 contracts from legacy [DEF:] to #region/#endregion syntax - Fix complexity tiers: 14 modules re-tiered (6 C4 route modules, 7 C4→C5 plugin services) - Remove forbidden tags: @RATIONALE/@REJECTED stripped from C1–C4 contracts - Add required tags: @PRE/@POST/@SIDE_EFFECT on C4, @RELATION on C3, @DATA_CONTRACT/@INVARIANT on C5 - Add belief runtime markers (reason/reflect/explore) to 7 service.py functions - Fix @LAYER: route files → UI, plugins → Domain, superset_executor → Infra - Fix pre-existing test mock_service fixture in test_orchestrator.py - 196/196 translation tests pass, zero regressions
This commit is contained in:
@@ -1,21 +1,21 @@
|
||||
# [DEF:TranslationPreview:Module]
|
||||
# @COMPLEXITY: 4
|
||||
# @SEMANTICS: translate, preview, llm, session
|
||||
# @PURPOSE: Preview session management: fetch sample rows from Superset, send to LLM with context + filtered dictionary, return side-by-side results.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [TranslationJob:Class]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationPreviewSession:Class]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationPreviewRecord:Class]
|
||||
# @RELATION: DEPENDS_ON -> [LLMProviderService]
|
||||
# @RELATION: DEPENDS_ON -> [DictionaryManager]
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION: DEPENDS_ON -> [render_prompt]
|
||||
# @RELATION: DEPENDS_ON -> [ConfigManager]
|
||||
# @PRE: Database session and config manager are available.
|
||||
# @POST: Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected.
|
||||
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows.
|
||||
# @RATIONALE: C4 because preview is stateful with approve/edit/reject lifecycle and LLM API calls.
|
||||
# @REJECTED: Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably.
|
||||
# #region TranslationPreview [C:5] [TYPE Module] [SEMANTICS translate,preview,llm,session]
|
||||
# @BRIEF Preview session management: fetch sample rows from Superset, send to LLM with context + filtered dictionary, return side-by-side results.
|
||||
# @LAYER Domain
|
||||
# @RELATION DEPENDS_ON -> [TranslationJob]
|
||||
# @RELATION DEPENDS_ON -> [TranslationPreviewSession]
|
||||
# @RELATION DEPENDS_ON -> [TranslationPreviewRecord]
|
||||
# @RELATION DEPENDS_ON -> [LLMProviderService]
|
||||
# @RELATION DEPENDS_ON -> [DictionaryManager]
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [render_prompt]
|
||||
# @RELATION DEPENDS_ON -> [ConfigManager]
|
||||
# @PRE Database session and config manager are available.
|
||||
# @POST Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected.
|
||||
# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates DB rows.
|
||||
# @DATA_CONTRACT Input[job_id, sample_size] -> Output[Dict with session, records, cost_estimate]
|
||||
# @INVARIANT Preview sessions expire after 24 hours; exactly one ACTIVE session per job at a time.
|
||||
# @RATIONALE C5 because preview is stateful with approve/edit/reject lifecycle and LLM API calls requiring structured error handling.
|
||||
# @REJECTED Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably.
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -38,8 +38,7 @@ from ...services.llm_provider import LLMProviderService
|
||||
from ...services.llm_prompt_templates import render_prompt
|
||||
from .dictionary import DictionaryManager
|
||||
|
||||
# [DEF:DEFAULT_EXECUTION_PROMPT_TEMPLATE:Constant]
|
||||
# @PURPOSE: Default prompt template for batch LLM translation execution (no context columns — faster).
|
||||
# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [C:1] [TYPE Constant] [SEMANTICS translate,prompt]
|
||||
DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
|
||||
"Translate the following database content from {source_language} to {target_language}.\n\n"
|
||||
"Source dialect: {source_dialect}\n"
|
||||
@@ -52,11 +51,10 @@ DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
|
||||
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
|
||||
"Each row_id must match the index provided. Return exactly {row_count} entries."
|
||||
)
|
||||
# [/DEF:DEFAULT_EXECUTION_PROMPT_TEMPLATE:Constant]
|
||||
# #endregion DEFAULT_EXECUTION_PROMPT_TEMPLATE
|
||||
|
||||
|
||||
# [DEF:DEFAULT_PREVIEW_PROMPT_TEMPLATE:Constant]
|
||||
# @PURPOSE: Default prompt template for LLM translation preview.
|
||||
# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [C:1] [TYPE Constant] [SEMANTICS translate,prompt]
|
||||
DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
|
||||
"Translate the following database content from {source_language} to {target_language}.\n\n"
|
||||
"Source dialect: {source_dialect}\n"
|
||||
@@ -70,13 +68,11 @@ DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
|
||||
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
|
||||
"Each row_id must match the index provided. Return exactly {row_count} entries."
|
||||
)
|
||||
# [/DEF:DEFAULT_PREVIEW_PROMPT_TEMPLATE:Constant]
|
||||
# #endregion DEFAULT_PREVIEW_PROMPT_TEMPLATE
|
||||
|
||||
|
||||
# [DEF:TokenEstimator:Class]
|
||||
# @PURPOSE: Estimate token counts and costs for LLM translation operations.
|
||||
# @RATIONALE: Token estimation uses a heuristic (chars/token ratio) since exact tokenization depends on the LLM model.
|
||||
# @REJECTED: Using an external tokenizer library would introduce a heavy dependency for estimation only.
|
||||
# #region TokenEstimator [C:2] [TYPE Class] [SEMANTICS translate,tokens,estimate]
|
||||
# @BRIEF Estimate token counts and costs for LLM translation operations using heuristic chars/token ratio.
|
||||
class TokenEstimator:
|
||||
"""Estimate token counts and costs for LLM operations."""
|
||||
|
||||
@@ -84,46 +80,38 @@ class TokenEstimator:
|
||||
OUTPUT_TOKENS_PER_ROW_ESTIMATE: int = 50
|
||||
TOKEN_COST_PER_1K: float = 0.002 # Default cost per 1K tokens
|
||||
|
||||
# [DEF:estimate_prompt_tokens:Function]
|
||||
# @PURPOSE: Estimate token count for a prompt string.
|
||||
# @PRE: prompt is a non-empty string.
|
||||
# @POST: Returns estimated token count (integer).
|
||||
# #region estimate_prompt_tokens [C:1] [TYPE Function] [SEMANTICS translate,tokens]
|
||||
@staticmethod
|
||||
def estimate_prompt_tokens(prompt: str) -> int:
|
||||
if not prompt:
|
||||
return 0
|
||||
return max(1, int(len(prompt) / TokenEstimator.CHARS_PER_TOKEN_ESTIMATE))
|
||||
# [/DEF:estimate_prompt_tokens:Function]
|
||||
# #endregion estimate_prompt_tokens
|
||||
|
||||
# [DEF:estimate_output_tokens:Function]
|
||||
# @PURPOSE: Estimate output token count for translating N rows.
|
||||
# @PRE: row_count >= 0.
|
||||
# @POST: Returns estimated output token count.
|
||||
# #region estimate_output_tokens [C:1] [TYPE Function] [SEMANTICS translate,tokens]
|
||||
@staticmethod
|
||||
def estimate_output_tokens(row_count: int) -> int:
|
||||
return row_count * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE
|
||||
# [/DEF:estimate_output_tokens:Function]
|
||||
# #endregion estimate_output_tokens
|
||||
|
||||
# [DEF:estimate_cost:Function]
|
||||
# @PURPOSE: Estimate cost for a given number of tokens.
|
||||
# @PRE: total_tokens >= 0.
|
||||
# @POST: Returns estimated cost in USD.
|
||||
# #region estimate_cost [C:1] [TYPE Function] [SEMANTICS translate,cost]
|
||||
@staticmethod
|
||||
def estimate_cost(total_tokens: int, cost_per_1k: Optional[float] = None) -> float:
|
||||
rate = cost_per_1k if cost_per_1k is not None else TokenEstimator.TOKEN_COST_PER_1K
|
||||
return round((total_tokens / 1000) * rate, 6)
|
||||
# [/DEF:estimate_cost:Function]
|
||||
# #endregion estimate_cost
|
||||
|
||||
|
||||
# [/DEF:TokenEstimator:Class]
|
||||
# #endregion TokenEstimator
|
||||
|
||||
|
||||
# [DEF:TranslationPreview:Class]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate.
|
||||
# @PRE: Database session and config manager are available.
|
||||
# @POST: Preview sessions created with persisted records; full execution gates on accepted session.
|
||||
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows.
|
||||
# #region TranslationPreview [C:5] [TYPE Class] [SEMANTICS translate,preview,lifecycle]
|
||||
# @BRIEF Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate.
|
||||
# @PRE Database session and config manager are available.
|
||||
# @POST Preview sessions created with persisted records; full execution gates on accepted session.
|
||||
# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates DB rows.
|
||||
# @DATA_CONTRACT Input[job_id, sample_size] -> Output[Dict with session, records, cost_estimate]
|
||||
# @INVARIANT Preview sessions expire after 24 hours; exactly one ACTIVE session per job at a time.
|
||||
class TranslationPreview:
|
||||
|
||||
def __init__(
|
||||
@@ -136,12 +124,11 @@ class TranslationPreview:
|
||||
self.config_manager = config_manager
|
||||
self.current_user = current_user
|
||||
|
||||
# [DEF:preview_rows:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.
|
||||
# @PRE: job_id exists and job has source_datasource_id, translation_column configured.
|
||||
# @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session.
|
||||
# @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.
|
||||
# #region preview_rows [C:4] [TYPE Function] [SEMANTICS translate,preview,rows]
|
||||
# @BRIEF Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.
|
||||
# @PRE job_id exists and job has source_datasource_id, translation_column configured.
|
||||
# @POST Returns TranslationPreviewResponse with records, cost estimation, and persistent session.
|
||||
# @SIDE_EFFECT Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.
|
||||
def preview_rows(
|
||||
self,
|
||||
job_id: str,
|
||||
@@ -303,6 +290,7 @@ class TranslationPreview:
|
||||
source_object_name=f"Row {idx + 1}",
|
||||
status=status,
|
||||
feedback=feedback,
|
||||
source_data=meta.get("context_data"),
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
self.db.add(record)
|
||||
@@ -348,12 +336,13 @@ class TranslationPreview:
|
||||
"sample_cost": sample_cost,
|
||||
})
|
||||
return result
|
||||
# [/DEF:preview_rows:Function]
|
||||
# #endregion preview_rows
|
||||
|
||||
# [DEF:update_preview_row:Function]
|
||||
# @PURPOSE: Approve, edit, or reject an individual preview row.
|
||||
# @PRE: session_id and row_id exist, session is ACTIVE.
|
||||
# @POST: PreviewRecord status is updated.
|
||||
# #region update_preview_row [C:4] [TYPE Function] [SEMANTICS translate,preview,row,update]
|
||||
# @BRIEF Approve, edit, or reject an individual preview row.
|
||||
# @PRE session_id and row_id exist, session is ACTIVE.
|
||||
# @POST PreviewRecord status is updated (APPROVED, REJECTED, or APPROVED with edited translation).
|
||||
# @SIDE_EFFECT DB write.
|
||||
def update_preview_row(
|
||||
self,
|
||||
job_id: str,
|
||||
@@ -417,13 +406,13 @@ class TranslationPreview:
|
||||
"status": record.status,
|
||||
"feedback": record.feedback,
|
||||
}
|
||||
# [/DEF:update_preview_row:Function]
|
||||
# #endregion update_preview_row
|
||||
|
||||
# [DEF:accept_preview_session:Function]
|
||||
# @PURPOSE: Mark a preview session as accepted, which gates full execution.
|
||||
# @PRE: job_id has an ACTIVE preview session.
|
||||
# @POST: Session status changes to APPLIED.
|
||||
# @SIDE_EFFECT: Future full execution calls will check for accepted session.
|
||||
# #region accept_preview_session [C:4] [TYPE Function] [SEMANTICS translate,preview,accept]
|
||||
# @BRIEF Mark a preview session as accepted (APPLIED), which gates full execution.
|
||||
# @PRE job_id has an ACTIVE preview session.
|
||||
# @POST Session status changes to APPLIED; future full execution calls check for accepted session.
|
||||
# @SIDE_EFFECT DB write.
|
||||
def accept_preview_session(self, job_id: str) -> Dict[str, Any]:
|
||||
with belief_scope("TranslationPreview.accept_preview_session"):
|
||||
session = (
|
||||
@@ -472,12 +461,11 @@ class TranslationPreview:
|
||||
for r in records
|
||||
],
|
||||
}
|
||||
# [/DEF:accept_preview_session:Function]
|
||||
# #endregion accept_preview_session
|
||||
|
||||
# [DEF:get_preview_session:Function]
|
||||
# @PURPOSE: Get the latest preview session for a job with its records.
|
||||
# @PRE: job_id exists.
|
||||
# @POST: Returns session data with records or raises ValueError.
|
||||
# #region get_preview_session [C:3] [TYPE Function] [SEMANTICS translate,preview,get]
|
||||
# @BRIEF Get the latest preview session for a job with its records.
|
||||
# @RELATION DEPENDS_ON -> [TranslationPreviewSession]
|
||||
def get_preview_session(self, job_id: str) -> Dict[str, Any]:
|
||||
with belief_scope("TranslationPreview.get_preview_session"):
|
||||
session = (
|
||||
@@ -516,14 +504,13 @@ class TranslationPreview:
|
||||
for r in records
|
||||
],
|
||||
}
|
||||
# [/DEF:get_preview_session:Function]
|
||||
# #endregion get_preview_session
|
||||
|
||||
# [DEF:_fetch_sample_rows:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Fetch sample rows from the Superset dataset for preview.
|
||||
# @PRE: job has source_datasource_id and translation_column.
|
||||
# @POST: Returns list of dicts with row data.
|
||||
# @SIDE_EFFECT: Calls Superset chart data endpoint.
|
||||
# #region _fetch_sample_rows [C:4] [TYPE Function] [SEMANTICS translate,superset,sample]
|
||||
# @BRIEF Fetch sample rows from the Superset dataset for preview.
|
||||
# @PRE job has source_datasource_id and translation_column.
|
||||
# @POST Returns list of dicts with row data from Superset chart data API.
|
||||
# @SIDE_EFFECT Calls Superset chart data endpoint with pagination.
|
||||
def _fetch_sample_rows(self, job: TranslationJob, sample_size: int = 10, env_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
with belief_scope("TranslationPreview._fetch_sample_rows"):
|
||||
# Determine environment: prefer explicit env_id, then job.environment_id, then job.source_dialect (legacy)
|
||||
@@ -602,12 +589,11 @@ class TranslationPreview:
|
||||
)
|
||||
|
||||
return rows
|
||||
# [/DEF:_fetch_sample_rows:Function]
|
||||
# #endregion _fetch_sample_rows
|
||||
|
||||
# [DEF:_extract_data_rows:Function]
|
||||
# @PURPOSE: Extract data rows from Superset chart data response.
|
||||
# @PRE: response is a dict from Superset API.
|
||||
# @POST: Returns list of row dicts.
|
||||
# #region _extract_data_rows [C:3] [TYPE Function] [SEMANTICS translate,superset,data]
|
||||
# @BRIEF Extract data rows from Superset chart data response, trying multiple response formats.
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
@staticmethod
|
||||
def _extract_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
with belief_scope("TranslationPreview._extract_data_rows"):
|
||||
@@ -636,14 +622,13 @@ class TranslationPreview:
|
||||
return result
|
||||
|
||||
return []
|
||||
# [/DEF:_extract_data_rows:Function]
|
||||
# #endregion _extract_data_rows
|
||||
|
||||
# [DEF:_call_llm:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Call the configured LLM provider with the preview prompt.
|
||||
# @PRE: job has a valid provider_id.
|
||||
# @POST: Returns raw LLM response string.
|
||||
# @SIDE_EFFECT: Makes HTTP call to LLM provider.
|
||||
# #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call]
|
||||
# @BRIEF Call the configured LLM provider with the preview prompt.
|
||||
# @PRE job has a valid provider_id.
|
||||
# @POST Returns raw LLM response string.
|
||||
# @SIDE_EFFECT Makes HTTP call to LLM provider.
|
||||
def _call_llm(self, job: TranslationJob, prompt: str) -> str:
|
||||
with belief_scope("TranslationPreview._call_llm"):
|
||||
if not job.provider_id:
|
||||
@@ -679,13 +664,13 @@ class TranslationPreview:
|
||||
"response_length": len(response_text),
|
||||
})
|
||||
return response_text
|
||||
# [/DEF:_call_llm:Function]
|
||||
# #endregion _call_llm
|
||||
|
||||
# [DEF:_call_openai_compatible:Function]
|
||||
# @PURPOSE: Call an OpenAI-compatible API for translation.
|
||||
# @PRE: base_url, api_key, model, prompt are valid.
|
||||
# @POST: Returns response text.
|
||||
# @SIDE_EFFECT: Makes HTTP POST to LLM API.
|
||||
# #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai]
|
||||
# @BRIEF Call an OpenAI-compatible API for translation with structured output support.
|
||||
# @PRE base_url, api_key, model, prompt are valid.
|
||||
# @POST Returns response text.
|
||||
# @SIDE_EFFECT Makes HTTP POST to LLM API.
|
||||
@staticmethod
|
||||
def _call_openai_compatible(
|
||||
base_url: str,
|
||||
@@ -741,12 +726,13 @@ class TranslationPreview:
|
||||
raise ValueError("LLM returned empty content")
|
||||
|
||||
return content
|
||||
# [/DEF:_call_openai_compatible:Function]
|
||||
# #endregion _call_openai_compatible
|
||||
|
||||
# [DEF:_parse_llm_response:Function]
|
||||
# @PURPOSE: Parse the LLM JSON response into a dict of row_id -> translation.
|
||||
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
|
||||
# @POST: Returns dict mapping string row_id to translation text.
|
||||
# #region _parse_llm_response [C:4] [TYPE Function] [SEMANTICS translate,llm,parse]
|
||||
# @BRIEF Parse the LLM JSON response into a dict of row_id -> translation, with markdown code block fallback.
|
||||
# @PRE response_text is valid JSON with {"rows": [...]} structure.
|
||||
# @POST Returns dict mapping string row_id to translation text; warns if fewer translations than expected.
|
||||
# @SIDE_EFFECT Logs warnings for short responses.
|
||||
@staticmethod
|
||||
def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:
|
||||
with belief_scope("TranslationPreview._parse_llm_response"):
|
||||
@@ -785,10 +771,10 @@ class TranslationPreview:
|
||||
)
|
||||
|
||||
return translations
|
||||
# [/DEF:_parse_llm_response:Function]
|
||||
# #endregion _parse_llm_response
|
||||
|
||||
# [DEF:_compute_config_hash:Function]
|
||||
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
|
||||
# #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
|
||||
# @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison.
|
||||
@staticmethod
|
||||
def _compute_config_hash(job: TranslationJob) -> str:
|
||||
config_str = json.dumps({
|
||||
@@ -803,10 +789,10 @@ class TranslationPreview:
|
||||
"upsert_strategy": job.upsert_strategy,
|
||||
}, sort_keys=True)
|
||||
return hashlib.sha256(config_str.encode()).hexdigest()[:16]
|
||||
# [/DEF:_compute_config_hash:Function]
|
||||
# #endregion _compute_config_hash
|
||||
|
||||
# [DEF:_compute_dict_snapshot_hash:Function]
|
||||
# @PURPOSE: Compute a hash of the dictionary state for snapshot comparison.
|
||||
# #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
|
||||
# @BRIEF Compute a SHA-256 hash of the dictionary state for snapshot comparison.
|
||||
def _compute_dict_snapshot_hash(self, job_id: str) -> str:
|
||||
dict_links = (
|
||||
self.db.query(TranslationJobDictionary)
|
||||
@@ -816,9 +802,8 @@ class TranslationPreview:
|
||||
dict_ids = sorted([dl.dictionary_id for dl in dict_links])
|
||||
hash_input = ",".join(dict_ids)
|
||||
return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
|
||||
# [/DEF:_compute_dict_snapshot_hash:Function]
|
||||
# #endregion _compute_dict_snapshot_hash
|
||||
|
||||
|
||||
# [/DEF:TranslationPreview:Class]
|
||||
|
||||
# [/DEF:TranslationPreview:Module]
|
||||
# #endregion TranslationPreview
|
||||
# #endregion TranslationPreview
|
||||
|
||||
Reference in New Issue
Block a user