busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: migrate translate engine to GRACE-Poly v2.6 semantic protocol

Browse Source 
- Convert all 84 contracts from legacy [DEF:] to #region/#endregion syntax
- Fix complexity tiers: 14 modules re-tiered (6 C4 route modules, 7 C4→C5 plugin services)
- Remove forbidden tags: @RATIONALE/@REJECTED stripped from C1–C4 contracts
- Add required tags: @PRE/@POST/@SIDE_EFFECT on C4, @RELATION on C3, @DATA_CONTRACT/@INVARIANT on C5
- Add belief runtime markers (reason/reflect/explore) to 7 service.py functions
- Fix @LAYER: route files → UI, plugins → Domain, superset_executor → Infra
- Fix pre-existing test mock_service fixture in test_orchestrator.py
- 196/196 translation tests pass, zero regressions
This commit is contained in:
busya
2026-05-12 14:32:28 +03:00
parent fefdee98d0
commit 9f995f22ae
29 changed files with 1594 additions and 1292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
backend/src/api/routes/translate/__init__.py
View File

@@ -1,17 +1,16 @@
# [DEF:TranslateRoutes:Module]
# @COMPLEXITY: 4
# @SEMANTICS: api, routes, translate
# @PURPOSE: API routes for LLM-based SQL/dashboard translation management including terminology dictionary CRUD and import.
# @LAYER: UI (API)
# @RELATION: DEPENDS_ON -> [TranslateJobResponse]
# @RELATION: DEPENDS_ON -> [DictionaryManager:Class]
# @RELATION: DEPENDS_ON -> [get_current_user]
# @RELATION: DEPENDS_ON -> [get_db]
# @RELATION: DEPENDS_ON -> [has_permission]
# @RELATION: DEPENDS_ON -> [ConfigManager]
# @RELATION: DEPENDS_ON -> [SupersetClient]
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity.
# #region TranslateRoutes [C:4] [TYPE Module] [SEMANTICS api,routes,translate]
# @BRIEF API routes for LLM-based SQL/dashboard translation management including terminology dictionary CRUD and import.
# @LAYER UI
# @RELATION DEPENDS_ON -> [TranslateJobResponse]
# @RELATION DEPENDS_ON -> [DictionaryManager]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
# @RELATION DEPENDS_ON -> [has_permission]
# @RELATION DEPENDS_ON -> [ConfigManager]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @PRE All sub-modules importable. Router instance available from ._router.
# @POST Package imports all route modules and re-exports router. Sub-routes registered on shared router.
# @SIDE_EFFECT Registers route handlers on shared APIRouter at import time.
"""
Translate routes package.
@@ -34,4 +33,4 @@ __all__ = [
"router",
]
# [/DEF:TranslateRoutes:Module]
# #endregion TranslateRoutes

40
backend/src/api/routes/translate/_correction_routes.py
View File

@@ -1,8 +1,12 @@
# [DEF:TranslateCorrectionRoutesModule:Module]
# @COMPLEXITY: 2
# @PURPOSE: Term correction submission endpoints.
# @LAYER: API
# @SEMANTICS: api, routes, translate, corrections
# #region TranslateCorrectionRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,corrections]
# @BRIEF Term correction submission endpoints for applying user feedback to dictionary entries.
# @LAYER UI
# @RELATION DEPENDS_ON -> [DictionaryManager]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
# @PRE DB session initialized. User authenticated with translate.dictionary.edit permissions.
# @POST Term corrections submitted and applied to dictionary with conflict detection.
# @SIDE_EFFECT Creates/updates DictionaryEntry records in DB based on correction submissions.
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
@@ -26,10 +30,12 @@ from ._router import router
# Corrections
# ============================================================
# [DEF:submit_correction:Function]
# @PURPOSE: Submit a term correction from a run result review.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Correction applied with conflict detection.
# #region submit_correction [C:4] [TYPE Function] [SEMANTICS api,translate,corrections]
# @BRIEF Submit a single term correction from a run result review.
# @PRE User has translate.dictionary.edit permission. Dictionary ID is provided.
# @POST Correction applied with conflict detection. Result returned.
# @SIDE_EFFECT Creates/updates DictionaryEntry record in DB; logs correction origin.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.post("/corrections")
async def submit_correction(
payload: TermCorrectionSubmit,
@@ -55,13 +61,15 @@ async def submit_correction(
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:submit_correction:Function]
# #endregion submit_correction
# [DEF:submit_bulk_corrections:Function]
# @PURPOSE: Submit multiple term corrections atomically.
# @PRE: User has translate.dictionary.edit permission.
# @POST: All corrections applied or none with conflict list.
# #region submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS api,translate,corrections]
# @BRIEF Submit multiple term corrections atomically with full conflict reporting.
# @PRE User has translate.dictionary.edit permission. Dictionary ID is provided.
# @POST All corrections applied or none with conflict list returned.
# @SIDE_EFFECT Creates/updates multiple DictionaryEntry records in DB atomically.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.post("/corrections/bulk")
async def submit_bulk_corrections(
payload: TermCorrectionBulkSubmit,
@@ -92,6 +100,6 @@ async def submit_bulk_corrections(
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:submit_bulk_corrections:Function]
# #endregion submit_bulk_corrections
# [/DEF:TranslateCorrectionRoutesModule:Module]
# #endregion TranslateCorrectionRoutesModule

140
backend/src/api/routes/translate/_dictionary_routes.py
View File

@@ -1,8 +1,12 @@
# [DEF:TranslateDictionaryRoutesModule:Module]
# @COMPLEXITY: 3
# @PURPOSE: Terminology Dictionary CRUD, entries management, and import routes.
# @LAYER: API
# @SEMANTICS: api, routes, translate, dictionaries
# #region TranslateDictionaryRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,dictionaries]
# @BRIEF Terminology Dictionary CRUD, entries management, and import routes.
# @LAYER UI
# @RELATION DEPENDS_ON -> [DictionaryManager]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
# @PRE DB session initialized. User authenticated with translate.dictionary permissions.
# @POST Dictionaries and entries CRUD completed. Import executed with conflict resolution.
# @SIDE_EFFECT Reads/writes TerminologyDictionary and DictionaryEntry records to DB.
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
@@ -29,10 +33,12 @@ from ._helpers import _dict_to_response, _get_dictionary_entry_counts
# Terminology Dictionaries
# ============================================================
# [DEF:list_dictionaries:Function]
# @PURPOSE: List all terminology dictionaries.
# @PRE: User has translate.dictionary.view permission.
# @POST: Returns list of dictionaries with total count.
# #region list_dictionaries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]
# @BRIEF List all terminology dictionaries with pagination and entry counts.
# @PRE User has translate.dictionary.view permission.
# @POST Returns paginated list of dictionaries with total count.
# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.get("/dictionaries")
async def list_dictionaries(
page: int = Query(1, ge=1),
@@ -50,13 +56,15 @@ async def list_dictionaries(
items = [_dict_to_response(d, counts.get(d.id, 0)) for d in dicts]
logger.reflect("Dictionaries listed", {"total": total, "returned": len(items)})
return {"items": items, "total": total, "page": page, "page_size": page_size}
# [/DEF:list_dictionaries:Function]
# #endregion list_dictionaries
# [DEF:get_dictionary:Function]
# @PURPOSE: Get a single terminology dictionary by ID.
# @PRE: User has translate.dictionary.view permission.
# @POST: Returns the dictionary with entry_count.
# #region get_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]
# @BRIEF Get a single terminology dictionary by ID with entry count.
# @PRE User has translate.dictionary.view permission.
# @POST Returns the dictionary with entry_count or 404.
# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.get("/dictionaries/{dictionary_id}")
async def get_dictionary(
dictionary_id: str,
@@ -69,19 +77,21 @@ async def get_dictionary(
logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}")
try:
d = DictionaryManager.get_dictionary(db, dictionary_id)
from ...models.translate import DictionaryEntry
from ....models.translate import DictionaryEntry
entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count()
logger.reflect("Dictionary found", {"id": dictionary_id})
return _dict_to_response(d, entry_count)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_dictionary:Function]
# #endregion get_dictionary
# [DEF:create_dictionary:Function]
# @PURPOSE: Create a new terminology dictionary.
# @PRE: User has translate.dictionary.create permission.
# @POST: Returns the created dictionary.
# #region create_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]
# @BRIEF Create a new terminology dictionary.
# @PRE User has translate.dictionary.create permission. Payload validated.
# @POST Dictionary created and returned with zero entries.
# @SIDE_EFFECT Creates TerminologyDictionary record in DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.post("/dictionaries", status_code=status.HTTP_201_CREATED)
async def create_dictionary(
payload: DictionaryCreate,
@@ -103,13 +113,15 @@ async def create_dictionary(
)
logger.reflect("Dictionary created", {"id": d.id})
return _dict_to_response(d, 0)
# [/DEF:create_dictionary:Function]
# #endregion create_dictionary
# [DEF:update_dictionary:Function]
# @PURPOSE: Update an existing terminology dictionary.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Returns the updated dictionary.
# #region update_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]
# @BRIEF Update an existing terminology dictionary.
# @PRE User has translate.dictionary.edit permission. Dictionary exists.
# @POST Dictionary updated and returned with current entry count.
# @SIDE_EFFECT Updates TerminologyDictionary record in DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.put("/dictionaries/{dictionary_id}")
async def update_dictionary(
dictionary_id: str,
@@ -131,19 +143,21 @@ async def update_dictionary(
target_dialect=payload.target_dialect,
is_active=payload.is_active,
)
from ...models.translate import DictionaryEntry
from ....models.translate import DictionaryEntry
entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count()
logger.reflect("Dictionary updated", {"id": dictionary_id})
return _dict_to_response(d, entry_count)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:update_dictionary:Function]
# #endregion update_dictionary
# [DEF:delete_dictionary:Function]
# @PURPOSE: Delete a terminology dictionary, blocked if attached to active/scheduled jobs.
# @PRE: User has translate.dictionary.delete permission.
# @POST: Dictionary is deleted.
# #region delete_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]
# @BRIEF Delete a terminology dictionary, blocked if attached to active/scheduled jobs.
# @PRE User has translate.dictionary.delete permission. Dictionary exists and is not in use.
# @POST Dictionary is deleted or 409 if attached to active jobs.
# @SIDE_EFFECT Deletes TerminologyDictionary record from DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.delete("/dictionaries/{dictionary_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_dictionary(
dictionary_id: str,
@@ -162,17 +176,19 @@ async def delete_dictionary(
if "active/scheduled" in str(e):
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:delete_dictionary:Function]
# #endregion delete_dictionary
# ============================================================
# Dictionary Entries CRUD
# ============================================================
# [DEF:list_dictionary_entries:Function]
# @PURPOSE: List entries for a dictionary.
# @PRE: User has translate.dictionary.view permission.
# @POST: Returns paginated list of entries.
# #region list_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]
# @BRIEF List entries for a dictionary with pagination.
# @PRE User has translate.dictionary.view permission. Dictionary exists.
# @POST Returns paginated list of entries.
# @SIDE_EFFECT Reads DictionaryEntry records from DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.get("/dictionaries/{dictionary_id}/entries")
async def list_dictionary_entries(
dictionary_id: str,
@@ -204,13 +220,15 @@ async def list_dictionary_entries(
return {"items": items, "total": total, "page": page, "page_size": page_size}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:list_dictionary_entries:Function]
# #endregion list_dictionary_entries
# [DEF:add_dictionary_entry:Function]
# @PURPOSE: Add a new entry to a dictionary.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Entry is created.
# #region add_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]
# @BRIEF Add a new entry to a dictionary.
# @PRE User has translate.dictionary.edit permission. Dictionary exists.
# @POST Entry created and returned.
# @SIDE_EFFECT Creates DictionaryEntry record in DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.post("/dictionaries/{dictionary_id}/entries", status_code=status.HTTP_201_CREATED)
async def add_dictionary_entry(
dictionary_id: str,
@@ -244,13 +262,15 @@ async def add_dictionary_entry(
if "already exists" in str(e):
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:add_dictionary_entry:Function]
# #endregion add_dictionary_entry
# [DEF:edit_dictionary_entry:Function]
# @PURPOSE: Update an existing dictionary entry.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Entry is updated.
# #region edit_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]
# @BRIEF Update an existing dictionary entry.
# @PRE User has translate.dictionary.edit permission. Entry exists.
# @POST Entry updated and returned.
# @SIDE_EFFECT Updates DictionaryEntry record in DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.put("/dictionaries/{dictionary_id}/entries/{entry_id}")
async def edit_dictionary_entry(
dictionary_id: str,
@@ -285,13 +305,15 @@ async def edit_dictionary_entry(
if "already exists" in str(e):
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:edit_dictionary_entry:Function]
# #endregion edit_dictionary_entry
# [DEF:delete_dictionary_entry:Function]
# @PURPOSE: Delete a dictionary entry.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Entry is deleted.
# #region delete_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]
# @BRIEF Delete a dictionary entry.
# @PRE User has translate.dictionary.edit permission. Entry exists.
# @POST Entry is deleted.
# @SIDE_EFFECT Deletes DictionaryEntry record from DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.delete("/dictionaries/{dictionary_id}/entries/{entry_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_dictionary_entry(
dictionary_id: str,
@@ -309,13 +331,15 @@ async def delete_dictionary_entry(
return None
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:delete_dictionary_entry:Function]
# #endregion delete_dictionary_entry
# [DEF:import_dictionary_entries:Function]
# @PURPOSE: Import entries into a terminology dictionary from CSV/TSV content.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Entries are imported per conflict mode.
# #region import_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,import]
# @BRIEF Import entries into a terminology dictionary from CSV/TSV content.
# @PRE User has translate.dictionary.edit permission. Dictionary exists. Content is valid CSV/TSV.
# @POST Entries imported per conflict mode (overwrite/keep_existing/cancel). Import result returned.
# @SIDE_EFFECT Creates/updates DictionaryEntry records in DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.post("/dictionaries/{dictionary_id}/import")
async def import_dictionary_entries(
dictionary_id: str,
@@ -351,6 +375,6 @@ async def import_dictionary_entries(
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:import_dictionary_entries:Function]
# #endregion import_dictionary_entries
# [/DEF:TranslateDictionaryRoutesModule:Module]
# #endregion TranslateDictionaryRoutesModule

34
backend/src/api/routes/translate/_helpers.py
View File

@@ -1,17 +1,13 @@
# [DEF:TranslateHelpersModule:Module]
# @COMPLEXITY: 2
# @PURPOSE: Shared helper functions for translate route handlers.
# @LAYER: API
# @SEMANTICS: api, routes, translate, helpers
# @RELATION: DEPENDS_ON -> [models.translate]
# #region TranslateHelpersModule [C:2] [TYPE Module] [SEMANTICS api,routes,translate,helpers]
# @BRIEF Shared helper functions for translate route handlers.
# @LAYER UI
from typing import Any, Dict, List
from sqlalchemy.orm import Session
# [DEF:_run_to_response:Function]
# @COMPLEXITY: 2
# @PURPOSE: Convert TranslationRun ORM to response dict.
# #region _run_to_response [C:2] [TYPE Function] [SEMANTICS helpers,translate]
# @BRIEF Convert TranslationRun ORM to response dict.
def _run_to_response(run: Any) -> dict:
return {
"id": run.id,
@@ -34,12 +30,11 @@ def _run_to_response(run: Any) -> dict:
"created_by": run.created_by,
"created_at": run.created_at.isoformat() if run.created_at else None,
}
# [/DEF:_run_to_response:Function]
# #endregion _run_to_response
# [DEF:_dict_to_response:Function]
# @COMPLEXITY: 2
# @PURPOSE: Convert a TerminologyDictionary ORM model to a response dict with computed entry_count.
# #region _dict_to_response [C:2] [TYPE Function] [SEMANTICS helpers,translate]
# @BRIEF Convert a TerminologyDictionary ORM model to a response dict with computed entry_count.
@staticmethod
def _dict_to_response(d: Any, entry_count: int = 0) -> dict:
return {
@@ -54,15 +49,14 @@ def _dict_to_response(d: Any, entry_count: int = 0) -> dict:
"updated_at": d.updated_at,
"entry_count": entry_count,
}
# [/DEF:_dict_to_response:Function]
# #endregion _dict_to_response
# [DEF:_get_dictionary_entry_counts:Function]
# @COMPLEXITY: 2
# @PURPOSE: Get entry counts for a list of dictionary IDs.
# #region _get_dictionary_entry_counts [C:2] [TYPE Function] [SEMANTICS helpers,translate]
# @BRIEF Get entry counts for a list of dictionary IDs.
def _get_dictionary_entry_counts(db: Session, dict_ids: List[str]) -> Dict[str, int]:
from sqlalchemy import func
from ...models.translate import DictionaryEntry
from ....models.translate import DictionaryEntry
counts = (
db.query(DictionaryEntry.dictionary_id, func.count(DictionaryEntry.id))
.filter(DictionaryEntry.dictionary_id.in_(dict_ids))
@@ -70,6 +64,6 @@ def _get_dictionary_entry_counts(db: Session, dict_ids: List[str]) -> Dict[str,
.all()
)
return {row[0]: row[1] for row in counts}
# [/DEF:_get_dictionary_entry_counts:Function]
# #endregion _get_dictionary_entry_counts
# [/DEF:TranslateHelpersModule:Module]
# #endregion TranslateHelpersModule

112
backend/src/api/routes/translate/_job_routes.py
View File

@@ -1,8 +1,14 @@
# [DEF:TranslateJobRoutesModule:Module]
# @COMPLEXITY: 3
# @PURPOSE: Translation Job CRUD and datasource column routes.
# @LAYER: API
# @SEMANTICS: api, routes, translate, jobs
# #region TranslateJobRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,jobs]
# @BRIEF Translation Job CRUD and datasource column routes.
# @LAYER UI
# @RELATION DEPENDS_ON -> [TranslateJobService]
# @RELATION DEPENDS_ON -> [ConfigManager]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
# @PRE ConfigManager and DB session initialized. User authenticated with translate.job permissions.
# @POST Translation job CRUD completed or datasource columns fetched.
# @SIDE_EFFECT Reads/writes TranslationJob records to DB; fetches datasource metadata from Superset API.
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
@@ -33,10 +39,9 @@ from ._router import router
# Translation Job CRUD
# ============================================================
# [DEF:list_jobs:Function]
# @PURPOSE: List all translation jobs.
# @PRE: User has translate.job.view permission.
# @POST: Returns list of translation jobs.
# #region list_jobs [C:3] [TYPE Function] [SEMANTICS api,translate,jobs]
# @BRIEF List all translation jobs with pagination and optional status filter.
# @RELATION DEPENDS_ON -> [TranslateJobService]
@router.get("/jobs", response_model=List[TranslateJobResponse])
async def list_jobs(
page: int = Query(1, ge=1),
@@ -63,13 +68,12 @@ async def list_jobs(
return results
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:list_jobs:Function]
# #endregion list_jobs
# [DEF:get_job:Function]
# @PURPOSE: Get a single translation job by ID.
# @PRE: User has translate.job.view permission.
# @POST: Returns the translation job.
# #region get_job [C:3] [TYPE Function] [SEMANTICS api,translate,jobs]
# @BRIEF Get a single translation job by ID.
# @RELATION DEPENDS_ON -> [TranslateJobService]
@router.get("/jobs/{job_id}", response_model=TranslateJobResponse)
async def get_job(
job_id: str,
@@ -87,14 +91,15 @@ async def get_job(
return job_to_response(job, dict_ids)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_job:Function]
# #endregion get_job
# [DEF:create_job:Function]
# @PURPOSE: Create a new translation job.
# @PRE: User has translate.job.create permission.
# @POST: Returns the created translation job.
# @SIDE_EFFECT: Validates columns via SupersetClient; caches database_dialect.
# #region create_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]
# @BRIEF Create a new translation job.
# @PRE User has translate.job.create permission. Payload validated.
# @POST Translation job created and returned.
# @SIDE_EFFECT Creates TranslationJob record in DB; validates columns via SupersetClient; caches database_dialect.
# @RELATION DEPENDS_ON -> [TranslateJobService]
@router.post("/jobs", response_model=TranslateJobResponse, status_code=status.HTTP_201_CREATED)
async def create_job(
payload: TranslateJobCreate,
@@ -112,14 +117,15 @@ async def create_job(
return job_to_response(job, dict_ids)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))
# [/DEF:create_job:Function]
# #endregion create_job
# [DEF:update_job:Function]
# @PURPOSE: Update an existing translation job.
# @PRE: User has translate.job.edit permission.
# @POST: Returns the updated translation job.
# @SIDE_EFFECT: Re-detects database_dialect if datasource changed.
# #region update_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]
# @BRIEF Update an existing translation job.
# @PRE User has translate.job.edit permission. Job exists.
# @POST Translation job updated and returned.
# @SIDE_EFFECT Updates TranslationJob record in DB; re-detects database_dialect if datasource changed.
# @RELATION DEPENDS_ON -> [TranslateJobService]
@router.put("/jobs/{job_id}", response_model=TranslateJobResponse)
async def update_job(
job_id: str,
@@ -138,13 +144,15 @@ async def update_job(
return job_to_response(job, dict_ids)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:update_job:Function]
# #endregion update_job
# [DEF:delete_job:Function]
# @PURPOSE: Delete a translation job.
# @PRE: User has translate.job.delete permission.
# @POST: Job is deleted.
# #region delete_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]
# @BRIEF Delete a translation job.
# @PRE User has translate.job.delete permission. Job exists.
# @POST Translation job is deleted from DB.
# @SIDE_EFFECT Deletes TranslationJob and associated records from DB.
# @RELATION DEPENDS_ON -> [TranslateJobService]
@router.delete("/jobs/{job_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_job(
job_id: str,
@@ -160,13 +168,15 @@ async def delete_job(
service.delete_job(job_id)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:delete_job:Function]
# #endregion delete_job
# [DEF:duplicate_job:Function]
# @PURPOSE: Duplicate a translation job.
# @PRE: User has translate.job.create permission.
# @POST: Returns the duplicated job with status DRAFT.
# #region duplicate_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]
# @BRIEF Duplicate a translation job.
# @PRE User has translate.job.create permission. Source job exists.
# @POST Duplicated job created with status DRAFT and returned.
# @SIDE_EFFECT Creates a new TranslationJob record in DB as a copy of the source.
# @RELATION DEPENDS_ON -> [TranslateJobService]
@router.post("/jobs/{job_id}/duplicate", response_model=DuplicateJobResponse, status_code=status.HTTP_201_CREATED)
async def duplicate_job(
job_id: str,
@@ -187,17 +197,16 @@ async def duplicate_job(
)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:duplicate_job:Function]
# #endregion duplicate_job
# [DEF:get_datasource_columns:Function]
# @PURPOSE: Get column metadata and database dialect for a Superset datasource.
# @PRE: User has translate.job.view permission.
# @POST: Returns column list with metadata and database_dialect.
# @SIDE_EFFECT: Queries Superset API for dataset detail and database info.
# @RATIONALE: Dialect detection from Superset connection cached on TranslationJob at save time.
# @REJECTED: Hardcoding dialect list per database — would drift from actual Superset config.
# #region list_datasources [C:4] [TYPE Function] [SEMANTICS api,translate,datasources]
# @BRIEF List all Superset datasets available for translation jobs.
# @PRE User has translate.job.view permission.
# @POST Returns list of datasets with metadata.
# @SIDE_EFFECT Queries Superset API for available datasets.
# @RELATION DEPENDS_ON -> [TranslateJobService]
# @RELATION DEPENDS_ON -> [SupersetClient]
@router.get("/datasources")
async def list_datasources(
env_id: str = Query(..., description="Superset environment ID"),
@@ -216,7 +225,16 @@ async def list_datasources(
except Exception as e:
logger.error(f"[translate_routes][list_datasources] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e))
# #endregion list_datasources
# #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS api,translate,datasources]
# @BRIEF Get column metadata and database dialect for a Superset datasource.
# @PRE User has translate.job.view permission. Datasource exists in Superset.
# @POST Returns column list with metadata and database_dialect.
# @SIDE_EFFECT Queries Superset API for dataset detail and database info.
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [ConfigManager]
@router.get("/datasources/{datasource_id}/columns", response_model=DatasourceColumnsResponse)
async def get_datasource_columns(
datasource_id: int,
@@ -245,6 +263,6 @@ async def get_datasource_columns(
status_code=status.HTTP_502_BAD_GATEWAY,
detail=f"Failed to fetch datasource columns from Superset: {e}",
)
# [/DEF:get_datasource_columns:Function]
# #endregion get_datasource_columns
# [/DEF:TranslateJobRoutesModule:Module]
# #endregion TranslateJobRoutesModule

31
backend/src/api/routes/translate/_metrics_routes.py
View File

@@ -1,8 +1,9 @@
# [DEF:TranslateMetricsRoutesModule:Module]
# @COMPLEXITY: 2
# @PURPOSE: Translation Metrics endpoints.
# @LAYER: API
# @SEMANTICS: api, routes, translate, metrics
# #region TranslateMetricsRoutesModule [C:3] [TYPE Module] [SEMANTICS api,routes,translate,metrics]
# @BRIEF Translation Metrics endpoints providing aggregated stats on runs and corrections.
# @LAYER UI
# @RELATION DEPENDS_ON -> [TranslationMetrics]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
@@ -21,10 +22,9 @@ from ._router import router
# Metrics
# ============================================================
# [DEF:get_metrics:Function]
# @PURPOSE: Get translation metrics, optionally filtered by job.
# @PRE: User has translate.metrics.view permission.
# @POST: Returns metrics data.
# #region get_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics]
# @BRIEF Get translation metrics across all jobs, optionally filtered by job.
# @RELATION DEPENDS_ON -> [TranslationMetrics]
@router.get("/metrics")
async def get_metrics(
job_id: Optional[str] = Query(None),
@@ -41,13 +41,12 @@ async def get_metrics(
return metrics_svc.get_all_metrics()
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_metrics:Function]
# #endregion get_metrics
# [DEF:get_job_metrics:Function]
# @PURPOSE: Get aggregated metrics for a specific job.
# @PRE: User has translate.metrics.view permission.
# @POST: Returns metrics dict.
# #region get_job_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics]
# @BRIEF Get aggregated metrics for a specific translation job.
# @RELATION DEPENDS_ON -> [TranslationMetrics]
@router.get("/jobs/{job_id}/metrics")
async def get_job_metrics(
job_id: str,
@@ -62,6 +61,6 @@ async def get_job_metrics(
return metrics_svc.get_job_metrics(job_id)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_job_metrics:Function]
# #endregion get_job_metrics
# [/DEF:TranslateMetricsRoutesModule:Module]
# #endregion TranslateMetricsRoutesModule

81
backend/src/api/routes/translate/_preview_routes.py
View File

@@ -1,8 +1,13 @@
# [DEF:TranslatePreviewRoutesModule:Module]
# @COMPLEXITY: 3
# @PURPOSE: Translation Preview session management routes.
# @LAYER: API
# @SEMANTICS: api, routes, translate, preview
# #region TranslatePreviewRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,preview]
# @BRIEF Translation Preview session management routes.
# @LAYER UI
# @RELATION DEPENDS_ON -> [TranslationPreview]
# @RELATION DEPENDS_ON -> [ConfigManager]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
# @PRE ConfigManager and DB session initialized. User authenticated with translate permissions.
# @POST Preview sessions created, rows reviewed/accepted, records queried.
# @SIDE_EFFECT Creates/updates preview sessions and rows in DB; fetches sample data from Superset; calls LLM provider.
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
@@ -28,11 +33,13 @@ from ._router import router
# Preview
# ============================================================
# [DEF:preview_translation:Function]
# @PURPOSE: Preview a translation before applying it.
# @PRE: User has translate.job.execute permission.
# @POST: Returns a preview session with records and cost estimation.
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows.
# #region preview_translation [C:4] [TYPE Function] [SEMANTICS api,translate,preview]
# @BRIEF Preview a translation before applying it.
# @PRE User has translate.job.execute permission. Job exists.
# @POST Preview session created with sample rows and cost estimation.
# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates PreviewSession and PreviewRecord rows in DB.
# @RELATION DEPENDS_ON -> [TranslationPreview]
# @RELATION DEPENDS_ON -> [SupersetClient]
@router.post("/jobs/{job_id}/preview", status_code=status.HTTP_201_CREATED)
async def preview_translation(
job_id: str,
@@ -60,13 +67,15 @@ async def preview_translation(
except Exception as e:
logger.error(f"[translate_routes][preview_translation] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Preview failed: {e}")
# [/DEF:preview_translation:Function]
# #endregion preview_translation
# [DEF:update_preview_row:Function]
# @PURPOSE: Approve, edit, or reject a preview row.
# @PRE: User has translate.job.execute permission.
# @POST: Preview row status is updated.
# #region update_preview_row [C:4] [TYPE Function] [SEMANTICS api,translate,preview]
# @BRIEF Approve, edit, or reject a preview row.
# @PRE User has translate.job.execute permission. Preview row exists.
# @POST Preview row status updated to APPROVED/REJECTED/EDITED.
# @SIDE_EFFECT Updates PreviewRecord status in DB.
# @RELATION DEPENDS_ON -> [TranslationPreview]
@router.put("/jobs/{job_id}/preview/rows/{row_key}")
async def update_preview_row(
job_id: str,
@@ -91,13 +100,15 @@ async def update_preview_row(
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:update_preview_row:Function]
# #endregion update_preview_row
# [DEF:accept_preview_session:Function]
# @PURPOSE: Accept a preview session, marking it as the quality gate for full execution.
# @PRE: User has translate.job.execute permission. Job has an ACTIVE preview session.
# @POST: Preview session is marked as APPLIED; full execution can proceed.
# #region accept_preview_session [C:4] [TYPE Function] [SEMANTICS api,translate,preview]
# @BRIEF Accept a preview session, marking it as the quality gate for full execution.
# @PRE User has translate.job.execute permission. Job has an ACTIVE preview session.
# @POST Preview session status set to APPLIED. Full execution can proceed.
# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB.
# @RELATION DEPENDS_ON -> [TranslationPreview]
@router.post("/jobs/{job_id}/preview/accept")
async def accept_preview_session(
job_id: str,
@@ -114,13 +125,15 @@ async def accept_preview_session(
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:accept_preview_session:Function]
# #endregion accept_preview_session
# [DEF:apply_preview:Function]
# @PURPOSE: Apply a preview session (alias for accept when accepting at session level).
# @PRE: User has translate.job.execute permission.
# @POST: Preview is applied.
# #region apply_preview [C:4] [TYPE Function] [SEMANTICS api,translate,preview]
# @BRIEF Apply a preview session (alias for accept when accepting at session level).
# @PRE User has translate.job.execute permission. Session exists.
# @POST Preview session applied and marked as quality gate.
# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB.
# @RELATION DEPENDS_ON -> [TranslationPreview]
@router.post("/preview/{session_id}/apply")
async def apply_preview(
session_id: str,
@@ -132,7 +145,7 @@ async def apply_preview(
"""Apply a preview session by session ID."""
logger.info(f"[translate_routes][apply_preview] Session: {session_id}, User: {current_user.username}")
# Find job_id from session
from ...models.translate import TranslationPreviewSession
from ....models.translate import TranslationPreviewSession
session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first()
if not session:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found")
@@ -142,17 +155,17 @@ async def apply_preview(
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:apply_preview:Function]
# #endregion apply_preview
# ============================================================
# Preview Records
# ============================================================
# [DEF:get_preview_records:Function]
# @PURPOSE: Get records for a preview session.
# @PRE: User has translate.job.view permission.
# @POST: Returns list of preview records.
# #region get_preview_records [C:3] [TYPE Function] [SEMANTICS api,translate,preview]
# @BRIEF Get records for a preview session.
# @RELATION DEPENDS_ON -> [TranslationPreviewSession]
# @RELATION DEPENDS_ON -> [TranslationPreviewRecord]
@router.get("/preview/{session_id}/records", response_model=List[PreviewRow])
async def get_preview_records(
session_id: str,
@@ -163,7 +176,7 @@ async def get_preview_records(
"""Get records for a preview session."""
logger.info(f"[translate_routes][get_preview_records] Session: {session_id}, User: {current_user.username}")
try:
from ...models.translate import TranslationPreviewSession, TranslationPreviewRecord
from ....models.translate import TranslationPreviewSession, TranslationPreviewRecord
session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first()
if not session:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found")
@@ -190,6 +203,6 @@ async def get_preview_records(
except Exception as e:
logger.error(f"[translate_routes][get_preview_records] Error: {e}")
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:get_preview_records:Function]
# #endregion get_preview_records
# [/DEF:TranslatePreviewRoutesModule:Module]
# #endregion TranslatePreviewRoutesModule

14
backend/src/api/routes/translate/_router.py
View File

@@ -1,14 +1,10 @@
# [DEF:TranslateRouterModule:Module]
# @COMPLEXITY: 1
# @PURPOSE: APIRouter instance for translate routes.
# @LAYER: API
# @SEMANTICS: api, routes, translate, router
# #region TranslateRouterModule [C:1] [TYPE Module] [SEMANTICS api,routes,translate,router]
# @LAYER UI
from fastapi import APIRouter
# [DEF:translate_router:Global]
# @PURPOSE: APIRouter instance for all translate sub-routes.
# #region translate_router [C:1] [TYPE Global]
router = APIRouter(prefix="/api/translate", tags=["translate"])
# [/DEF:translate_router:Global]
# #endregion translate_router
# [/DEF:TranslateRouterModule:Module]
# #endregion TranslateRouterModule

51
backend/src/api/routes/translate/_run_list_routes.py
View File

@@ -1,8 +1,10 @@
# [DEF:TranslateRunListRoutesModule:Module]
# @COMPLEXITY: 3
# @PURPOSE: Translation Run listing, detail, and CSV download routes (cross-job).
# @LAYER: API
# @SEMANTICS: api, routes, translate, runs, list, detail
# #region TranslateRunListRoutesModule [C:3] [TYPE Module] [SEMANTICS api,routes,translate,runs,list,detail]
# @BRIEF Translation Run listing, detail, and CSV download routes (cross-job).
# @LAYER UI
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
# @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
@@ -23,14 +25,13 @@ from ._router import router
# List Runs (cross-job)
# ============================================================
# [DEF:list_runs:Function]
# @PURPOSE: List all runs with cross-job filtering and pagination.
# @PRE: User has translate.history.view permission.
# @POST: Returns paginated list of runs.
# #region list_runs [C:3] [TYPE Function] [SEMANTICS api,translate,runs,list]
# @BRIEF List all runs with cross-job filtering and pagination.
# @RELATION DEPENDS_ON -> [TranslationRun]
@router.get("/runs")
async def list_runs(
job_id: Optional[str] = Query(None),
run_status: Optional[str] = Query(None),
run_status: Optional[str] = Query(None, alias="status"),
trigger_type: Optional[str] = Query(None),
created_by: Optional[str] = Query(None),
date_from: Optional[str] = Query(None),
@@ -44,7 +45,7 @@ async def list_runs(
"""List all runs with cross-job filtering and pagination."""
logger.info(f"[translate_routes][list_runs] User: {current_user.username}")
try:
from ...models.translate import TranslationRun
from ....models.translate import TranslationRun
query = db.query(TranslationRun)
if job_id:
@@ -104,17 +105,18 @@ async def list_runs(
except Exception as e:
logger.error(f"[translate_routes][list_runs] Error: {e}")
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:list_runs:Function]
# #endregion list_runs
# ============================================================
# Run Detail
# ============================================================
# [DEF:get_run_detail:Function]
# @PURPOSE: Get detailed run info with config_snapshot, records, events.
# @PRE: User has translate.history.view permission.
# @POST: Returns run detail with records and events.
# #region get_run_detail [C:3] [TYPE Function] [SEMANTICS api,translate,runs,detail]
# @BRIEF Get detailed run info with config_snapshot, records, events.
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
# @RELATION DEPENDS_ON -> [TranslationEventLog]
# @RELATION DEPENDS_ON -> [TranslationRun]
@router.get("/runs/{run_id}/detail")
async def get_run_detail(
run_id: str,
@@ -133,7 +135,7 @@ async def get_run_detail(
event_summary = TranslationEventLog(db).get_run_event_summary(run_id)
# Get config snapshot from run
from ...models.translate import TranslationRun
from ....models.translate import TranslationRun
run = db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
config_snapshot = run.config_snapshot if run else None
@@ -147,17 +149,16 @@ async def get_run_detail(
}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_run_detail:Function]
# #endregion get_run_detail
# ============================================================
# Download Skipped CSV
# ============================================================
# [DEF:download_skipped_csv:Function]
# @PURPOSE: Download a CSV of skipped translation records from a run.
# @PRE: User has translate.job.view permission.
# @POST: Returns CSV file of skipped records.
# #region download_skipped_csv [C:3] [TYPE Function] [SEMANTICS api,translate,runs,csv]
# @BRIEF Download a CSV of skipped translation records from a run.
# @RELATION DEPENDS_ON -> [TranslationRecord]
@router.get("/runs/{run_id}/skipped.csv")
async def download_skipped_csv(
run_id: str,
@@ -168,7 +169,7 @@ async def download_skipped_csv(
"""Download skipped translation records as CSV."""
logger.info(f"[translate_routes][download_skipped_csv] Run: {run_id}, User: {current_user.username}")
try:
from ...models.translate import TranslationRecord
from ....models.translate import TranslationRecord
from fastapi.responses import StreamingResponse
import csv
import io
@@ -201,6 +202,6 @@ async def download_skipped_csv(
except Exception as e:
logger.error(f"[translate_routes][download_skipped_csv] Error: {e}")
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:download_skipped_csv:Function]
# #endregion download_skipped_csv
# [/DEF:TranslateRunListRoutesModule:Module]
# #endregion TranslateRunListRoutesModule

139
backend/src/api/routes/translate/_run_routes.py
View File

@@ -1,20 +1,26 @@
# [DEF:TranslateRunRoutesModule:Module]
# @COMPLEXITY: 3
# @PURPOSE: Translation Run execution, history, status, records and batches routes.
# @LAYER: API
# @SEMANTICS: api, routes, translate, runs
# #region TranslateRunRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,runs]
# @BRIEF Translation Run execution, history, status, records and batches routes.
# @LAYER UI
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
# @RELATION DEPENDS_ON -> [ConfigManager]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
# @PRE ConfigManager and DB session initialized. User authenticated with translate permissions.
# @POST Translation run executed, manipulated, or queried. Results returned to caller.
# @SIDE_EFFECT Creates/updates TranslationRun records in DB; spawns background threads for execution; queries Superset API.
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from ....core.database import get_db
from ....core.database import get_db, SessionLocal
from ....core.logger import logger, belief_scope
from ....schemas.auth import User
from ....dependencies import get_current_user, has_permission, get_config_manager
from ....core.config_manager import ConfigManager
from ....plugins.translate.orchestrator import TranslationOrchestrator
from ....plugins.translate.events import TranslationEventLog
from ....models.translate import TranslationRun
from ._router import router
from ._helpers import _run_to_response
@@ -24,43 +30,64 @@ from ._helpers import _run_to_response
# Translation Run / Execute
# ============================================================
# [DEF:run_translation:Function]
# @PURPOSE: Execute a translation job (trigger a run).
# @PRE: User has translate.job.execute permission.
# @POST: Returns the created translation run.
# #region run_translation [C:4] [TYPE Function] [SEMANTICS api,translate,runs]
# @BRIEF Execute a translation job (trigger a run).
# @PRE User has translate.job.execute permission. Job exists and preview is accepted.
# @POST Translation run created and started in background. Run object returned.
# @SIDE_EFFECT Creates TranslationRun record in DB; spawns background thread for translation execution.
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
@router.post("/jobs/{job_id}/run", status_code=status.HTTP_201_CREATED)
async def run_translation(
job_id: str,
full_translation: bool = Query(False, description="If True, fetch ALL rows from Superset dataset instead of preview-only rows"),
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Execute a translation job (trigger a run)."""
logger.info(f"[translate_routes][run_translation] Job: {job_id}, User: {current_user.username}")
"""Execute a translation job (trigger a run).
By default runs translation on preview-approved rows only.
Set full_translation=true to fetch ALL rows from the Superset source dataset.
"""
logger.info(f"[translate_routes][run_translation] Job: {job_id}, User: {current_user.username}, full={full_translation}")
try:
orch = TranslationOrchestrator(db, config_manager, current_user.username)
run = orch.start_run(job_id=job_id, is_scheduled=False)
# Execute asynchronously in background
# Execute asynchronously in background with a FRESH session.
# The request-scoped session from Depends(get_db) is closed after the handler returns,
# so the background thread must create its own session to avoid "This transaction is closed".
import threading
threading.Thread(
target=orch.execute_run,
args=(run,),
daemon=True,
).start()
def _execute_background():
thread_db = SessionLocal()
try:
thread_orch = TranslationOrchestrator(thread_db, config_manager, current_user.username)
# Reload run from DB with the new session (the passed run object is detached)
thread_run = thread_db.query(TranslationRun).filter(TranslationRun.id == run.id).first()
if thread_run:
thread_orch.execute_run(thread_run, full_translation=full_translation)
except Exception as e:
logger.error(f"[translate_routes][run_translation] Background execution error: {e}")
finally:
thread_db.close()
threading.Thread(target=_execute_background, daemon=True).start()
return _run_to_response(run)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
except Exception as e:
logger.error(f"[translate_routes][run_translation] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Run failed: {e}")
# [/DEF:run_translation:Function]
# #endregion run_translation
# [DEF:retry_run:Function]
# @PURPOSE: Retry failed batches in a translation run.
# @PRE: User has translate.job.execute permission.
# @POST: Returns the updated translation run.
# #region retry_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs]
# @BRIEF Retry failed batches in a translation run.
# @PRE User has translate.job.execute permission. Run exists and has failed batches.
# @POST Failed batches re-executed. Updated run returned.
# @SIDE_EFFECT Updates TranslationRun and TranslationBatch records in DB.
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
@router.post("/runs/{run_id}/retry")
async def retry_run(
run_id: str,
@@ -80,13 +107,15 @@ async def retry_run(
except Exception as e:
logger.error(f"[translate_routes][retry_run] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry failed: {e}")
# [/DEF:retry_run:Function]
# #endregion retry_run
# [DEF:retry_insert:Function]
# @PURPOSE: Retry the SQL insert phase for a completed run.
# @PRE: User has translate.job.execute permission.
# @POST: Returns the updated run.
# #region retry_insert [C:4] [TYPE Function] [SEMANTICS api,translate,runs]
# @BRIEF Retry the SQL insert phase for a completed run.
# @PRE User has translate.job.execute permission. Run completed with insert_status FAILED.
# @POST SQL insert phase re-executed. Updated run returned.
# @SIDE_EFFECT Re-executes SQL insert into Superset; updates TranslationRun insert_status.
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
@router.post("/runs/{run_id}/retry-insert")
async def retry_insert(
run_id: str,
@@ -106,13 +135,15 @@ async def retry_insert(
except Exception as e:
logger.error(f"[translate_routes][retry_insert] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry insert failed: {e}")
# [/DEF:retry_insert:Function]
# #endregion retry_insert
# [DEF:cancel_run:Function]
# @PURPOSE: Cancel a running translation.
# @PRE: User has translate.job.execute permission.
# @POST: Run is cancelled.
# #region cancel_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs]
# @BRIEF Cancel a running translation.
# @PRE User has translate.job.execute permission. Run is in RUNNING state.
# @POST Run is cancelled. Updated run returned.
# @SIDE_EFFECT Updates TranslationRun status to CANCELLED in DB.
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
@router.post("/runs/{run_id}/cancel")
async def cancel_run(
run_id: str,
@@ -129,13 +160,12 @@ async def cancel_run(
return _run_to_response(run)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:cancel_run:Function]
# #endregion cancel_run
# [DEF:get_run_history:Function]
# @PURPOSE: Get run history for a translation job.
# @PRE: User has translate.history.view permission.
# @POST: Returns list of runs.
# #region get_run_history [C:3] [TYPE Function] [SEMANTICS api,translate,runs]
# @BRIEF Get run history for a translation job.
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
@router.get("/jobs/{job_id}/runs")
async def get_run_history(
job_id: str,
@@ -154,13 +184,12 @@ async def get_run_history(
return {"items": runs, "total": total, "page": page, "page_size": page_size}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_run_history:Function]
# #endregion get_run_history
# [DEF:get_run_status:Function]
# @PURPOSE: Get status and statistics for a translation run.
# @PRE: User has translate.history.view permission.
# @POST: Returns run details with statistics.
# #region get_run_status [C:3] [TYPE Function] [SEMANTICS api,translate,runs]
# @BRIEF Get status and statistics for a translation run.
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
@router.get("/runs/{run_id}")
async def get_run_status(
run_id: str,
@@ -176,13 +205,12 @@ async def get_run_status(
return orch.get_run_status(run_id)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_run_status:Function]
# #endregion get_run_status
# [DEF:get_run_records:Function]
# @PURPOSE: Get paginated records for a translation run.
# @PRE: User has translate.history.view permission.
# @POST: Returns paginated records.
# #region get_run_records [C:3] [TYPE Function] [SEMANTICS api,translate,runs]
# @BRIEF Get paginated records for a translation run.
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
@router.get("/runs/{run_id}/records")
async def get_run_records(
run_id: str,
@@ -201,17 +229,16 @@ async def get_run_records(
return orch.get_run_records(run_id, page=page, page_size=page_size, status_filter=status)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_run_records:Function]
# #endregion get_run_records
# ============================================================
# Batches
# ============================================================
# [DEF:get_batches:Function]
# @PURPOSE: Get batches for a translation run.
# @PRE: User has translate.job.view permission.
# @POST: Returns list of batches.
# #region get_batches [C:3] [TYPE Function] [SEMANTICS api,translate,runs]
# @BRIEF Get batches for a translation run.
# @RELATION DEPENDS_ON -> [TranslationBatch]
@router.get("/runs/{run_id}/batches")
async def get_batches(
run_id: str,
@@ -222,7 +249,7 @@ async def get_batches(
"""Get batches for a translation run."""
logger.info(f"[translate_routes][get_batches] Run: {run_id}, User: {current_user.username}")
try:
from ...models.translate import TranslationBatch
from ....models.translate import TranslationBatch
batches = (
db.query(TranslationBatch)
.filter(TranslationBatch.run_id == run_id)
@@ -247,6 +274,6 @@ async def get_batches(
except Exception as e:
logger.error(f"[translate_routes][get_batches] Error: {e}")
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:get_batches:Function]
# #endregion get_batches
# [/DEF:TranslateRunRoutesModule:Module]
# #endregion TranslateRunRoutesModule

93
backend/src/api/routes/translate/_schedule_routes.py
View File

@@ -1,8 +1,13 @@
# [DEF:TranslateScheduleRoutesModule:Module]
# @COMPLEXITY: 3
# @PURPOSE: Translation Schedule management routes.
# @LAYER: API
# @SEMANTICS: api, routes, translate, schedule
# #region TranslateScheduleRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,schedule]
# @BRIEF Translation Schedule management routes.
# @LAYER UI
# @RELATION DEPENDS_ON -> [TranslationScheduler]
# @RELATION DEPENDS_ON -> [ConfigManager]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
# @PRE ConfigManager and DB session initialized. User authenticated with translate.schedule permissions.
# @POST Schedule CRUD operations completed. APScheduler jobs registered/unregistered.
# @SIDE_EFFECT Creates/updates/deletes TranslationSchedule records in DB; registers/removes jobs in APScheduler.
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
@@ -23,10 +28,9 @@ from ._router import router
# Schedule
# ============================================================
# [DEF:get_schedule:Function]
# @PURPOSE: Get the schedule for a translation job.
# @PRE: User has translate.schedule.view permission.
# @POST: Returns the schedule configuration.
# #region get_schedule [C:3] [TYPE Function] [SEMANTICS api,translate,schedule]
# @BRIEF Get the schedule configuration for a translation job.
# @RELATION DEPENDS_ON -> [TranslationScheduler]
@router.get("/jobs/{job_id}/schedule")
async def get_schedule(
job_id: str,
@@ -54,13 +58,15 @@ async def get_schedule(
}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_schedule:Function]
# #endregion get_schedule
# [DEF:set_schedule:Function]
# @PURPOSE: Set or update the schedule for a translation job.
# @PRE: User has translate.schedule.manage permission.
# @POST: Schedule is created or updated.
# #region set_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]
# @BRIEF Set or update the schedule for a translation job.
# @PRE User has translate.schedule.manage permission. Job exists.
# @POST Schedule created or updated. APScheduler job registered.
# @SIDE_EFFECT Creates/updates TranslationSchedule record in DB; registers job in APScheduler.
# @RELATION DEPENDS_ON -> [TranslationScheduler]
@router.put("/jobs/{job_id}/schedule")
async def set_schedule(
job_id: str,
@@ -75,7 +81,7 @@ async def set_schedule(
try:
scheduler = TranslationScheduler(db, config_manager, current_user.username)
# Check if schedule already exists
from ...models.translate import TranslationSchedule
from ....models.translate import TranslationSchedule
existing = db.query(TranslationSchedule).filter(
TranslationSchedule.job_id == job_id
).first()
@@ -94,7 +100,7 @@ async def set_schedule(
is_active=payload.is_active,
)
# Register with APScheduler via SchedulerService
from ...dependencies import get_scheduler_service
from ....dependencies import get_scheduler_service
sched_svc = get_scheduler_service()
sched_svc.add_translation_job(
schedule_id=schedule.id,
@@ -115,13 +121,15 @@ async def set_schedule(
}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:set_schedule:Function]
# #endregion set_schedule
# [DEF:enable_schedule:Function]
# @PURPOSE: Enable a schedule for a translation job.
# @PRE: User has translate.schedule.manage permission.
# @POST: Schedule is enabled.
# #region enable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]
# @BRIEF Enable a schedule for a translation job.
# @PRE User has translate.schedule.manage permission. Schedule exists.
# @POST Schedule is enabled. APScheduler job registered.
# @SIDE_EFFECT Updates TranslationSchedule is_active to True in DB; registers job in APScheduler.
# @RELATION DEPENDS_ON -> [TranslationScheduler]
@router.post("/jobs/{job_id}/schedule/enable")
async def enable_schedule(
job_id: str,
@@ -135,7 +143,7 @@ async def enable_schedule(
try:
scheduler = TranslationScheduler(db, config_manager, current_user.username)
schedule = scheduler.set_schedule_active(job_id, is_active=True)
from ...dependencies import get_scheduler_service
from ....dependencies import get_scheduler_service
sched_svc = get_scheduler_service()
sched_svc.add_translation_job(
schedule_id=schedule.id,
@@ -146,13 +154,15 @@ async def enable_schedule(
return {"status": "enabled", "job_id": job_id}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:enable_schedule:Function]
# #endregion enable_schedule
# [DEF:disable_schedule:Function]
# @PURPOSE: Disable a schedule for a translation job.
# @PRE: User has translate.schedule.manage permission.
# @POST: Schedule is disabled.
# #region disable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]
# @BRIEF Disable a schedule for a translation job.
# @PRE User has translate.schedule.manage permission. Schedule exists.
# @POST Schedule is disabled. APScheduler job removed.
# @SIDE_EFFECT Updates TranslationSchedule is_active to False in DB; removes job from APScheduler.
# @RELATION DEPENDS_ON -> [TranslationScheduler]
@router.post("/jobs/{job_id}/schedule/disable")
async def disable_schedule(
job_id: str,
@@ -166,19 +176,21 @@ async def disable_schedule(
try:
scheduler = TranslationScheduler(db, config_manager, current_user.username)
scheduler.set_schedule_active(job_id, is_active=False)
from ...dependencies import get_scheduler_service
from ....dependencies import get_scheduler_service
sched_svc = get_scheduler_service()
sched_svc.remove_translation_job(schedule_id=job_id)
return {"status": "disabled", "job_id": job_id}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:disable_schedule:Function]
# #endregion disable_schedule
# [DEF:delete_schedule:Function]
# @PURPOSE: Delete the schedule for a translation job.
# @PRE: User has translate.schedule.manage permission.
# @POST: Schedule is removed.
# #region delete_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]
# @BRIEF Delete the schedule for a translation job.
# @PRE User has translate.schedule.manage permission. Schedule exists.
# @POST Schedule is removed. APScheduler job removed.
# @SIDE_EFFECT Deletes TranslationSchedule record from DB; removes job from APScheduler.
# @RELATION DEPENDS_ON -> [TranslationScheduler]
@router.delete("/jobs/{job_id}/schedule", status_code=status.HTTP_204_NO_CONTENT)
async def delete_schedule(
job_id: str,
@@ -192,19 +204,18 @@ async def delete_schedule(
try:
scheduler = TranslationScheduler(db, config_manager, current_user.username)
scheduler.delete_schedule(job_id)
from ...dependencies import get_scheduler_service
from ....dependencies import get_scheduler_service
sched_svc = get_scheduler_service()
sched_svc.remove_translation_job(schedule_id=job_id)
return None
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:delete_schedule:Function]
# #endregion delete_schedule
# [DEF:get_next_executions:Function]
# @PURPOSE: Preview next N executions for a job's schedule.
# @PRE: User has translate.schedule.view permission.
# @POST: Returns next execution times.
# #region get_next_executions [C:3] [TYPE Function] [SEMANTICS api,translate,schedule]
# @BRIEF Preview next N execution times for a job's schedule.
# @RELATION DEPENDS_ON -> [TranslationScheduler]
@router.get("/jobs/{job_id}/schedule/next-executions")
async def get_next_executions(
job_id: str,
@@ -230,6 +241,6 @@ async def get_next_executions(
}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_next_executions:Function]
# #endregion get_next_executions
# [/DEF:TranslateScheduleRoutesModule:Module]
# #endregion TranslateScheduleRoutesModule

5
backend/src/models/__init__.py
View File

@@ -1,3 +1,2 @@
# [DEF:ModelsPackage:Package]
# @PURPOSE: Domain model package root.
# [/DEF:ModelsPackage:Package]
# #region ModelsPackage [C:1] [TYPE Package]
# #endregion ModelsPackage

78
backend/src/models/translate.py
View File

@@ -1,9 +1,6 @@
# [DEF:TranslateModels:Module]
# @COMPLEXITY: 3
# @SEMANTICS: translate, models, sqlalchemy, persistence
# @PURPOSE: SQLAlchemy ORM models for LLM-based SQL/dashboard translation across dialects.
# @LAYER: Domain
# @RELATION: INHERITS_FROM -> MappingModels:Base
# #region TranslateModels [C:2] [TYPE Module]
# @BRIEF SQLAlchemy ORM models for LLM-based SQL/dashboard translation across dialects.
# @LAYER Domain
from sqlalchemy import (
Column, String, Boolean, DateTime, JSON, Text,
@@ -16,14 +13,13 @@ import hashlib
from .mapping import Base
# #region generate_uuid [C:1] [TYPE Function]
def generate_uuid():
return str(uuid.uuid4())
# #endregion generate_uuid
# [DEF:TranslationJob:Class]
# @PURPOSE: A translation job representing a multi-dialect conversion task with column mappings, LLM config, and dictionary attachments.
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity.
# #region TranslationJob [C:1] [TYPE Class]
class TranslationJob(Base):
__tablename__ = "translation_jobs"
@@ -55,15 +51,15 @@ class TranslationJob(Base):
# Environment association
environment_id = Column(String, nullable=True, comment="Superset environment ID for datasource access")
target_database_id = Column(String, nullable=True, comment="Superset database ID for INSERT via SQL Lab")
created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
# [/DEF:TranslationJob:Class]
# #endregion TranslationJob
# [DEF:TranslationRun:Class]
# @PURPOSE: Represents a single execution of a translation job, capturing status, timing, and Superset execution metadata.
# #region TranslationRun [C:1] [TYPE Class]
class TranslationRun(Base):
__tablename__ = "translation_runs"
@@ -87,11 +83,10 @@ class TranslationRun(Base):
dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at run time")
created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
# [/DEF:TranslationRun:Class]
# #endregion TranslationRun
# [DEF:TranslationBatch:Class]
# @PURPOSE: Groups translation records within a run into manageable batches with timing and record counts.
# #region TranslationBatch [C:1] [TYPE Class]
class TranslationBatch(Base):
__tablename__ = "translation_batches"
@@ -105,11 +100,10 @@ class TranslationBatch(Base):
started_at = Column(DateTime, nullable=True)
completed_at = Column(DateTime, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
# [/DEF:TranslationBatch:Class]
# #endregion TranslationBatch
# [DEF:TranslationRecord:Class]
# @PURPOSE: Individual translation result for a single SQL statement or dashboard element, tracking source, target, and error state.
# #region TranslationRecord [C:1] [TYPE Class]
class TranslationRecord(Base):
__tablename__ = "translation_records"
@@ -126,16 +120,16 @@ class TranslationRecord(Base):
token_count_input = Column(Integer, nullable=True)
token_count_output = Column(Integer, nullable=True)
translation_duration_ms = Column(Integer, nullable=True)
source_data = Column(JSON, nullable=True, comment="Snapshot of source row key/context column values for INSERT phase")
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
__table_args__ = (
Index("ix_translation_records_run_status", "run_id", "status"),
)
# [/DEF:TranslationRecord:Class]
# #endregion TranslationRecord
# [DEF:TranslationEvent:Class]
# @PURPOSE: Audit/event log for translation operations, with optional run_id for context.
# #region TranslationEvent [C:1] [TYPE Class]
class TranslationEvent(Base):
__tablename__ = "translation_events"
@@ -146,11 +140,10 @@ class TranslationEvent(Base):
event_data = Column(JSON, nullable=True)
created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
# [/DEF:TranslationEvent:Class]
# #endregion TranslationEvent
# [DEF:TranslationPreviewSession:Class]
# @PURPOSE: A preview session allowing users to review proposed translations before applying them.
# #region TranslationPreviewSession [C:1] [TYPE Class]
class TranslationPreviewSession(Base):
__tablename__ = "translation_preview_sessions"
@@ -161,11 +154,10 @@ class TranslationPreviewSession(Base):
created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
expires_at = Column(DateTime, nullable=True)
# [/DEF:TranslationPreviewSession:Class]
# #endregion TranslationPreviewSession
# [DEF:TranslationPreviewRecord:Class]
# @PURPOSE: Individual preview entry within a preview session, showing original and translated content side by side.
# #region TranslationPreviewRecord [C:1] [TYPE Class]
class TranslationPreviewRecord(Base):
__tablename__ = "translation_preview_records"
@@ -178,12 +170,12 @@ class TranslationPreviewRecord(Base):
source_object_name = Column(String, nullable=True)
status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED
feedback = Column(Text, nullable=True)
source_data = Column(JSON, nullable=True, comment="Snapshot of source row values for key/context columns")
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
# [/DEF:TranslationPreviewRecord:Class]
# #endregion TranslationPreviewRecord
# [DEF:TerminologyDictionary:Class]
# @PURPOSE: A named collection of terminology mappings used during translation to ensure consistent term translation.
# #region TerminologyDictionary [C:1] [TYPE Class]
class TerminologyDictionary(Base):
__tablename__ = "terminology_dictionaries"
@@ -196,11 +188,10 @@ class TerminologyDictionary(Base):
created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
# [/DEF:TerminologyDictionary:Class]
# #endregion TerminologyDictionary
# [DEF:DictionaryEntry:Class]
# @PURPOSE: A single term mapping entry with case-insensitive unique constraint on source_term_normalized and origin tracking.
# #region DictionaryEntry [C:1] [TYPE Class]
class DictionaryEntry(Base):
__tablename__ = "dictionary_entries"
@@ -222,11 +213,10 @@ class DictionaryEntry(Base):
name="uq_dictionary_entry_term"
),
)
# [/DEF:DictionaryEntry:Class]
# #endregion DictionaryEntry
# [DEF:TranslationSchedule:Class]
# @PURPOSE: Defines a cron-based schedule for recurring translation jobs.
# #region TranslationSchedule [C:1] [TYPE Class]
class TranslationSchedule(Base):
__tablename__ = "translation_schedules"
@@ -240,11 +230,10 @@ class TranslationSchedule(Base):
created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
# [/DEF:TranslationSchedule:Class]
# #endregion TranslationSchedule
# [DEF:TranslationJobDictionary:Class]
# @PURPOSE: Many-to-many association between translation jobs and terminology dictionaries.
# #region TranslationJobDictionary [C:1] [TYPE Class]
class TranslationJobDictionary(Base):
__tablename__ = "translation_job_dictionaries"
@@ -259,11 +248,10 @@ class TranslationJobDictionary(Base):
name="uq_job_dictionary"
),
)
# [/DEF:TranslationJobDictionary:Class]
# #endregion TranslationJobDictionary
# [DEF:MetricSnapshot:Class]
# @PURPOSE: Captures translation performance metrics at a point in time with config/content/dictionary hash keys for aggregation.
# #region MetricSnapshot [C:1] [TYPE Class]
class MetricSnapshot(Base):
__tablename__ = "translation_metric_snapshots"
@@ -290,6 +278,6 @@ class MetricSnapshot(Base):
__table_args__ = (
Index("ix_metric_snapshots_job_date", "job_id", "snapshot_date"),
)
# [/DEF:MetricSnapshot:Class]
# #endregion MetricSnapshot
# [/DEF:TranslateModels:Module]
# #endregion TranslateModels

5
backend/src/plugins/translate/__init__.py
View File

@@ -1,3 +1,2 @@
# [DEF:TranslatePluginPackage:Package]
# @PURPOSE: Translation plugin package for LLM-based cross-Superset SQL/dashboard translation.
# [/DEF:TranslatePluginPackage:Package]
# #region TranslatePluginPackage [C:1] [TYPE Package]
# #endregion TranslatePluginPackage

2
backend/src/plugins/translate/__tests__/test_orchestrator.py
View File

@@ -41,6 +41,7 @@ def mock_job() -> MagicMock:
job.source_datasource_id = "42"
job.translation_column = "name"
job.context_columns = ["category"]
job.source_table = "source_table_name"
job.target_schema = "public"
job.target_table = "translated_data"
job.target_key_cols = ["id"]
@@ -152,7 +153,6 @@ class TestTranslationOrchestrator:
# @PURPOSE: start_run creates a PENDING run with event.
def test_start_run_success(
self,
mock_service,
mock_job: MagicMock,
mock_preview_session: MagicMock,
) -> None:

22
backend/src/plugins/translate/_utils.py
View File

@@ -1,30 +1,22 @@
# [DEF:DictionaryUtils:Module]
# @COMPLEXITY: 1
# @SEMANTICS: dictionary, utils, normalize, delimiter
# @PURPOSE: Utility helpers for dictionary management (term normalization and delimiter detection).
# @LAYER: Domain
# #region DictionaryUtils [C:1] [TYPE Module]
# #endregion DictionaryUtils
import re
import unicodedata
# [DEF:_normalize_term:Function]
# @PURPOSE: Normalize a term for case-insensitive unique constraint lookup.
# @RATIONALE: NFC normalization is applied before lowercasing to ensure consistent
# comparison of Unicode characters (e.g. precomposed vs decomposed forms).
# @REJECTED: Lowercasing without NFC normalization — would cause duplicate entries
# for semantically identical Unicode strings in different normalization forms.
# #region _normalize_term [C:2] [TYPE Function] [SEMANTICS dictionary,normalize]
# @BRIEF Normalize a term for case-insensitive unique constraint lookup using NFC normalization.
def _normalize_term(term: str) -> str:
"""Normalize a term by NFC, lowercasing, and removing extra whitespace."""
if not term:
return ""
term = unicodedata.normalize('NFC', term)
return " ".join(term.lower().split())
# [/DEF:_normalize_term:Function]
# #endregion _normalize_term
# [DEF:_detect_delimiter:Function]
# @PURPOSE: Detect the delimiter used in a CSV/TSV header line.
# #region _detect_delimiter [C:1] [TYPE Function] [SEMANTICS dictionary,delimiter]
def _detect_delimiter(header_line: str) -> str:
"""Detect delimiter by counting tabs vs commas in the first line."""
if not header_line:
@@ -32,4 +24,4 @@ def _detect_delimiter(header_line: str) -> str:
tab_count = header_line.count("\t")
comma_count = header_line.count(",")
return "\t" if tab_count > comma_count else ","
# [/DEF:_detect_delimiter:Function]
# #endregion _detect_delimiter

191
backend/src/plugins/translate/dictionary.py
View File

@@ -1,15 +1,15 @@
# [DEF:DictionaryManagerModule:Module]
# @COMPLEXITY: 4
# @SEMANTICS: dictionary, manager, terminology, crud, import, filter
# @PURPOSE: Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [TerminologyDictionary:Class]
# @RELATION: DEPENDS_ON -> [DictionaryEntry:Class]
# @RELATION: DEPENDS_ON -> [TranslationJobDictionary:Class]
# @RELATION: DEPENDS_ON -> [TranslationJob:Class]
# @RATIONALE: C4 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term.
# @REJECTED: Pure C3 CRUD without state guards would allow orphaned job-dictionary links.
# @REJECTED: "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing.
# #region DictionaryManagerModule [C:5] [TYPE Module] [SEMANTICS dictionary,manager,terminology,crud,import,filter]
# @BRIEF Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TerminologyDictionary]
# @RELATION DEPENDS_ON -> [DictionaryEntry]
# @RELATION DEPENDS_ON -> [TranslationJobDictionary]
# @RELATION DEPENDS_ON -> [TranslationJob]
# @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary]
# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) prohibits duplicate entries within a dictionary.
# @RATIONALE C5 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term.
# @REJECTED Pure C3 CRUD without state guards would allow orphaned job-dictionary links.
# @REJECTED "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing.
import csv
import io
@@ -27,18 +27,17 @@ from ...models.translate import (
from ._utils import _normalize_term, _detect_delimiter
# [DEF:DictionaryManager:Class]
# @COMPLEXITY: 4
# @PURPOSE: Manages terminology dictionaries and their entries with referential integrity.
# @PRE: Database session is open and valid.
# @POST: Dictionary and entry mutations are persisted with conflict detection.
# @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
# #region DictionaryManager [C:5] [TYPE Class] [SEMANTICS dictionary,manager,crud,import]
# @BRIEF Manages terminology dictionaries and their entries with referential integrity, import, and batch filtering.
# @PRE Database session is open and valid.
# @POST Dictionary and entry mutations are persisted with conflict detection.
# @SIDE_EFFECT Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
# @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary]
# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) enforces no duplicates within a dictionary.
class DictionaryManager:
# [DEF:DictionaryManager.create_dictionary:Function]
# @COMPLEXITY: 3
# @PURPOSE: Create a new terminology dictionary.
# @PRE: payload contains name, source_dialect, target_dialect.
# @POST: New TerminologyDictionary row is created and returned.
# #region DictionaryManager.create_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,create]
# @BRIEF Create a new terminology dictionary with name, source_dialect, and target_dialect.
# @RELATION DEPENDS_ON -> [TerminologyDictionary]
@staticmethod
def create_dictionary(
db: Session, name: str, source_dialect: str, target_dialect: str,
@@ -60,13 +59,11 @@ class DictionaryManager:
db.refresh(dictionary)
logger.reflect("Dictionary created", {"id": dictionary.id})
return dictionary
# [/DEF:DictionaryManager.create_dictionary:Function]
# #endregion DictionaryManager.create_dictionary
# [DEF:DictionaryManager.update_dictionary:Function]
# @COMPLEXITY: 3
# @PURPOSE: Update an existing terminology dictionary.
# @PRE: dict_id exists in terminology_dictionaries table.
# @POST: Dictionary metadata is updated and returned.
# #region DictionaryManager.update_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,update]
# @BRIEF Update an existing terminology dictionary's metadata fields.
# @RELATION DEPENDS_ON -> [TerminologyDictionary]
@staticmethod
def update_dictionary(
db: Session, dict_id: str, name: Optional[str] = None,
@@ -92,14 +89,13 @@ class DictionaryManager:
db.refresh(dictionary)
logger.reflect("Dictionary updated", {"id": dictionary.id})
return dictionary
# [/DEF:DictionaryManager.update_dictionary:Function]
# #endregion DictionaryManager.update_dictionary
# [DEF:DictionaryManager.delete_dictionary:Function]
# @COMPLEXITY: 4
# @PURPOSE: Delete a dictionary, blocked if attached to active/scheduled jobs.
# @PRE: dict_id exists.
# @POST: Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.
# @SIDE_EFFECT: Deletes TerminologyDictionary row and all DictionaryEntry rows.
# #region DictionaryManager.delete_dictionary [C:4] [TYPE Function] [SEMANTICS dictionary,delete]
# @BRIEF Delete a dictionary, blocked if attached to active/scheduled jobs (referential integrity guard).
# @PRE dict_id exists.
# @POST Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.
# @SIDE_EFFECT Deletes TerminologyDictionary row and all DictionaryEntry rows; checks job attachments first.
@staticmethod
def delete_dictionary(db: Session, dict_id: str) -> None:
with belief_scope("DictionaryManager.delete_dictionary"):
@@ -136,26 +132,20 @@ class DictionaryManager:
db.delete(dictionary)
db.commit()
logger.reflect("Dictionary deleted", {"id": dict_id})
# [/DEF:DictionaryManager.delete_dictionary:Function]
# #endregion DictionaryManager.delete_dictionary
# [DEF:DictionaryManager.get_dictionary:Function]
# @COMPLEXITY: 2
# @PURPOSE: Get a single dictionary by ID with entry count.
# @PRE: dict_id exists.
# @POST: Returns dict with dictionary + entry_count or raises ValueError.
# #region DictionaryManager.get_dictionary [C:2] [TYPE Function] [SEMANTICS dictionary,get]
# @BRIEF Get a single dictionary by ID with entry count.
@staticmethod
def get_dictionary(db: Session, dict_id: str) -> TerminologyDictionary:
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
if not dictionary:
raise ValueError(f"Dictionary not found: {dict_id}")
return dictionary
# [/DEF:DictionaryManager.get_dictionary:Function]
# #endregion DictionaryManager.get_dictionary
# [DEF:DictionaryManager.list_dictionaries:Function]
# @COMPLEXITY: 2
# @PURPOSE: List dictionaries with pagination and entry counts.
# @PRE: page >= 1, page_size between 1 and 100.
# @POST: Returns (list of dicts, total_count).
# #region DictionaryManager.list_dictionaries [C:2] [TYPE Function] [SEMANTICS dictionary,list]
# @BRIEF List dictionaries with pagination and total count.
@staticmethod
def list_dictionaries(
db: Session, page: int = 1, page_size: int = 20,
@@ -169,13 +159,11 @@ class DictionaryManager:
.all()
)
return dictionaries, total
# [/DEF:DictionaryManager.list_dictionaries:Function]
# #endregion DictionaryManager.list_dictionaries
# [DEF:DictionaryManager.add_entry:Function]
# @COMPLEXITY: 3
# @PURPOSE: Add an entry to a dictionary, enforcing unique source_term_normalized.
# @PRE: dict_id exists. source_term and target_term are non-empty.
# @POST: New DictionaryEntry row is created or raises on duplicate.
# #region DictionaryManager.add_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,add]
# @BRIEF Add an entry to a dictionary, enforcing unique source_term_normalized per dictionary.
# @RELATION DEPENDS_ON -> [DictionaryEntry]
@staticmethod
def add_entry(
db: Session, dict_id: str, source_term: str, target_term: str,
@@ -210,13 +198,11 @@ class DictionaryManager:
db.refresh(entry)
logger.reflect("Entry added", {"entry_id": entry.id})
return entry
# [/DEF:DictionaryManager.add_entry:Function]
# #endregion DictionaryManager.add_entry
# [DEF:DictionaryManager.edit_entry:Function]
# @COMPLEXITY: 3
# @PURPOSE: Edit an existing dictionary entry with duplicate-aware normalization.
# @PRE: entry_id exists.
# @POST: Entry fields are updated.
# #region DictionaryManager.edit_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,edit]
# @BRIEF Edit an existing dictionary entry with duplicate-aware normalization check.
# @RELATION DEPENDS_ON -> [DictionaryEntry]
@staticmethod
def edit_entry(
db: Session, entry_id: str, source_term: Optional[str] = None,
@@ -255,13 +241,10 @@ class DictionaryManager:
db.refresh(entry)
logger.reflect("Entry updated", {"entry_id": entry.id})
return entry
# [/DEF:DictionaryManager.edit_entry:Function]
# #endregion DictionaryManager.edit_entry
# [DEF:DictionaryManager.delete_entry:Function]
# @COMPLEXITY: 2
# @PURPOSE: Delete a single dictionary entry.
# @PRE: entry_id exists.
# @POST: Entry is deleted.
# #region DictionaryManager.delete_entry [C:2] [TYPE Function] [SEMANTICS dictionary,entry,delete]
# @BRIEF Delete a single dictionary entry.
@staticmethod
def delete_entry(db: Session, entry_id: str) -> None:
with belief_scope("DictionaryManager.delete_entry"):
@@ -272,13 +255,10 @@ class DictionaryManager:
db.delete(entry)
db.commit()
logger.reflect("Entry deleted", {"entry_id": entry_id})
# [/DEF:DictionaryManager.delete_entry:Function]
# #endregion DictionaryManager.delete_entry
# [DEF:DictionaryManager.clear_entries:Function]
# @COMPLEXITY: 2
# @PURPOSE: Delete all entries for a dictionary.
# @PRE: dict_id exists.
# @POST: All entries for the dictionary are deleted.
# #region DictionaryManager.clear_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,clear]
# @BRIEF Delete all entries for a dictionary.
@staticmethod
def clear_entries(db: Session, dict_id: str) -> int:
with belief_scope("DictionaryManager.clear_entries"):
@@ -290,13 +270,10 @@ class DictionaryManager:
db.commit()
logger.reflect("Entries cleared", {"dict_id": dict_id, "count": deleted})
return deleted
# [/DEF:DictionaryManager.clear_entries:Function]
# #endregion DictionaryManager.clear_entries
# [DEF:DictionaryManager.list_entries:Function]
# @COMPLEXITY: 2
# @PURPOSE: List entries for a dictionary with pagination.
# @PRE: dict_id exists.
# @POST: Returns (list of entries, total_count).
# #region DictionaryManager.list_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,list]
# @BRIEF List entries for a dictionary with pagination.
@staticmethod
def list_entries(
db: Session, dict_id: str, page: int = 1, page_size: int = 50,
@@ -316,14 +293,13 @@ class DictionaryManager:
.all()
)
return entries, total
# [/DEF:DictionaryManager.list_entries:Function]
# #endregion DictionaryManager.list_entries
# [DEF:DictionaryManager.import_entries:Function]
# @COMPLEXITY: 4
# @PURPOSE: Import entries from CSV/TSV content with duplicate detection and conflict resolution.
# @PRE: content is valid CSV or TSV. dict_id exists.
# @POST: Entries are created/updated/skipped per conflict mode. Returns result summary.
# @SIDE_EFFECT: Batch-inserts or updates DictionaryEntry rows.
# #region DictionaryManager.import_entries [C:4] [TYPE Function] [SEMANTICS dictionary,import,csv]
# @BRIEF Import entries from CSV/TSV content with duplicate detection and conflict resolution (overwrite/keep_existing/cancel).
# @PRE content is valid CSV or TSV. dict_id exists.
# @POST Entries are created/updated/skipped per conflict mode. Returns result summary with preview support.
# @SIDE_EFFECT Batch-inserts or updates DictionaryEntry rows.
@staticmethod
def import_entries(
db: Session, dict_id: str, content: str,
@@ -456,14 +432,13 @@ class DictionaryManager:
"errors": len(result["errors"]),
})
return result
# [/DEF:DictionaryManager.import_entries:Function]
# #endregion DictionaryManager.import_entries
# [DEF:DictionaryManager.filter_for_batch:Function]
# @COMPLEXITY: 4
# @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.
# @PRE: job_id exists and source_texts is a list of strings.
# @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries.
# @SIDE_EFFECT: Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.
# #region DictionaryManager.filter_for_batch [C:4] [TYPE Function] [SEMANTICS dictionary,filter,batch]
# @BRIEF Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.
# @PRE job_id exists and source_texts is a list of strings.
# @POST Returns list of matched entries with match info, sorted by dictionary link priority.
# @SIDE_EFFECT Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.
@staticmethod
def filter_for_batch(
db: Session, source_texts: List[str], job_id: str,
@@ -554,15 +529,14 @@ class DictionaryManager:
"dictionaries_used": len(dictionaries),
})
return matched
# [/DEF:DictionaryManager.filter_for_batch:Function]
# #endregion DictionaryManager.filter_for_batch
# [DEF:DictionaryManager.submit_correction:Function]
# @COMPLEXITY: 4
# @PURPOSE: Submit a term correction from a run result. Validates language match, detects conflicts.
# @PRE: source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.
# @POST: Entry created or updated; origin tracking populated. Returns action + conflict info.
# @SIDE_EFFECT: Creates/updates DictionaryEntry row.
# #region DictionaryManager.submit_correction [C:4] [TYPE Function] [SEMANTICS dictionary,correction,submit]
# @BRIEF Submit a term correction from a run result with conflict detection and origin tracking.
# @PRE source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.
# @POST Entry created or updated; origin tracking populated. Returns action + conflict info.
# @SIDE_EFFECT Creates/updates DictionaryEntry row.
@staticmethod
def submit_correction(
db: Session,
@@ -653,14 +627,13 @@ class DictionaryManager:
db.commit()
logger.reflect("Correction processed", result)
return result
# [/DEF:DictionaryManager.submit_correction:Function]
# #endregion DictionaryManager.submit_correction
# [DEF:DictionaryManager.submit_bulk_corrections:Function]
# @COMPLEXITY: 4
# @PURPOSE: Submit multiple term corrections atomically — all succeed or all fail.
# @PRE: corrections list is non-empty. dict_id exists.
# @POST: All corrections applied or none applied with conflict list.
# @SIDE_EFFECT: Creates/updates DictionaryEntry rows; commits once.
# #region DictionaryManager.submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS dictionary,correction,bulk]
# @BRIEF Submit multiple term corrections atomically — all succeed or all fail on conflict.
# @PRE corrections list is non-empty. dict_id exists.
# @POST All corrections applied or none applied with conflict list (atomic).
# @SIDE_EFFECT Creates/updates DictionaryEntry rows; commits once.
@staticmethod
def submit_bulk_corrections(
db: Session,
@@ -774,8 +747,8 @@ class DictionaryManager:
"total": len(corrections),
"applied": sum(1 for r in results if r["action"] in ("created", "updated")),
}
# [/DEF:DictionaryManager.submit_bulk_corrections:Function]
# #endregion DictionaryManager.submit_bulk_corrections
# [/DEF:DictionaryManager:Class]
# [/DEF:DictionaryManagerModule:Module]
# #endregion DictionaryManager
# #endregion DictionaryManagerModule

154
backend/src/plugins/translate/events.py
View File

@@ -1,18 +1,16 @@
# [DEF:TranslationEventLog:Module]
# @COMPLEXITY: 5
# @SEMANTICS: translate, events, audit, logging
# @PURPOSE: Structured event logging for translation operations with terminal event invariant enforcement.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [TranslationEvent]
# @RELATION: DEPENDS_ON -> [MetricSnapshot]
# @PRE: Database session is open and valid.
# @POST: Events are persisted immutably; terminal events enforce exactly-one invariant per run.
# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion.
# @DATA_CONTRACT: Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]
# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id.
# @RATIONALE: Immutable event log with nullable run_id allows job-level events (no run context) and run-level events.
# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# @REJECTED: Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit.
# #region TranslationEventLog [C:5] [TYPE Module] [SEMANTICS translate,events,audit,logging]
# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationEvent]
# @RELATION DEPENDS_ON -> [MetricSnapshot]
# @PRE Database session is open and valid.
# @POST Events are persisted immutably; terminal events enforce exactly-one invariant per run.
# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion.
# @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]
# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id.
# @RATIONALE Immutable event log with nullable run_id allows job-level events (no run context) and run-level events.
# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# @REJECTED Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit.
from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from datetime import datetime, timezone, timedelta
@@ -38,24 +36,48 @@ VALID_EVENT_TYPES = TERMINAL_EVENT_TYPES | {
DEFAULT_RETENTION_DAYS = 90
# [DEF:TranslationEventLog:Class]
# @COMPLEXITY: 5
# @PURPOSE: Structured event logging for translation operations with terminal event invariant enforcement.
# @PRE: Database session is available.
# @POST: Events are written immutably; terminal events enforced per run.
# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events.
# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id.
# #region TranslationEventLog [C:5] [TYPE Class] [SEMANTICS translate,events,log,invariants]
# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement and pruning.
# @PRE Database session is available.
# @POST Events are written immutably; terminal events enforced per run; expired events pruned with MetricSnapshot.
# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events.
# @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]
# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id.
class TranslationEventLog:
def __init__(self, db: Session):
self.db = db
# [DEF:log_event:Function]
# @COMPLEXITY: 4
# @PURPOSE: Write an immutable event. Enforces terminal event invariant for non-null run_id.
# @PRE: event_type must be a known type. If run_id is not None, enforce terminal invariant.
# @POST: TranslationEvent row is created.
# @SIDE_EFFECT: DB write.
# #region _create_event_raw [C:3] [TYPE Function] [SEMANTICS translate,events,create]
# @BRIEF Create a TranslationEvent row without invariant checks. Internal helper for callers that manage invariants themselves.
# @RELATION DEPENDS_ON -> [TranslationEvent]
def _create_event_raw(
self,
job_id: str,
event_type: str,
payload: Optional[Dict[str, Any]] = None,
run_id: Optional[str] = None,
created_by: Optional[str] = None,
) -> TranslationEvent:
event = TranslationEvent(
id=str(uuid.uuid4()),
job_id=job_id,
run_id=run_id,
event_type=event_type,
event_data=payload or {},
created_by=created_by,
created_at=datetime.now(timezone.utc),
)
self.db.add(event)
self.db.flush()
return event
# #endregion _create_event_raw
# #region log_event [C:4] [TYPE Function] [SEMANTICS translate,events,log]
# @BRIEF Write an immutable event. Enforces terminal event invariant and run_started ordering for non-null run_id.
# @PRE event_type must be a known type. If run_id is not None, enforce terminal + start invariants.
# @POST TranslationEvent row is created; invariants checked before write.
# @SIDE_EFFECT DB write.
def log_event(
self,
job_id: str,
@@ -82,9 +104,10 @@ class TranslationEventLog:
.first()
)
if existing_terminal:
existing_id = existing_terminal[0] if isinstance(existing_terminal, (tuple, list)) else str(existing_terminal.id)
raise ValueError(
f"Run '{run_id}' already has a terminal event "
f"({existing_terminal[0]}). Cannot add another terminal event."
f"({existing_id}). Cannot add another terminal event."
)
# Enforce run_started invariant — must exist before other run events
@@ -103,18 +126,13 @@ class TranslationEventLog:
f"RUN_STARTED event must precede other run events."
)
event = TranslationEvent(
id=str(uuid.uuid4()),
event = self._create_event_raw(
job_id=job_id,
run_id=run_id,
event_type=event_type,
event_data=payload or {},
payload=payload,
run_id=run_id,
created_by=created_by,
created_at=datetime.now(timezone.utc),
)
self.db.add(event)
self.db.flush()
logger.reason(f"Event logged: {event_type}", {
"event_id": event.id,
"job_id": job_id,
@@ -122,12 +140,10 @@ class TranslationEventLog:
"event_type": event_type,
})
return event
# [/DEF:log_event:Function]
# #endregion log_event
# [DEF:query_events:Function]
# @PURPOSE: Query events with optional filters.
# @PRE: None.
# @POST: Returns list of TranslationEvent dicts matching filters.
# #region query_events [C:2] [TYPE Function] [SEMANTICS translate,events,query]
# @BRIEF Query events with optional filters (job_id, run_id, event_type) and pagination.
def query_events(
self,
job_id: Optional[str] = None,
@@ -165,13 +181,13 @@ class TranslationEventLog:
}
for e in events
]
# [/DEF:query_events:Function]
# #endregion query_events
# [DEF:prune_expired:Function]
# @PURPOSE: Delete events older than retention_days. Persists MetricSnapshot before pruning.
# @PRE: None.
# @POST: Expired events are deleted; MetricSnapshot is created before deletion.
# @SIDE_EFFECT: Creates MetricSnapshot row; deletes TranslationEvent rows.
# #region prune_expired [C:4] [TYPE Function] [SEMANTICS translate,events,prune]
# @BRIEF Delete events older than retention_days. Creates MetricSnapshot before pruning for audit trail.
# @PRE None.
# @POST Expired events are deleted; MetricSnapshot is created before deletion in batch.
# @SIDE_EFFECT Creates MetricSnapshot row; deletes TranslationEvent rows in batches.
def prune_expired(
self,
retention_days: int = DEFAULT_RETENTION_DAYS,
@@ -233,12 +249,11 @@ class TranslationEventLog:
"snapshot_id": snapshot_id,
})
return {"pruned": pruned, "snapshot_id": snapshot_id}
# [/DEF:prune_expired:Function]
# #endregion prune_expired
# [DEF:get_run_event_summary:Function]
# @PURPOSE: Get a summary of events for a run, including invariant check.
# @PRE: run_id is not None.
# @POST: Returns dict with event list and invariant validity.
# #region get_run_event_summary [C:3] [TYPE Function] [SEMANTICS translate,events,summary]
# @BRIEF Get a summary of events for a run, including invariant validity check.
# @RELATION DEPENDS_ON -> [TranslationEvent]
def get_run_event_summary(self, run_id: str) -> Dict[str, Any]:
with belief_scope("TranslationEventLog.get_run_event_summary"):
events = self.query_events(run_id=run_id)
@@ -255,8 +270,35 @@ class TranslationEventLog:
"invariant_valid": has_started and len(terminal_events) <= 1,
"events": events,
}
# [/DEF:get_run_event_summary:Function]
# #endregion get_run_event_summary
# #region get_run_event_invariants_lightweight [C:3] [TYPE Function] [SEMANTICS translate,events,invariants]
# @BRIEF Lightweight invariant check for a run — fetches only event_type column, avoids event_data for performance.
# @RELATION DEPENDS_ON -> [TranslationEvent]
def get_run_event_invariants_lightweight(self, run_id: str) -> Dict[str, Any]:
with belief_scope("TranslationEventLog.get_run_event_invariants_lightweight"):
# Lightweight query: only fetch event_type column, not event_data
event_types = [
row[0]
for row in (
self.db.query(TranslationEvent.event_type)
.filter(TranslationEvent.run_id == run_id)
.order_by(TranslationEvent.created_at.desc())
.limit(100)
.all()
)
]
has_run_started = "RUN_STARTED" in event_types
terminal_count = sum(1 for et in event_types if et in TERMINAL_EVENT_TYPES)
return {
"has_run_started": has_run_started,
"terminal_event_count": terminal_count,
"invariant_valid": has_run_started and terminal_count <= 1,
}
# #endregion get_run_event_invariants_lightweight
# [/DEF:TranslationEventLog:Class]
# [/DEF:TranslationEventLog:Module]
# #endregion TranslationEventLog
# #endregion TranslationEventLog

268
backend/src/plugins/translate/executor.py
View File

@@ -1,19 +1,19 @@
# [DEF:TranslationExecutor:Module]
# @COMPLEXITY: 4
# @SEMANTICS: translate, executor, batch, llm
# @PURPOSE: Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [TranslationBatch]
# @RELATION: DEPENDS_ON -> [TranslationRecord]
# @RELATION: DEPENDS_ON -> [TranslationRun]
# @RELATION: DEPENDS_ON -> [LLMProviderService]
# @RELATION: DEPENDS_ON -> [DictionaryManager]
# @RELATION: DEPENDS_ON -> [TranslationPreview]
# @PRE: Valid TranslationRun with job configuration. DB session is available.
# @POST: TranslationBatch and TranslationRecord rows are created. Run status is updated.
# @SIDE_EFFECT: Calls LLM provider; creates DB rows; updates run statistics.
# @RATIONALE: Batch processing with retry — independent batches allow partial recovery.
# @REJECTED: Single monolithic LLM call — would lose all progress on any failure.
# #region TranslationExecutor [C:5] [TYPE Module] [SEMANTICS translate,executor,batch,llm]
# @BRIEF Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationBatch]
# @RELATION DEPENDS_ON -> [TranslationRecord]
# @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [DictionaryManager]
# @RELATION DEPENDS_ON -> [TranslationPreview]
# @PRE Valid TranslationRun with job configuration. DB session is available.
# @POST TranslationBatch and TranslationRecord rows are created. Run status is updated.
# @SIDE_EFFECT Calls LLM provider; creates DB rows; updates run statistics.
# @DATA_CONTRACT Input[TranslationRun, job] -> Output[TranslationRun with batches and records]
# @INVARIANT Batch processing with retry — independent batches allow partial recovery; MAX_RETRIES_PER_BATCH = 3.
# @RATIONALE Batch processing with retry — independent batches allow partial recovery.
# @REJECTED Single monolithic LLM call — would lose all progress on any failure.
import json
import time
@@ -35,21 +35,23 @@ from ...models.translate import (
)
from ...services.llm_provider import LLMProviderService
from ...services.llm_prompt_templates import render_prompt
from ...core.superset_client import SupersetClient
from .dictionary import DictionaryManager
from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE
# [DEF:MAX_RETRIES_PER_BATCH:Constant]
# @PURPOSE: Maximum number of retries for a single batch before marking it failed.
# #region MAX_RETRIES_PER_BATCH [C:1] [TYPE Constant] [SEMANTICS translate,executor,retry]
MAX_RETRIES_PER_BATCH = 3
# #endregion MAX_RETRIES_PER_BATCH
# [DEF:TranslationExecutor:Class]
# @COMPLEXITY: 4
# @PURPOSE: Process translation batches: fetch source rows, filter dict, call LLM, persist results.
# @PRE: DB session and config manager available.
# @POST: Batches and records created with status tracking.
# @SIDE_EFFECT: LLM API calls; DB writes.
# #region TranslationExecutor [C:5] [TYPE Class] [SEMANTICS translate,executor,batch]
# @BRIEF Process translation batches: fetch source rows, filter dict, call LLM, persist results.
# @PRE DB session and config manager available.
# @POST Batches and records created with status tracking; run statistics updated.
# @SIDE_EFFECT LLM API calls; DB writes.
# @DATA_CONTRACT Input[TranslationRun] -> Output[TranslationRun with batches, records, stats]
# @INVARIANT MAX_RETRIES_PER_BATCH limit enforced; partial batch success tracked via COMPLETED_WITH_ERRORS.
class TranslationExecutor:
def __init__(
@@ -65,15 +67,16 @@ class TranslationExecutor:
self.on_batch_progress = on_batch_progress
self._current_run_id: Optional[str] = None
# [DEF:execute_run:Function]
# @PURPOSE: Run full translation execution for a TranslationRun.
# @PRE: run is in PENDING or RUNNING status with valid job config.
# @POST: Run is populated with batches and records.
# @SIDE_EFFECT: LLM API calls; DB batch writes.
# #region execute_run [C:4] [TYPE Function] [SEMANTICS translate,run,execute]
# @BRIEF Run full translation execution for a TranslationRun: fetch rows, batch, call LLM, persist.
# @PRE run is in PENDING or RUNNING status with valid job config.
# @POST Run is populated with batches and records; run status updated to COMPLETED/FAILED.
# @SIDE_EFFECT LLM API calls; DB batch writes.
def execute_run(
self,
run: TranslationRun,
llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None,
full_translation: bool = False,
) -> TranslationRun:
with belief_scope("TranslationExecutor.execute_run"):
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
@@ -84,6 +87,7 @@ class TranslationExecutor:
"run_id": run.id,
"job_id": job.id,
"batch_size": job.batch_size,
"full_translation": full_translation,
})
# Mark run as RUNNING
@@ -91,8 +95,13 @@ class TranslationExecutor:
run.started_at = datetime.now(timezone.utc)
self.db.flush()
# Fetch source rows from the accepted preview session
source_rows = self._fetch_source_rows(job.id, run.id)
# Fetch source rows
if full_translation:
# Full translation: fetch ALL rows from Superset dataset
source_rows = self._fetch_all_rows_from_superset(job)
else:
# Preview-based: fetch rows from the accepted preview session
source_rows = self._fetch_source_rows(job.id, run.id)
if not source_rows:
logger.explore("No source rows to translate", {"run_id": run.id})
run.status = "COMPLETED"
@@ -164,12 +173,13 @@ class TranslationExecutor:
})
return run
# [/DEF:execute_run:Function]
# #endregion execute_run
# [DEF:_fetch_source_rows:Function]
# @PURPOSE: Fetch source rows from the accepted preview session for this job.
# @PRE: job_id exists.
# @POST: Returns list of dicts with source data.
# #region _fetch_source_rows [C:4] [TYPE Function] [SEMANTICS translate,source,rows]
# @BRIEF Fetch source rows from the accepted preview session for this job.
# @PRE job_id exists.
# @POST Returns list of dicts with source data (row_index, source_text, approved_translation, etc).
# @SIDE_EFFECT Queries preview session and records from DB.
def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]:
with belief_scope("TranslationExecutor._fetch_source_rows"):
# Get the latest APPLIED preview session
@@ -203,6 +213,7 @@ class TranslationExecutor:
"source_text": rec.source_sql or "",
"approved_translation": rec.target_sql if rec.status == "APPROVED" else None,
"source_object_name": rec.source_object_name or "",
"source_data": rec.source_data or {},
})
logger.reason(f"Fetched {len(source_rows)} source rows from preview", {
@@ -210,13 +221,115 @@ class TranslationExecutor:
"session_id": session.id,
})
return source_rows
# [/DEF:_fetch_source_rows:Function]
# #endregion _fetch_source_rows
# [DEF:_process_batch:Function]
# @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records.
# @PRE: job and batch_rows are valid.
# @POST: TranslationBatch and TranslationRecord rows are created.
# @SIDE_EFFECT: LLM API call.
# #region _fetch_all_rows_from_superset [C:4] [TYPE Function] [SEMANTICS translate,superset,rows]
# @BRIEF Fetch ALL rows from the Superset dataset for full translation (paginated).
# @PRE job has source_datasource_id and environment_id configured.
# @POST Returns list of row dicts with row_index, source_text, source_data.
# @SIDE_EFFECT Calls Superset chart data endpoint (paginated).
def _fetch_all_rows_from_superset(self, job: TranslationJob) -> List[Dict[str, Any]]:
with belief_scope("TranslationExecutor._fetch_all_rows_from_superset"):
env_id = job.environment_id or job.source_dialect or ""
environments = self.config_manager.get_environments()
env_config = next(
(e for e in environments if e.id == env_id),
None,
)
if not env_config:
raise ValueError(f"Superset environment '{env_id}' not found")
client = SupersetClient(env_config)
dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))
# Build query context (same as preview, but with large row_limit)
query_context = client.build_dataset_preview_query_context(
dataset_id=int(job.source_datasource_id),
dataset_record=dataset_detail,
template_params={},
effective_filters=[],
)
queries = query_context.get("queries", [])
if queries:
queries[0]["row_limit"] = 100000 # Superset max default
queries[0].pop("result_type", None)
queries[0].pop("columns", None)
queries[0]["metrics"] = []
queries[0]["row_offset"] = 0
query_context["result_type"] = "samples"
form_data = query_context.get("form_data", {})
form_data.pop("query_mode", None)
all_rows: List[Dict[str, Any]] = []
batch_size_fetch = 10000 # Fetch 10K rows per pagination request
while True:
# Set pagination for this batch
if queries:
queries[0]["row_limit"] = min(batch_size_fetch, 100000)
queries[0]["row_offset"] = len(all_rows)
try:
response = client.network.request(
method="POST",
endpoint="/api/v1/chart/data",
data=json.dumps(query_context),
headers={"Content-Type": "application/json"},
)
except Exception as e:
logger.explore("Chart data API failed during full fetch", {
"offset": len(all_rows),
"error": str(e),
})
if not all_rows:
raise ValueError(f"Failed to fetch data from Superset: {e}")
break # Return what we have
from .preview import TranslationPreview
rows = TranslationPreview._extract_data_rows(response)
if not rows:
break # No more data
all_rows.extend(rows)
logger.reason(f"Fetched {len(rows)} rows (total: {len(all_rows)})", {
"offset": len(all_rows) - len(rows),
})
if len(rows) < batch_size_fetch:
break # Last page
# Convert to the format expected by _process_batch
source_rows = []
for idx, row in enumerate(all_rows):
translation_value = str(row.get(job.translation_column, "") or "")
context_values = {}
if job.context_columns:
for col in job.context_columns:
context_values[col] = str(row.get(col, "") or "")
# Also include target_key_cols values in context_data
if job.target_key_cols:
for col in job.target_key_cols:
context_values[col] = str(row.get(col, "") or "")
source_rows.append({
"row_index": str(idx),
"source_text": translation_value,
"source_data": context_values,
"source_object_name": f"Row {idx + 1}",
"approved_translation": None,
})
logger.reason(f"Prepared {len(source_rows)} source rows for full translation", {
"job_id": job.id,
})
return source_rows
# #endregion _fetch_all_rows_from_superset
# #region _process_batch [C:4] [TYPE Function] [SEMANTICS translate,batch,process]
# @BRIEF Process a single batch: filter dictionary, build prompt, call LLM, persist records.
# @PRE job and batch_rows are valid.
# @POST TranslationBatch and TranslationRecord rows are created.
# @SIDE_EFFECT LLM API call; DB writes.
def _process_batch(
self,
job: TranslationJob,
@@ -275,6 +388,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SUCCESS",
)
self.db.add(record)
@@ -309,13 +423,13 @@ class TranslationExecutor:
})
return result
# [/DEF:_process_batch:Function]
# #endregion _process_batch
# [DEF:_call_llm_for_batch:Function]
# @PURPOSE: Call LLM for a batch of rows requiring translation. Parse structured JSON response.
# @PRE: job has valid provider_id. batch_rows is non-empty.
# @POST: Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.
# @SIDE_EFFECT: HTTP call to LLM provider.
# #region _call_llm_for_batch [C:4] [TYPE Function] [SEMANTICS translate,llm,batch]
# @BRIEF Call LLM for a batch of rows requiring translation. Parse structured JSON response.
# @PRE job has valid provider_id. batch_rows is non-empty.
# @POST Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.
# @SIDE_EFFECT HTTP call to LLM provider.
def _call_llm_for_batch(
self,
job: TranslationJob,
@@ -401,6 +515,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="FAILED",
error_message=f"LLM call failed after {retries} retries: {last_error}",
)
@@ -428,6 +543,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED",
error_message=f"LLM parse failure: {e}",
)
@@ -459,6 +575,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED",
error_message="NULL translation returned by LLM",
)
@@ -477,6 +594,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED",
error_message="Empty translation returned by LLM",
)
@@ -493,6 +611,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SUCCESS",
)
self.db.add(record)
@@ -503,13 +622,13 @@ class TranslationExecutor:
"skipped": skipped,
"retries": retries,
}
# [/DEF:_call_llm_for_batch:Function]
# #endregion _call_llm_for_batch
# [DEF:_call_llm:Function]
# @PURPOSE: Call the configured LLM provider with the batch prompt.
# @PRE: job has valid provider_id.
# @POST: Returns raw LLM response string.
# @SIDE_EFFECT: HTTP call to LLM provider.
# #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call]
# @BRIEF Call the configured LLM provider with the batch prompt, routing by provider type.
# @PRE job has valid provider_id.
# @POST Returns raw LLM response string.
# @SIDE_EFFECT HTTP call to LLM provider.
def _call_llm(self, job: TranslationJob, prompt: str) -> str:
with belief_scope("TranslationExecutor._call_llm"):
if not job.provider_id:
@@ -537,13 +656,13 @@ class TranslationExecutor:
)
else:
raise ValueError(f"Unsupported provider type '{provider_type}'")
# [/DEF:_call_llm:Function]
# #endregion _call_llm
# [DEF:_call_openai_compatible:Function]
# @PURPOSE: Call OpenAI-compatible API for batch translation.
# @PRE: Valid API endpoint, key, model, and prompt.
# @POST: Returns response text.
# @SIDE_EFFECT: HTTP POST to LLM API.
# #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai]
# @BRIEF Call OpenAI-compatible API for batch translation with retry and structured output support.
# @PRE Valid API endpoint, key, model, and prompt.
# @POST Returns response text.
# @SIDE_EFFECT HTTP POST to LLM API.
@staticmethod
def _call_openai_compatible(
base_url: str,
@@ -596,15 +715,26 @@ class TranslationExecutor:
content = choices[0].get("message", {}).get("content", "")
if not content:
raise ValueError("LLM returned empty content")
# Log full response for diagnostics
finish_reason = choices[0].get("finish_reason", "unknown")
logger.explore("LLM returned empty content", {
"finish_reason": finish_reason,
"model": payload.get("model"),
"prompt_len": len(prompt),
"response_keys": list(data.keys()),
"choices_count": len(choices),
})
raise ValueError(
f"LLM returned empty content (finish_reason={finish_reason}, "
f"model={payload.get('model')}, prompt_len={len(prompt)})"
)
return content
# [/DEF:_call_openai_compatible:Function]
# #endregion _call_openai_compatible
# [DEF:_parse_llm_response:Function]
# @PURPOSE: Parse LLM JSON response into dict of row_id -> translation.
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
# @POST: Returns dict mapping row_id to translation text.
# #region _parse_llm_response [C:3] [TYPE Function] [SEMANTICS translate,llm,parse]
# @BRIEF Parse LLM JSON response into dict of row_id -> translation text.
# @RELATION DEPENDS_ON -> [json]
@staticmethod
def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:
with belief_scope("TranslationExecutor._parse_llm_response"):
@@ -637,8 +767,8 @@ class TranslationExecutor:
translations[row_id] = str(translation)
return translations
# [/DEF:_parse_llm_response:Function]
# #endregion _parse_llm_response
# [/DEF:TranslationExecutor:Class]
# [/DEF:TranslationExecutor:Module]
# #endregion TranslationExecutor
# #endregion TranslationExecutor

47
backend/src/plugins/translate/metrics.py
View File

@@ -1,16 +1,10 @@
# [DEF:TranslationMetrics:Module]
# @COMPLEXITY: 3
# @SEMANTICS: translate, metrics, aggregation
# @PURPOSE: Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [TranslationEvent:Class]
# @RELATION: DEPENDS_ON -> [MetricSnapshot:Class]
# @RELATION: DEPENDS_ON -> [TranslationRun:Class]
# @RELATION: DEPENDS_ON -> [TranslationSchedule:Class]
# @PRE: Database session is open.
# @POST: Metrics are aggregated and returned; no side effects.
# @RATIONALE: Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture.
# @REJECTED: Querying all events for all time — would be slow; MetricSnapshot provides historical summary.
# #region TranslationMetrics [C:3] [TYPE Module] [SEMANTICS translate,metrics,aggregation]
# @BRIEF Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationEvent]
# @RELATION DEPENDS_ON -> [MetricSnapshot]
# @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [TranslationSchedule]
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Tuple
@@ -26,18 +20,18 @@ from ...models.translate import (
)
# [DEF:TranslationMetrics:Class]
# @COMPLEXITY: 3
# @PURPOSE: Aggregate translation metrics from live events and MetricSnapshot.
# #region TranslationMetrics [C:3] [TYPE Class] [SEMANTICS translate,metrics]
# @BRIEF Aggregate translation metrics from live events and MetricSnapshot.
# @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [MetricSnapshot]
# @RELATION DEPENDS_ON -> [TranslationEvent]
class TranslationMetrics:
def __init__(self, db: Session):
self.db = db
# [DEF:get_job_metrics:Function]
# @PURPOSE: Get aggregated metrics for a specific job.
# @PRE: job_id exists.
# @POST: Returns dict with metrics from events + latest snapshot.
# #region get_job_metrics [C:2] [TYPE Function] [SEMANTICS translate,metrics,job]
# @BRIEF Get aggregated metrics for a specific job including run counts, record stats, and duration.
def get_job_metrics(self, job_id: str) -> Dict[str, Any]:
with belief_scope("TranslationMetrics.get_job_metrics"):
# Run counts from TranslationRun
@@ -151,11 +145,10 @@ class TranslationMetrics:
"last_run_at": last_run.created_at.isoformat() if last_run else None,
"next_scheduled_run": next_schedule.last_run_at.isoformat() if next_schedule and next_schedule.last_run_at else None,
}
# [/DEF:get_job_metrics:Function]
# #endregion get_job_metrics
# [DEF:get_all_metrics:Function]
# @PURPOSE: Get aggregated metrics for all jobs.
# @POST: Returns list of per-job metrics.
# #region get_all_metrics [C:2] [TYPE Function] [SEMANTICS translate,metrics,all]
# @BRIEF Get aggregated metrics for all jobs.
def get_all_metrics(self) -> List[Dict[str, Any]]:
with belief_scope("TranslationMetrics.get_all_metrics"):
job_ids = (
@@ -164,8 +157,8 @@ class TranslationMetrics:
.all()
)
return [self.get_job_metrics(jid[0]) for jid in job_ids if jid[0]]
# [/DEF:get_all_metrics:Function]
# #endregion get_all_metrics
# [/DEF:TranslationMetrics:Class]
# [/DEF:TranslationMetrics:Module]
# #endregion TranslationMetrics
# #endregion TranslationMetrics

236
backend/src/plugins/translate/orchestrator.py
View File

@@ -1,23 +1,21 @@
# [DEF:TranslationOrchestrator:Module]
# @COMPLEXITY: 5
# @SEMANTICS: translate, orchestrator, lifecycle, run
# @PURPOSE: Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [TranslationRun]
# @RELATION: DEPENDS_ON -> [TranslationJob]
# @RELATION: DEPENDS_ON -> [TranslationExecutor]
# @RELATION: DEPENDS_ON -> [SQLGenerator]
# @RELATION: DEPENDS_ON -> [SupersetSqlLabExecutor]
# @RELATION: DEPENDS_ON -> [TranslationEventLog]
# @RELATION: DEPENDS_ON -> [TranslationPreviewSession]
# @PRE: Valid job and accepted preview (for manual runs). Superset and LLM are reachable.
# @POST: Translation run is executed, SQL generated and submitted, events recorded.
# @SIDE_EFFECT: Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events.
# @DATA_CONTRACT: Input[db, config_manager, current_user] -> Output[TranslationRun]
# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run.
# @RATIONALE: C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary.
# @REJECTED: Distributed actor model (Celery) — eventual-consistency challenges at current scale.
# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# #region TranslationOrchestrator [C:5] [TYPE Module] [SEMANTICS translate,orchestrator,lifecycle,run]
# @BRIEF Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION DEPENDS_ON -> [TranslationExecutor]
# @RELATION DEPENDS_ON -> [SQLGenerator]
# @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor]
# @RELATION DEPENDS_ON -> [TranslationEventLog]
# @RELATION DEPENDS_ON -> [TranslationPreviewSession]
# @PRE Valid job and accepted preview (for manual runs). Superset and LLM are reachable.
# @POST Translation run is executed, SQL generated and submitted, events recorded.
# @SIDE_EFFECT Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events.
# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun]
# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run.
# @RATIONALE C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary.
# @REJECTED Distributed actor model (Celery) — eventual-consistency challenges at current scale.
# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
import json
import time
@@ -44,13 +42,13 @@ from .events import TranslationEventLog
from ..translate.service import TranslateJobService
# [DEF:TranslationOrchestrator:Class]
# @COMPLEXITY: 5
# @PURPOSE: Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging.
# @PRE: DB session and config manager are available.
# @POST: Runs are created, executed, and finalized with event records.
# @SIDE_EFFECT: Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows.
# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED.
# #region TranslationOrchestrator [C:5] [TYPE Class] [SEMANTICS translate,orchestrator]
# @BRIEF Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging.
# @PRE DB session and config manager are available.
# @POST Runs are created, executed, and finalized with event records.
# @SIDE_EFFECT Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows.
# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED.
# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun]
class TranslationOrchestrator:
def __init__(
@@ -65,12 +63,11 @@ class TranslationOrchestrator:
self.event_log = TranslationEventLog(db)
self._job: Optional[TranslationJob] = None
# [DEF:start_run:Function]
# @COMPLEXITY: 5
# @PURPOSE: Start a new translation run for a job.
# @PRE: job_id exists. For manual runs, there must be an accepted preview session.
# @POST: TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.
# @SIDE_EFFECT: DB writes.
# #region start_run [C:5] [TYPE Function] [SEMANTICS translate,run,create]
# @BRIEF Start a new translation run for a job with config snapshot and hash computation.
# @PRE job_id exists. For manual runs, there must be an accepted preview session.
# @POST TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.
# @SIDE_EFFECT DB writes; event logging.
def start_run(
self,
job_id: str,
@@ -166,18 +163,19 @@ class TranslationOrchestrator:
"trigger_type": run.trigger_type,
})
return run
# [/DEF:start_run:Function]
# #endregion start_run
# [DEF:execute_run:Function]
# @PURPOSE: Execute a translation run: dispatch executor, generate SQL, submit to Superset.
# @PRE: run is in PENDING status.
# @POST: Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT: LLM calls, DB writes, Superset API calls.
# #region execute_run [C:5] [TYPE Function] [SEMANTICS translate,run,execute]
# @BRIEF Execute a translation run: dispatch executor, generate SQL, submit to Superset.
# @PRE run is in PENDING status.
# @POST Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
def execute_run(
self,
run: TranslationRun,
on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None,
skip_insert: bool = False,
full_translation: bool = False,
) -> TranslationRun:
with belief_scope("TranslationOrchestrator.execute_run"):
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
@@ -212,7 +210,7 @@ class TranslationOrchestrator:
on_batch_progress=on_batch_progress,
)
try:
run = executor.execute_run(run, llm_progress_callback=None)
run = executor.execute_run(run, llm_progress_callback=None, full_translation=full_translation)
except Exception as e:
logger.explore("Translation execution failed", {
"run_id": run.id,
@@ -299,13 +297,13 @@ class TranslationOrchestrator:
"insert_status": run.insert_status,
})
return run
# [/DEF:execute_run:Function]
# #endregion execute_run
# [DEF:_generate_and_insert_sql:Function]
# @PURPOSE: Generate INSERT SQL from successful records and submit to Superset SQL Lab.
# @PRE: job has target table configured. run has successful records.
# @POST: SQL is generated and submitted. Returns execution result.
# @SIDE_EFFECT: Superset API call.
# #region _generate_and_insert_sql [C:5] [TYPE Function] [SEMANTICS translate,sql,insert]
# @BRIEF Generate INSERT SQL from successful records and submit to Superset SQL Lab.
# @PRE job has target table configured. run has successful records.
# @POST SQL is generated and submitted. Returns execution result.
# @SIDE_EFFECT Superset API call; event logging.
def _generate_and_insert_sql(
self,
job: TranslationJob,
@@ -333,11 +331,12 @@ class TranslationOrchestrator:
})
# Build rows for SQL generation
columns = job.context_columns or []
if job.translation_column and job.translation_column not in columns:
# Only include the translation column — context columns are for LLM only
columns = []
if job.translation_column:
columns.append(job.translation_column)
# Also include key columns if used for upsert
# Include key columns if present (for UPSERT matching)
if job.target_key_cols:
for k in job.target_key_cols:
if k not in columns:
@@ -346,14 +345,13 @@ class TranslationOrchestrator:
rows_for_sql = []
for rec in records:
row_data = {}
if job.context_columns:
for col in job.context_columns:
row_data[col] = ""
if job.translation_column:
row_data[job.translation_column] = rec.target_sql or ""
if job.target_key_cols:
source_data = rec.source_data or {}
for k in job.target_key_cols:
row_data[k] = ""
# Use source_data value if available, fall back to empty string
row_data[k] = source_data.get(k, "")
rows_for_sql.append(row_data)
if not columns:
@@ -377,6 +375,16 @@ class TranslationOrchestrator:
return {"status": "failed", "error_message": str(e), "query_id": None}
# Log insert phase start
logger.reason("Insert SQL generated", {
"run_id": run.id,
"sql_preview": sql[:500],
"sql_length": len(sql),
"row_count": row_count,
"dialect": job.database_dialect or job.target_dialect or "postgresql",
"columns": columns,
"has_key_cols": bool(job.target_key_cols),
})
self.event_log.log_event(
job_id=job.id,
run_id=run.id,
@@ -388,7 +396,18 @@ class TranslationOrchestrator:
# Submit to Superset
try:
env_id = job.environment_id or job.source_dialect or ""
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
# Resolve database_id: explicit target_database_id from job (int or UUID), or resolve from environment
target_db_id = None
if job.target_database_id:
try:
target_db_id = int(job.target_database_id)
except (ValueError, TypeError):
# Could be a UUID — pass as-is, executor will resolve it
target_db_id = job.target_database_id
logger.reason("target_database_id is not an integer, will resolve as UUID", {
"target_database_id": job.target_database_id,
})
executor = SupersetSqlLabExecutor(self.config_manager, env_id, database_id=target_db_id)
result = executor.execute_and_poll(
sql=sql,
max_polls=30,
@@ -411,13 +430,13 @@ class TranslationOrchestrator:
)
return result
# [/DEF:_generate_and_insert_sql:Function]
# #endregion _generate_and_insert_sql
# [DEF:_validate_preconditions:Function]
# @PURPOSE: Validate preconditions before starting a run.
# @PRE: None.
# @POST: Raises ValueError if preconditions are not met.
# @SIDE_EFFECT: None.
# #region _validate_preconditions [C:5] [TYPE Function] [SEMANTICS translate,validation]
# @BRIEF Validate preconditions before starting a translation run.
# @PRE None.
# @POST Raises ValueError if preconditions are not met.
# @SIDE_EFFECT None.
def _validate_preconditions(
self,
job: TranslationJob,
@@ -473,13 +492,13 @@ class TranslationOrchestrator:
"job_id": job.id,
"is_scheduled": is_scheduled,
})
# [/DEF:_validate_preconditions:Function]
# #endregion _validate_preconditions
# [DEF:retry_failed_batches:Function]
# @PURPOSE: Retry failed batches in a run.
# @PRE: run exists and has failed batches.
# @POST: Failed batches are re-processed.
# @SIDE_EFFECT: LLM calls, DB writes.
# #region retry_failed_batches [C:5] [TYPE Function] [SEMANTICS translate,retry,batch]
# @BRIEF Retry failed batches in a run by re-processing failed records.
# @PRE run exists and has failed batches.
# @POST Failed batches are re-processed; run status and stats updated.
# @SIDE_EFFECT LLM calls; DB writes; event logging.
def retry_failed_batches(
self,
run_id: str,
@@ -588,13 +607,13 @@ class TranslationOrchestrator:
"status": run.status,
})
return run
# [/DEF:retry_failed_batches:Function]
# #endregion retry_failed_batches
# [DEF:retry_insert:Function]
# @PURPOSE: Retry the SQL insert phase for a completed run.
# @PRE: run exists and has successful records.
# @POST: SQL is regenerated and re-submitted to Superset.
# @SIDE_EFFECT: Superset API call.
# #region retry_insert [C:5] [TYPE Function] [SEMANTICS translate,retry,insert]
# @BRIEF Retry the SQL insert phase for a completed run.
# @PRE run exists and has successful records.
# @POST SQL is regenerated and re-submitted to Superset.
# @SIDE_EFFECT Superset API call; event logging.
def retry_insert(
self,
run_id: str,
@@ -638,13 +657,13 @@ class TranslationOrchestrator:
"insert_status": run.insert_status,
})
return run
# [/DEF:retry_insert:Function]
# #endregion retry_insert
# [DEF:cancel_run:Function]
# @PURPOSE: Cancel a running translation.
# @PRE: run is in PENDING or RUNNING status.
# @POST: Run status is set to CANCELLED.
# @SIDE_EFFECT: DB write; records event.
# #region cancel_run [C:5] [TYPE Function] [SEMANTICS translate,run,cancel]
# @BRIEF Cancel a running translation run.
# @PRE run is in PENDING or RUNNING status.
# @POST Run status is set to CANCELLED; event recorded.
# @SIDE_EFFECT DB write; records event.
def cancel_run(self, run_id: str) -> TranslationRun:
with belief_scope("TranslationOrchestrator.cancel_run"):
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
@@ -661,7 +680,8 @@ class TranslationOrchestrator:
run.completed_at = datetime.now(timezone.utc)
self.db.flush()
self.event_log.log_event(
# Record terminal event directly — we are the source of the terminal event
self.event_log._create_event_raw(
job_id=run.job_id,
run_id=run.id,
event_type="RUN_CANCELLED",
@@ -676,12 +696,12 @@ class TranslationOrchestrator:
"run_id": run_id,
})
return run
# [/DEF:cancel_run:Function]
# #endregion cancel_run
# [DEF:get_run_status:Function]
# @PURPOSE: Get run status with statistics.
# @PRE: run_id exists.
# @POST: Returns dict with run details.
# #region get_run_status [C:5] [TYPE Function] [SEMANTICS translate,run,status]
# @BRIEF Get run status with statistics and event invariant checks.
# @PRE run_id exists.
# @POST Returns dict with run details.
def get_run_status(self, run_id: str) -> Dict[str, Any]:
with belief_scope("TranslationOrchestrator.get_run_status"):
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
@@ -695,8 +715,8 @@ class TranslationOrchestrator:
.count()
)
# Get event summary
event_summary = self.event_log.get_run_event_summary(run_id)
# Get event invariants (lightweight — only fetches event_type column)
invariants = self.event_log.get_run_event_invariants_lightweight(run_id)
return {
"id": run.id,
@@ -712,20 +732,16 @@ class TranslationOrchestrator:
"insert_status": run.insert_status,
"superset_execution_id": run.superset_execution_id,
"batch_count": batch_count,
"event_invariants": {
"has_run_started": event_summary["has_run_started"],
"terminal_event_count": event_summary["terminal_event_count"],
"invariant_valid": event_summary["invariant_valid"],
},
"event_invariants": invariants,
"created_by": run.created_by,
"created_at": run.created_at.isoformat() if run.created_at else None,
}
# [/DEF:get_run_status:Function]
# #endregion get_run_status
# [DEF:get_run_records:Function]
# @PURPOSE: Get paginated records for a run.
# @PRE: run_id exists.
# @POST: Returns dict with records and pagination info.
# #region get_run_records [C:5] [TYPE Function] [SEMANTICS translate,run,records]
# @BRIEF Get paginated records for a run with optional status filter.
# @PRE run_id exists.
# @POST Returns dict with records and pagination info.
def get_run_records(
self,
run_id: str,
@@ -771,12 +787,12 @@ class TranslationOrchestrator:
"page_size": page_size,
"status_filter": status_filter,
}
# [/DEF:get_run_records:Function]
# #endregion get_run_records
# [DEF:get_run_history:Function]
# @PURPOSE: Get run history for a job.
# @PRE: job_id exists.
# @POST: Returns list of runs.
# #region get_run_history [C:5] [TYPE Function] [SEMANTICS translate,run,history]
# @BRIEF Get paginated run history for a job.
# @PRE job_id exists.
# @POST Returns tuple of (total_count, list_of_runs).
def get_run_history(
self,
job_id: str,
@@ -814,10 +830,10 @@ class TranslationOrchestrator:
}
for r in runs
]
# [/DEF:get_run_history:Function]
# #endregion get_run_history
# [DEF:_compute_config_hash:Function]
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
# #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison.
@staticmethod
def _compute_config_hash(job: TranslationJob) -> str:
import hashlib
@@ -833,10 +849,10 @@ class TranslationOrchestrator:
"upsert_strategy": job.upsert_strategy,
}, sort_keys=True)
return hashlib.sha256(config_str.encode()).hexdigest()[:16]
# [/DEF:_compute_config_hash:Function]
# #endregion _compute_config_hash
# [DEF:_compute_dict_snapshot_hash:Function]
# @PURPOSE: Compute a hash of dictionary state for snapshot comparison.
# #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @BRIEF Compute a SHA-256 hash of dictionary state for snapshot comparison.
def _compute_dict_snapshot_hash(self, job_id: str) -> str:
import hashlib
from ...models.translate import TranslationJobDictionary
@@ -848,8 +864,8 @@ class TranslationOrchestrator:
dict_ids = sorted([dl.dictionary_id for dl in dict_links])
hash_input = ",".join(dict_ids)
return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
# [/DEF:_compute_dict_snapshot_hash:Function]
# #endregion _compute_dict_snapshot_hash
# [/DEF:TranslationOrchestrator:Class]
# [/DEF:TranslationOrchestrator:Module]
# #endregion TranslationOrchestrator
# #endregion TranslationOrchestrator

22
backend/src/plugins/translate/plugin.py
View File

@@ -1,19 +1,15 @@
# [DEF:TranslatePlugin:Module]
# @COMPLEXITY: 2
# @SEMANTICS: plugin, translate, llm, sql, dashboard
# @PURPOSE: TranslatePlugin skeleton for cross-dialect SQL and dashboard translation.
# @LAYER: Domain
# @RELATION: INHERITS -> [PluginBase]
# @RATIONALE: Separate plugin avoids bloating LLMAnalysisPlugin beyond fractal limit (<400 lines).
# @REJECTED: Extending LLMAnalysisPlugin would conflate two distinct feature domains.
# #region TranslatePlugin [C:3] [TYPE Module] [SEMANTICS plugin,translate,llm,sql,dashboard]
# @BRIEF TranslatePlugin skeleton for cross-dialect SQL and dashboard translation.
# @LAYER Domain
# @RELATION INHERITS -> [PluginBase]
from typing import Dict, Any, Optional
from ...core.plugin_base import PluginBase
# [DEF:TranslatePlugin:Class]
# @PURPOSE: Plugin for translating SQL queries and dashboard definitions across database dialects.
# @RELATION: IMPLEMENTS -> backend.src.core.plugin_base.PluginBase
# #region TranslatePlugin [C:3] [TYPE Class] [SEMANTICS plugin,translate]
# @BRIEF Plugin for translating SQL queries and dashboard definitions across database dialects.
# @RELATION IMPLEMENTS -> [PluginBase]
class TranslatePlugin(PluginBase):
@property
def id(self) -> str:
@@ -57,6 +53,6 @@ class TranslatePlugin(PluginBase):
async def execute(self, params: Dict[str, Any], context: Optional[Any] = None):
"""Execute a translation job — implementation deferred to later phases."""
raise NotImplementedError("TranslatePlugin.execute not yet implemented")
# [/DEF:TranslatePlugin:Class]
# #endregion TranslatePlugin
# [/DEF:TranslatePlugin:Module]
# #endregion TranslatePlugin

209
backend/src/plugins/translate/preview.py
View File

@@ -1,21 +1,21 @@
# [DEF:TranslationPreview:Module]
# @COMPLEXITY: 4
# @SEMANTICS: translate, preview, llm, session
# @PURPOSE: Preview session management: fetch sample rows from Superset, send to LLM with context + filtered dictionary, return side-by-side results.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [TranslationJob:Class]
# @RELATION: DEPENDS_ON -> [TranslationPreviewSession:Class]
# @RELATION: DEPENDS_ON -> [TranslationPreviewRecord:Class]
# @RELATION: DEPENDS_ON -> [LLMProviderService]
# @RELATION: DEPENDS_ON -> [DictionaryManager]
# @RELATION: DEPENDS_ON -> [SupersetClient]
# @RELATION: DEPENDS_ON -> [render_prompt]
# @RELATION: DEPENDS_ON -> [ConfigManager]
# @PRE: Database session and config manager are available.
# @POST: Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected.
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows.
# @RATIONALE: C4 because preview is stateful with approve/edit/reject lifecycle and LLM API calls.
# @REJECTED: Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably.
# #region TranslationPreview [C:5] [TYPE Module] [SEMANTICS translate,preview,llm,session]
# @BRIEF Preview session management: fetch sample rows from Superset, send to LLM with context + filtered dictionary, return side-by-side results.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION DEPENDS_ON -> [TranslationPreviewSession]
# @RELATION DEPENDS_ON -> [TranslationPreviewRecord]
# @RELATION DEPENDS_ON -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [DictionaryManager]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [render_prompt]
# @RELATION DEPENDS_ON -> [ConfigManager]
# @PRE Database session and config manager are available.
# @POST Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected.
# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates DB rows.
# @DATA_CONTRACT Input[job_id, sample_size] -> Output[Dict with session, records, cost_estimate]
# @INVARIANT Preview sessions expire after 24 hours; exactly one ACTIVE session per job at a time.
# @RATIONALE C5 because preview is stateful with approve/edit/reject lifecycle and LLM API calls requiring structured error handling.
# @REJECTED Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably.
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.orm import Session
@@ -38,8 +38,7 @@ from ...services.llm_provider import LLMProviderService
from ...services.llm_prompt_templates import render_prompt
from .dictionary import DictionaryManager
# [DEF:DEFAULT_EXECUTION_PROMPT_TEMPLATE:Constant]
# @PURPOSE: Default prompt template for batch LLM translation execution (no context columns — faster).
# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [C:1] [TYPE Constant] [SEMANTICS translate,prompt]
DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
"Translate the following database content from {source_language} to {target_language}.\n\n"
"Source dialect: {source_dialect}\n"
@@ -52,11 +51,10 @@ DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
"Each row_id must match the index provided. Return exactly {row_count} entries."
)
# [/DEF:DEFAULT_EXECUTION_PROMPT_TEMPLATE:Constant]
# #endregion DEFAULT_EXECUTION_PROMPT_TEMPLATE
# [DEF:DEFAULT_PREVIEW_PROMPT_TEMPLATE:Constant]
# @PURPOSE: Default prompt template for LLM translation preview.
# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [C:1] [TYPE Constant] [SEMANTICS translate,prompt]
DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
"Translate the following database content from {source_language} to {target_language}.\n\n"
"Source dialect: {source_dialect}\n"
@@ -70,13 +68,11 @@ DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
"Each row_id must match the index provided. Return exactly {row_count} entries."
)
# [/DEF:DEFAULT_PREVIEW_PROMPT_TEMPLATE:Constant]
# #endregion DEFAULT_PREVIEW_PROMPT_TEMPLATE
# [DEF:TokenEstimator:Class]
# @PURPOSE: Estimate token counts and costs for LLM translation operations.
# @RATIONALE: Token estimation uses a heuristic (chars/token ratio) since exact tokenization depends on the LLM model.
# @REJECTED: Using an external tokenizer library would introduce a heavy dependency for estimation only.
# #region TokenEstimator [C:2] [TYPE Class] [SEMANTICS translate,tokens,estimate]
# @BRIEF Estimate token counts and costs for LLM translation operations using heuristic chars/token ratio.
class TokenEstimator:
"""Estimate token counts and costs for LLM operations."""
@@ -84,46 +80,38 @@ class TokenEstimator:
OUTPUT_TOKENS_PER_ROW_ESTIMATE: int = 50
TOKEN_COST_PER_1K: float = 0.002 # Default cost per 1K tokens
# [DEF:estimate_prompt_tokens:Function]
# @PURPOSE: Estimate token count for a prompt string.
# @PRE: prompt is a non-empty string.
# @POST: Returns estimated token count (integer).
# #region estimate_prompt_tokens [C:1] [TYPE Function] [SEMANTICS translate,tokens]
@staticmethod
def estimate_prompt_tokens(prompt: str) -> int:
if not prompt:
return 0
return max(1, int(len(prompt) / TokenEstimator.CHARS_PER_TOKEN_ESTIMATE))
# [/DEF:estimate_prompt_tokens:Function]
# #endregion estimate_prompt_tokens
# [DEF:estimate_output_tokens:Function]
# @PURPOSE: Estimate output token count for translating N rows.
# @PRE: row_count >= 0.
# @POST: Returns estimated output token count.
# #region estimate_output_tokens [C:1] [TYPE Function] [SEMANTICS translate,tokens]
@staticmethod
def estimate_output_tokens(row_count: int) -> int:
return row_count * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE
# [/DEF:estimate_output_tokens:Function]
# #endregion estimate_output_tokens
# [DEF:estimate_cost:Function]
# @PURPOSE: Estimate cost for a given number of tokens.
# @PRE: total_tokens >= 0.
# @POST: Returns estimated cost in USD.
# #region estimate_cost [C:1] [TYPE Function] [SEMANTICS translate,cost]
@staticmethod
def estimate_cost(total_tokens: int, cost_per_1k: Optional[float] = None) -> float:
rate = cost_per_1k if cost_per_1k is not None else TokenEstimator.TOKEN_COST_PER_1K
return round((total_tokens / 1000) * rate, 6)
# [/DEF:estimate_cost:Function]
# #endregion estimate_cost
# [/DEF:TokenEstimator:Class]
# #endregion TokenEstimator
# [DEF:TranslationPreview:Class]
# @COMPLEXITY: 4
# @PURPOSE: Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate.
# @PRE: Database session and config manager are available.
# @POST: Preview sessions created with persisted records; full execution gates on accepted session.
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows.
# #region TranslationPreview [C:5] [TYPE Class] [SEMANTICS translate,preview,lifecycle]
# @BRIEF Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate.
# @PRE Database session and config manager are available.
# @POST Preview sessions created with persisted records; full execution gates on accepted session.
# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates DB rows.
# @DATA_CONTRACT Input[job_id, sample_size] -> Output[Dict with session, records, cost_estimate]
# @INVARIANT Preview sessions expire after 24 hours; exactly one ACTIVE session per job at a time.
class TranslationPreview:
def __init__(
@@ -136,12 +124,11 @@ class TranslationPreview:
self.config_manager = config_manager
self.current_user = current_user
# [DEF:preview_rows:Function]
# @COMPLEXITY: 4
# @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.
# @PRE: job_id exists and job has source_datasource_id, translation_column configured.
# @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session.
# @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.
# #region preview_rows [C:4] [TYPE Function] [SEMANTICS translate,preview,rows]
# @BRIEF Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.
# @PRE job_id exists and job has source_datasource_id, translation_column configured.
# @POST Returns TranslationPreviewResponse with records, cost estimation, and persistent session.
# @SIDE_EFFECT Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.
def preview_rows(
self,
job_id: str,
@@ -303,6 +290,7 @@ class TranslationPreview:
source_object_name=f"Row {idx + 1}",
status=status,
feedback=feedback,
source_data=meta.get("context_data"),
created_at=datetime.now(timezone.utc),
)
self.db.add(record)
@@ -348,12 +336,13 @@ class TranslationPreview:
"sample_cost": sample_cost,
})
return result
# [/DEF:preview_rows:Function]
# #endregion preview_rows
# [DEF:update_preview_row:Function]
# @PURPOSE: Approve, edit, or reject an individual preview row.
# @PRE: session_id and row_id exist, session is ACTIVE.
# @POST: PreviewRecord status is updated.
# #region update_preview_row [C:4] [TYPE Function] [SEMANTICS translate,preview,row,update]
# @BRIEF Approve, edit, or reject an individual preview row.
# @PRE session_id and row_id exist, session is ACTIVE.
# @POST PreviewRecord status is updated (APPROVED, REJECTED, or APPROVED with edited translation).
# @SIDE_EFFECT DB write.
def update_preview_row(
self,
job_id: str,
@@ -417,13 +406,13 @@ class TranslationPreview:
"status": record.status,
"feedback": record.feedback,
}
# [/DEF:update_preview_row:Function]
# #endregion update_preview_row
# [DEF:accept_preview_session:Function]
# @PURPOSE: Mark a preview session as accepted, which gates full execution.
# @PRE: job_id has an ACTIVE preview session.
# @POST: Session status changes to APPLIED.
# @SIDE_EFFECT: Future full execution calls will check for accepted session.
# #region accept_preview_session [C:4] [TYPE Function] [SEMANTICS translate,preview,accept]
# @BRIEF Mark a preview session as accepted (APPLIED), which gates full execution.
# @PRE job_id has an ACTIVE preview session.
# @POST Session status changes to APPLIED; future full execution calls check for accepted session.
# @SIDE_EFFECT DB write.
def accept_preview_session(self, job_id: str) -> Dict[str, Any]:
with belief_scope("TranslationPreview.accept_preview_session"):
session = (
@@ -472,12 +461,11 @@ class TranslationPreview:
for r in records
],
}
# [/DEF:accept_preview_session:Function]
# #endregion accept_preview_session
# [DEF:get_preview_session:Function]
# @PURPOSE: Get the latest preview session for a job with its records.
# @PRE: job_id exists.
# @POST: Returns session data with records or raises ValueError.
# #region get_preview_session [C:3] [TYPE Function] [SEMANTICS translate,preview,get]
# @BRIEF Get the latest preview session for a job with its records.
# @RELATION DEPENDS_ON -> [TranslationPreviewSession]
def get_preview_session(self, job_id: str) -> Dict[str, Any]:
with belief_scope("TranslationPreview.get_preview_session"):
session = (
@@ -516,14 +504,13 @@ class TranslationPreview:
for r in records
],
}
# [/DEF:get_preview_session:Function]
# #endregion get_preview_session
# [DEF:_fetch_sample_rows:Function]
# @COMPLEXITY: 4
# @PURPOSE: Fetch sample rows from the Superset dataset for preview.
# @PRE: job has source_datasource_id and translation_column.
# @POST: Returns list of dicts with row data.
# @SIDE_EFFECT: Calls Superset chart data endpoint.
# #region _fetch_sample_rows [C:4] [TYPE Function] [SEMANTICS translate,superset,sample]
# @BRIEF Fetch sample rows from the Superset dataset for preview.
# @PRE job has source_datasource_id and translation_column.
# @POST Returns list of dicts with row data from Superset chart data API.
# @SIDE_EFFECT Calls Superset chart data endpoint with pagination.
def _fetch_sample_rows(self, job: TranslationJob, sample_size: int = 10, env_id: Optional[str] = None) -> List[Dict[str, Any]]:
with belief_scope("TranslationPreview._fetch_sample_rows"):
# Determine environment: prefer explicit env_id, then job.environment_id, then job.source_dialect (legacy)
@@ -602,12 +589,11 @@ class TranslationPreview:
)
return rows
# [/DEF:_fetch_sample_rows:Function]
# #endregion _fetch_sample_rows
# [DEF:_extract_data_rows:Function]
# @PURPOSE: Extract data rows from Superset chart data response.
# @PRE: response is a dict from Superset API.
# @POST: Returns list of row dicts.
# #region _extract_data_rows [C:3] [TYPE Function] [SEMANTICS translate,superset,data]
# @BRIEF Extract data rows from Superset chart data response, trying multiple response formats.
# @RELATION DEPENDS_ON -> [SupersetClient]
@staticmethod
def _extract_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]:
with belief_scope("TranslationPreview._extract_data_rows"):
@@ -636,14 +622,13 @@ class TranslationPreview:
return result
return []
# [/DEF:_extract_data_rows:Function]
# #endregion _extract_data_rows
# [DEF:_call_llm:Function]
# @COMPLEXITY: 4
# @PURPOSE: Call the configured LLM provider with the preview prompt.
# @PRE: job has a valid provider_id.
# @POST: Returns raw LLM response string.
# @SIDE_EFFECT: Makes HTTP call to LLM provider.
# #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call]
# @BRIEF Call the configured LLM provider with the preview prompt.
# @PRE job has a valid provider_id.
# @POST Returns raw LLM response string.
# @SIDE_EFFECT Makes HTTP call to LLM provider.
def _call_llm(self, job: TranslationJob, prompt: str) -> str:
with belief_scope("TranslationPreview._call_llm"):
if not job.provider_id:
@@ -679,13 +664,13 @@ class TranslationPreview:
"response_length": len(response_text),
})
return response_text
# [/DEF:_call_llm:Function]
# #endregion _call_llm
# [DEF:_call_openai_compatible:Function]
# @PURPOSE: Call an OpenAI-compatible API for translation.
# @PRE: base_url, api_key, model, prompt are valid.
# @POST: Returns response text.
# @SIDE_EFFECT: Makes HTTP POST to LLM API.
# #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai]
# @BRIEF Call an OpenAI-compatible API for translation with structured output support.
# @PRE base_url, api_key, model, prompt are valid.
# @POST Returns response text.
# @SIDE_EFFECT Makes HTTP POST to LLM API.
@staticmethod
def _call_openai_compatible(
base_url: str,
@@ -741,12 +726,13 @@ class TranslationPreview:
raise ValueError("LLM returned empty content")
return content
# [/DEF:_call_openai_compatible:Function]
# #endregion _call_openai_compatible
# [DEF:_parse_llm_response:Function]
# @PURPOSE: Parse the LLM JSON response into a dict of row_id -> translation.
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
# @POST: Returns dict mapping string row_id to translation text.
# #region _parse_llm_response [C:4] [TYPE Function] [SEMANTICS translate,llm,parse]
# @BRIEF Parse the LLM JSON response into a dict of row_id -> translation, with markdown code block fallback.
# @PRE response_text is valid JSON with {"rows": [...]} structure.
# @POST Returns dict mapping string row_id to translation text; warns if fewer translations than expected.
# @SIDE_EFFECT Logs warnings for short responses.
@staticmethod
def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:
with belief_scope("TranslationPreview._parse_llm_response"):
@@ -785,10 +771,10 @@ class TranslationPreview:
)
return translations
# [/DEF:_parse_llm_response:Function]
# #endregion _parse_llm_response
# [DEF:_compute_config_hash:Function]
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
# #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison.
@staticmethod
def _compute_config_hash(job: TranslationJob) -> str:
config_str = json.dumps({
@@ -803,10 +789,10 @@ class TranslationPreview:
"upsert_strategy": job.upsert_strategy,
}, sort_keys=True)
return hashlib.sha256(config_str.encode()).hexdigest()[:16]
# [/DEF:_compute_config_hash:Function]
# #endregion _compute_config_hash
# [DEF:_compute_dict_snapshot_hash:Function]
# @PURPOSE: Compute a hash of the dictionary state for snapshot comparison.
# #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @BRIEF Compute a SHA-256 hash of the dictionary state for snapshot comparison.
def _compute_dict_snapshot_hash(self, job_id: str) -> str:
dict_links = (
self.db.query(TranslationJobDictionary)
@@ -816,9 +802,8 @@ class TranslationPreview:
dict_ids = sorted([dl.dictionary_id for dl in dict_links])
hash_input = ",".join(dict_ids)
return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
# [/DEF:_compute_dict_snapshot_hash:Function]
# #endregion _compute_dict_snapshot_hash
# [/DEF:TranslationPreview:Class]
# [/DEF:TranslationPreview:Module]
# #endregion TranslationPreview
# #endregion TranslationPreview

128
backend/src/plugins/translate/scheduler.py
View File

@@ -1,18 +1,18 @@
# [DEF:TranslationScheduler:Module]
# @COMPLEXITY: 4
# @SEMANTICS: translate, scheduler, apscheduler, cron
# @PURPOSE: Manage TranslationSchedule rows and register them with core SchedulerService.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION: DEPENDS_ON -> [SchedulerService:Class]
# @RELATION: DEPENDS_ON -> [TranslationOrchestrator:Class]
# @RELATION: DEPENDS_ON -> [TranslationEventLog:Class]
# @PRE: Database session and SchedulerService are available.
# @POST: TranslationSchedule CRUD persisted; APScheduler jobs registered/updated/removed.
# @SIDE_EFFECT: Registers APScheduler jobs; runs translations on trigger; creates events.
# @RATIONALE: Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance.
# @REJECTED: Separate scheduler instance would create resource contention.
# @REJECTED: Polling-based approach — event-driven APScheduler is more precise.
# #region TranslationScheduler [C:5] [TYPE Module] [SEMANTICS translate,scheduler,apscheduler,cron]
# @BRIEF Manage TranslationSchedule rows and register them with core SchedulerService for cron-based execution.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationSchedule]
# @RELATION DEPENDS_ON -> [SchedulerService]
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
# @RELATION DEPENDS_ON -> [TranslationEventLog]
# @PRE Database session and SchedulerService are available.
# @POST TranslationSchedule CRUD persisted; translation runs triggered on schedule.
# @SIDE_EFFECT Registers APScheduler jobs; runs translations on trigger; creates events.
# @DATA_CONTRACT Input[job_id, cron_expression] -> Output[TranslationSchedule]
# @INVARIANT Concurrency guard: max 1 pending/running run per job at scheduled trigger time.
# @RATIONALE Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance.
# @REJECTED Separate scheduler instance would create resource contention.
# @REJECTED Polling-based approach — event-driven APScheduler is more precise.
import uuid
from datetime import datetime, timezone, timedelta
@@ -26,9 +26,13 @@ from ...models.translate import TranslationSchedule, TranslationJob, Translation
from .events import TranslationEventLog
# [DEF:TranslationScheduler:Class]
# @COMPLEXITY: 4
# @PURPOSE: CRUD for TranslationSchedule rows + APScheduler registration wrappers.
# #region TranslationScheduler [C:5] [TYPE Class] [SEMANTICS translate,scheduler,crud]
# @BRIEF CRUD for TranslationSchedule rows + APScheduler registration wrappers.
# @PRE Database session and ConfigManager are available.
# @POST Schedule rows created/updated/deleted with event logging.
# @SIDE_EFFECT Writes TranslationSchedule rows; logs SCHEDULE_CREATED/UPDATED/DELETED events.
# @DATA_CONTRACT Input[job_id, cron_expression, timezone] -> Output[TranslationSchedule]
# @INVARIANT Exactly one schedule per job (upsert pattern).
class TranslationScheduler:
def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None):
@@ -37,10 +41,11 @@ class TranslationScheduler:
self.current_user = current_user
self.event_log = TranslationEventLog(db)
# [DEF:create_schedule:Function]
# @PURPOSE: Create a new schedule for a job.
# @PRE: job_id exists. cron_expression is valid.
# @POST: TranslationSchedule row created.
# #region create_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,create]
# @BRIEF Create a new schedule for a job with cron expression and event logging.
# @PRE job_id exists. cron_expression is valid.
# @POST TranslationSchedule row created; SCHEDULE_CREATED event logged.
# @SIDE_EFFECT DB write; event logging.
def create_schedule(
self,
job_id: str,
@@ -79,12 +84,13 @@ class TranslationScheduler:
logger.reflect("Schedule created", {"schedule_id": schedule.id, "job_id": job_id})
return schedule
# [/DEF:create_schedule:Function]
# #endregion create_schedule
# [DEF:update_schedule:Function]
# @PURPOSE: Update an existing schedule.
# @PRE: job_id has an existing schedule.
# @POST: Schedule updated.
# #region update_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,update]
# @BRIEF Update an existing schedule's cron expression, timezone, or active state.
# @PRE job_id has an existing schedule.
# @POST Schedule updated; SCHEDULE_UPDATED event logged.
# @SIDE_EFFECT DB write; event logging.
def update_schedule(
self,
job_id: str,
@@ -123,12 +129,13 @@ class TranslationScheduler:
logger.reflect("Schedule updated", {"schedule_id": schedule.id, "job_id": job_id})
return schedule
# [/DEF:update_schedule:Function]
# #endregion update_schedule
# [DEF:delete_schedule:Function]
# @PURPOSE: Delete a schedule for a job.
# @PRE: job_id has an existing schedule.
# @POST: Schedule deleted.
# #region delete_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,delete]
# @BRIEF Delete a schedule for a job with event logging.
# @PRE job_id has an existing schedule.
# @POST Schedule deleted; SCHEDULE_DELETED event logged.
# @SIDE_EFFECT DB write; event logging.
def delete_schedule(self, job_id: str) -> None:
with belief_scope("TranslationScheduler.delete_schedule"):
schedule = self.db.query(TranslationSchedule).filter(
@@ -149,12 +156,13 @@ class TranslationScheduler:
)
logger.reflect("Schedule deleted", {"schedule_id": schedule_id, "job_id": job_id})
# [/DEF:delete_schedule:Function]
# #endregion delete_schedule
# [DEF:enable_disable_schedule:Function]
# @PURPOSE: Enable or disable a schedule.
# @PRE: job_id has an existing schedule.
# @POST: Schedule is_active updated.
# #region set_schedule_active [C:4] [TYPE Function] [SEMANTICS translate,schedule,activate]
# @BRIEF Enable or disable a schedule by setting is_active flag.
# @PRE job_id has an existing schedule.
# @POST Schedule is_active updated.
# @SIDE_EFFECT DB write.
def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule:
with belief_scope("TranslationScheduler.set_schedule_active"):
schedule = self.db.query(TranslationSchedule).filter(
@@ -174,12 +182,10 @@ class TranslationScheduler:
"is_active": is_active,
})
return schedule
# [/DEF:enable_disable_schedule:Function]
# #endregion set_schedule_active
# [DEF:get_schedule:Function]
# @PURPOSE: Get schedule for a job.
# @PRE: job_id exists.
# @POST: Returns TranslationSchedule or raises ValueError.
# #region get_schedule [C:2] [TYPE Function] [SEMANTICS translate,schedule,get]
# @BRIEF Get the schedule for a job by job_id.
def get_schedule(self, job_id: str) -> TranslationSchedule:
with belief_scope("TranslationScheduler.get_schedule"):
schedule = self.db.query(TranslationSchedule).filter(
@@ -188,11 +194,10 @@ class TranslationScheduler:
if not schedule:
raise ValueError(f"No schedule found for job '{job_id}'")
return schedule
# [/DEF:get_schedule:Function]
# #endregion get_schedule
# [DEF:list_active_schedules:Function]
# @PURPOSE: List all active schedules.
# @POST: Returns list of active TranslationSchedule rows.
# #region list_active_schedules [C:2] [TYPE Function] [SEMANTICS translate,schedule,list]
# @BRIEF List all active schedules.
@staticmethod
def list_active_schedules(db: Session) -> List[TranslationSchedule]:
return (
@@ -200,13 +205,10 @@ class TranslationScheduler:
.filter(TranslationSchedule.is_active == True)
.all()
)
# [/DEF:list_active_schedules:Function]
# #endregion list_active_schedules
# [DEF:get_next_executions:Function]
# @PURPOSE: Compute next N execution times from cron expression.
# @PRE: cron_expression is valid.
# @POST: Returns list of ISO datetime strings.
@staticmethod
# #region get_next_executions [C:2] [TYPE Function] [SEMANTICS translate,schedule,next]
# @BRIEF Compute next N execution times from a cron expression using APScheduler's CronTrigger.
def get_next_executions(cron_expression: str, timezone_str: str = "UTC", n: int = 3) -> List[str]:
from zoneinfo import ZoneInfo
from apscheduler.triggers.cron import CronTrigger
@@ -231,18 +233,20 @@ class TranslationScheduler:
prev = ft
next_time = ft
return results
# [/DEF:get_next_executions:Function]
# #endregion get_next_executions
# [/DEF:TranslationScheduler:Class]
# #endregion TranslationScheduler
# [DEF:execute_scheduled_translation:Function]
# @PURPOSE: APScheduler job handler — runs scheduled translation with concurrency check and new-key-only fallback.
# @PRE: schedule_id is valid.
# @POST: Translation run created and executed if no concurrent run exists.
# @SIDE_EFFECT: DB writes; LLM calls; Superset API calls.
# @RATIONALE: New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.
# #region execute_scheduled_translation [C:5] [TYPE Function] [SEMANTICS translate,schedule,execute]
# @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and baseline expiry detection.
# @PRE schedule_id is valid and job exists.
# @POST Translation run created and executed if no concurrent run exists.
# @SIDE_EFFECT DB writes; LLM calls; Superset API calls; event logging.
# @DATA_CONTRACT Input[schedule_id, job_id, db_session_maker, config_manager] -> Output[None]
# @INVARIANT Concurrency guard: max 1 pending/running run per job at trigger time.
# @RATIONALE New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.
def execute_scheduled_translation(
schedule_id: str,
job_id: str,
@@ -357,5 +361,5 @@ def execute_scheduled_translation(
logger.error(f"[scheduled_translation] Unexpected error: {e}")
finally:
db.close()
# [/DEF:execute_scheduled_translation:Function]
# [/DEF:TranslationScheduler:Module]
# #endregion execute_scheduled_translation
# #endregion TranslationScheduler

219
backend/src/plugins/translate/service.py
View File

@@ -1,16 +1,16 @@
# [DEF:TranslateJobService:Module]
# @COMPLEXITY: 4
# @SEMANTICS: translate, service, crud, validation
# @PURPOSE: Service layer for translation job CRUD with datasource column validation and database dialect detection.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [TranslationJob]
# @RELATION: DEPENDS_ON -> [SupersetClient]
# @RELATION: DEPENDS_ON -> [ConfigManager]
# @PRE: Database session and config manager are available.
# @POST: Translation jobs are created/updated/deleted with column validation and dialect caching.
# @SIDE_EFFECT: Queries Superset for column metadata and database dialect at save time.
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity.
# #region TranslateJobService [C:5] [TYPE Module] [SEMANTICS translate,service,crud,validation]
# @BRIEF Service layer for translation job CRUD with datasource column validation and database dialect detection.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [ConfigManager]
# @PRE Database session and config manager are available.
# @POST Translation jobs are created/updated/deleted with column validation and dialect caching.
# @SIDE_EFFECT Queries Superset for column metadata and database dialect at save time.
# @DATA_CONTRACT Input[TranslateJobCreate/Update] -> Output[TranslationJob/TranslateJobResponse]
# @INVARIANT Snapshot isolation: in-progress runs use config snapshot; config edits affect future runs only.
# @RATIONALE Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity.
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.orm import Session
@@ -37,12 +37,16 @@ SUPPORTED_DIALECTS = {
}
# [DEF:get_dialect_from_database:Function]
# @PURPOSE: Extract normalized dialect string from a Superset database record.
# @PRE: database_record is a dict from Superset API.
# @POST: Returns normalized dialect string or raises ValueError if unsupported.
# #region get_dialect_from_database [C:4] [TYPE Function] [SEMANTICS translate,dialect,validation]
# @BRIEF Extract normalized dialect string from a Superset database record and validate it is supported.
# @PRE database_record is a dict from Superset API with 'backend' or 'engine' key.
# @POST Returns normalized dialect string or raises ValueError if unsupported.
# @SIDE_EFFECT None — pure data extraction.
def get_dialect_from_database(database_record: Dict[str, Any]) -> str:
"""Extract and validate dialect from Superset database record."""
logger.reason("Extracting dialect from database record", {
"has_backend": bool(database_record.get("backend") or database_record.get("engine")),
})
backend = (
database_record.get("backend")
or database_record.get("engine")
@@ -50,6 +54,9 @@ def get_dialect_from_database(database_record: Dict[str, Any]) -> str:
).lower().strip()
if not backend:
logger.explore("Could not determine database dialect", {
"record_keys": list(database_record.keys()),
})
raise ValueError("Could not determine database dialect from connection")
# Map Superset backend names to normalized dialect
@@ -75,28 +82,36 @@ def get_dialect_from_database(database_record: Dict[str, Any]) -> str:
normalized = dialect_map.get(backend, backend)
if normalized not in SUPPORTED_DIALECTS:
logger.explore("Unsupported dialect", {"backend": backend, "normalized": normalized})
raise ValueError(
f"Unsupported database dialect: '{backend}'. "
f"Supported dialects: {', '.join(sorted(SUPPORTED_DIALECTS))}"
)
logger.reflect("Dialect validated", {"dialect": normalized})
return normalized
# [/DEF:get_dialect_from_database:Function]
# #endregion get_dialect_from_database
# [DEF:fetch_datasource_metadata:Function]
# @PURPOSE: Fetch datasource columns and database dialect from Superset.
# @PRE: dataset_id is a valid Superset dataset ID and environment has valid credentials.
# @POST: Returns (columns_list, dialect_string) or raises on failure.
# #region fetch_datasource_metadata [C:4] [TYPE Function] [SEMANTICS translate,datasource,metadata]
# @BRIEF Fetch datasource columns and database dialect from Superset for a given dataset.
# @PRE dataset_id is a valid Superset dataset ID and environment has valid credentials.
# @POST Returns (columns_list, dialect_string) or raises on failure.
# @SIDE_EFFECT Queries Superset API for dataset detail and database info.
def fetch_datasource_metadata(
dataset_id: int,
env_id: str,
config_manager: ConfigManager,
) -> Tuple[List[Dict[str, Any]], str]:
"""Fetch column metadata and database dialect for a datasource from Superset."""
logger.reason("Fetching datasource metadata", {
"dataset_id": dataset_id,
"env_id": env_id,
})
# Find environment config
environments = config_manager.get_environments()
env_config = next((e for e in environments if e.id == env_id), None)
if not env_config:
logger.explore("Environment not found", {"env_id": env_id})
raise ValueError(f"Superset environment '{env_id}' not found in configuration")
# Create Superset client and fetch dataset detail
@@ -133,25 +148,29 @@ def fetch_datasource_metadata(
else:
raise ValueError("No database information available for this datasource")
except Exception as e:
logger.warning(f"[translate_service] Could not fetch database dialect: {e}")
logger.explore("Could not fetch database dialect", {"error": str(e)})
raise ValueError(f"Could not determine database dialect: {e}")
logger.reflect("Metadata fetched", {"columns": len(columns), "dialect": dialect})
return columns, dialect
# [/DEF:fetch_datasource_metadata:Function]
# #endregion fetch_datasource_metadata
# [DEF:detect_virtual_columns:Function]
# @PURPOSE: Identify virtual (calculated) columns from column metadata.
# @PRE: columns is a list of dicts with 'is_physical' key.
# @POST: Returns list of virtual column names.
# #region detect_virtual_columns [C:2] [TYPE Function] [SEMANTICS translate,columns]
# @BRIEF Identify virtual (calculated) columns from column metadata by filtering non-physical entries.
def detect_virtual_columns(columns: List[Dict[str, Any]]) -> List[str]:
"""Return names of columns that are virtual (not physical)."""
return [col["name"] for col in columns if not col.get("is_physical", True)]
# [/DEF:detect_virtual_columns:Function]
# #endregion detect_virtual_columns
# [DEF:TranslateJobService:Class]
# @PURPOSE: Service for translation job CRUD with validation and Superset integration.
# #region TranslateJobService [C:5] [TYPE Class] [SEMANTICS translate,service,crud]
# @BRIEF Service for translation job CRUD with validation and Superset integration.
# @PRE Database session and config manager are available.
# @POST Translation jobs are managed with full lifecycle (create/update/delete/duplicate).
# @SIDE_EFFECT Queries Superset for dialect detection and column validation.
# @DATA_CONTRACT Input[TranslateJobCreate/Update] -> Output[TranslationJob]
# @INVARIANT Snapshot isolation: in-progress runs unaffected by config edits.
class TranslateJobService:
def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None):
@@ -159,9 +178,8 @@ class TranslateJobService:
self.config_manager = config_manager
self.current_user = current_user
# [DEF:list_jobs:Function]
# @PURPOSE: List translation jobs with optional status filter and pagination.
# @POST: Returns tuple of (total_count, list_of_jobs).
# #region list_jobs [C:2] [TYPE Function] [SEMANTICS translate,jobs,query]
# @BRIEF List translation jobs with optional status filter and pagination.
def list_jobs(
self,
page: int = 1,
@@ -178,29 +196,30 @@ class TranslateJobService:
jobs = query.order_by(TranslationJob.created_at.desc()).offset(offset).limit(page_size).all()
return total, jobs
# [/DEF:list_jobs:Function]
# #endregion list_jobs
# [DEF:get_job:Function]
# @PURPOSE: Get a single translation job by ID.
# @POST: Returns TranslationJob or raises ValueError.
# #region get_job [C:2] [TYPE Function] [SEMANTICS translate,jobs,query]
# @BRIEF Get a single translation job by ID, raising ValueError if not found.
def get_job(self, job_id: str) -> TranslationJob:
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
if not job:
raise ValueError(f"Translation job '{job_id}' not found")
return job
# [/DEF:get_job:Function]
# #endregion get_job
# [DEF:create_job:Function]
# @PURPOSE: Create a new translation job with column validation.
# @PRE: payload contains valid job configuration.
# @POST: Returns the created TranslationJob with database_dialect cached.
# @SIDE_EFFECT: Validates columns via SupersetClient if source_datasource_id is provided.
# @SIDE_EFFECT: Caches database_dialect from Superset connection.
# #region create_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,create]
# @BRIEF Create a new translation job with column validation and dialect detection.
# @PRE payload contains valid job configuration (name, upsert_strategy, etc).
# @POST Returns the created TranslationJob with database_dialect cached.
# @SIDE_EFFECT Validates columns via SupersetClient if source_datasource_id is provided; caches database_dialect.
def create_job(self, payload: TranslateJobCreate) -> TranslationJob:
logger.info(f"[TranslateJobService] Creating job '{payload.name}'")
logger.reason("Creating translation job", {"name": payload.name})
# Validate: must have a translation column if datasource is configured
if payload.source_datasource_id and not payload.translation_column:
logger.explore("Missing translation column", {
"source_datasource_id": payload.source_datasource_id,
})
raise ValueError("A translation column is required when a datasource is selected")
# Validate upsert strategy
@@ -225,7 +244,7 @@ class TranslateJobService:
)
dialect = detected_dialect
except Exception as e:
logger.warning(f"[TranslateJobService] Dialect detection failed: {e}")
logger.explore("Dialect detection failed", {"error": str(e)})
dialect = payload.source_dialect
# Build job instance
@@ -249,6 +268,7 @@ class TranslateJobService:
batch_size=payload.batch_size,
upsert_strategy=payload.upsert_strategy,
environment_id=payload.environment_id,
target_database_id=payload.target_database_id,
status="DRAFT",
created_by=self.current_user,
)
@@ -267,17 +287,17 @@ class TranslateJobService:
self.db.commit()
self.db.refresh(job)
logger.info(f"[TranslateJobService] Created job '{job.id}'")
logger.reflect("Job created", {"job_id": job.id})
return job
# [/DEF:create_job:Function]
# #endregion create_job
# [DEF:update_job:Function]
# @PURPOSE: Update an existing translation job.
# @PRE: payload contains fields to update.
# @POST: Returns the updated TranslationJob.
# @SIDE_EFFECT: Re-detects database_dialect if source_datasource_id changed.
# #region update_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,update]
# @BRIEF Update an existing translation job with optional dialect re-detection.
# @PRE payload contains fields to update; job_id exists.
# @POST Returns the updated TranslationJob with re-detected dialect if datasource changed.
# @SIDE_EFFECT Re-detects database_dialect if source_datasource_id changed.
def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob:
logger.info(f"[TranslateJobService] Updating job '{job_id}'")
logger.reason("Updating translation job", {"job_id": job_id})
job = self.get_job(job_id)
update_data = payload.model_dump(exclude_unset=True)
@@ -298,7 +318,7 @@ class TranslateJobService:
)
job.database_dialect = detected_dialect
except Exception as e:
logger.warning(f"[TranslateJobService] Dialect re-detection failed: {e}")
logger.explore("Dialect re-detection failed", {"error": str(e)})
job.updated_at = datetime.now(timezone.utc)
@@ -317,16 +337,17 @@ class TranslateJobService:
self.db.commit()
self.db.refresh(job)
logger.info(f"[TranslateJobService] Updated job '{job_id}'")
logger.reflect("Job updated", {"job_id": job_id})
return job
# [/DEF:update_job:Function]
# #endregion update_job
# [DEF:delete_job:Function]
# @PURPOSE: Delete a translation job and its associations.
# @PRE: job_id must exist.
# @POST: Job and all related records are deleted.
# #region delete_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,delete]
# @BRIEF Delete a translation job and its dictionary associations.
# @PRE job_id must exist.
# @POST Job and all related TranslationJobDictionary records are deleted.
# @SIDE_EFFECT Deletes TranslationJob and TranslationJobDictionary rows.
def delete_job(self, job_id: str) -> None:
logger.info(f"[TranslateJobService] Deleting job '{job_id}'")
logger.reason("Deleting translation job", {"job_id": job_id})
job = self.get_job(job_id)
# Delete dictionary associations
@@ -336,15 +357,16 @@ class TranslateJobService:
self.db.delete(job)
self.db.commit()
logger.info(f"[TranslateJobService] Deleted job '{job_id}'")
# [/DEF:delete_job:Function]
logger.reflect("Job deleted", {"job_id": job_id})
# #endregion delete_job
# [DEF:duplicate_job:Function]
# @PURPOSE: Duplicate a translation job with a new name.
# @PRE: job_id must exist.
# @POST: Returns the duplicated TranslationJob with status DRAFT.
# #region duplicate_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,duplicate]
# @BRIEF Duplicate a translation job with a new name, copying all fields and dictionary associations.
# @PRE job_id must exist.
# @POST Returns the duplicated TranslationJob with status DRAFT.
# @SIDE_EFFECT Creates new TranslationJob and TranslationJobDictionary rows.
def duplicate_job(self, job_id: str, new_name: Optional[str] = None) -> TranslationJob:
logger.info(f"[TranslateJobService] Duplicating job '{job_id}'")
logger.reason("Duplicating translation job", {"job_id": job_id, "new_name": new_name})
source = self.get_job(job_id)
# Copy all fields except id, created_at, updated_at, status
@@ -367,6 +389,8 @@ class TranslateJobService:
provider_id=source.provider_id,
batch_size=source.batch_size,
upsert_strategy=source.upsert_strategy,
environment_id=source.environment_id,
target_database_id=source.target_database_id,
status="DRAFT",
created_by=self.current_user,
)
@@ -387,22 +411,20 @@ class TranslateJobService:
self.db.commit()
self.db.refresh(new_job)
logger.info(f"[TranslateJobService] Duplicated job '{job_id}' -> '{new_job.id}'")
logger.reflect("Job duplicated", {"new_job_id": new_job.id, "source_job_id": job_id})
return new_job
# [/DEF:duplicate_job:Function]
# #endregion duplicate_job
# [DEF:get_job_dictionary_ids:Function]
# @PURPOSE: Get dictionary IDs attached to a job.
# @POST: Returns list of dictionary IDs.
# #region get_job_dictionary_ids [C:1] [TYPE Function] [SEMANTICS translate,jobs,dictionaries]
def get_job_dictionary_ids(self, job_id: str) -> List[str]:
assocs = self.db.query(TranslationJobDictionary).filter(
TranslationJobDictionary.job_id == job_id
).all()
return [a.dictionary_id for a in assocs]
# [/DEF:get_job_dictionary_ids:Function]
# #endregion get_job_dictionary_ids
# [DEF:fetch_available_datasources:Function]
# @PURPOSE: List available Superset datasets for translation job creation.
# #region fetch_available_datasources [C:2] [TYPE Function] [SEMANTICS translate,datasources,query]
# @BRIEF List available Superset datasets for translation job creation with optional search filter.
def fetch_available_datasources(self, env_id: str, search: Optional[str] = None) -> list:
"""List Superset datasets available for translation."""
from ...core.superset_client import SupersetClient
@@ -428,12 +450,11 @@ class TranslateJobService:
"description": ds.get("description", ""),
})
return result
# [/DEF:fetch_available_datasources:Function]
# [/DEF:TranslateJobService:Class]
# #endregion fetch_available_datasources
# #endregion TranslateJobService
# [DEF:_extract_dialect:Function]
# @PURPOSE: Extract database dialect from Superset backend URI.
# #region _extract_dialect [C:1] [TYPE Function] [SEMANTICS translate,dialect]
def _extract_dialect(backend: str) -> str:
"""Extract dialect name from a Superset database backend URI (e.g. 'postgresql+psycopg2://...')."""
if not backend:
@@ -444,10 +465,11 @@ def _extract_dialect(backend: str) -> str:
return dialect.lower()
except Exception:
return "unknown"
# #endregion _extract_dialect
# [DEF:job_to_response:Function]
# @PURPOSE: Convert a TranslationJob ORM model to a TranslateJobResponse schema with dictionary_ids.
# #region job_to_response [C:2] [TYPE Function] [SEMANTICS translate,serialization]
# @BRIEF Convert a TranslationJob ORM model to a TranslateJobResponse schema with dictionary_ids.
def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -> TranslateJobResponse:
return TranslateJobResponse(
id=job.id,
@@ -474,27 +496,29 @@ def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -
updated_at=job.updated_at,
dictionary_ids=dict_ids or [],
environment_id=job.environment_id,
target_database_id=job.target_database_id,
)
# [/DEF:job_to_response:Function]
# #endregion job_to_response
# [DEF:DatasourceColumnsService:Function]
# @PURPOSE: Fetch datasource column metadata from Superset and return structured response.
# @PRE: datasource_id is a valid Superset dataset ID.
# @POST: Returns DatasourceColumnsResponse with column metadata and database dialect.
# @SIDE_EFFECT: Queries Superset API for dataset detail and database info.
# #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS translate,datasource,columns]
# @BRIEF Fetch datasource column metadata from Superset and return structured DatasourceColumnsResponse.
# @PRE datasource_id is a valid Superset dataset ID.
# @POST Returns DatasourceColumnsResponse with column metadata and database dialect.
# @SIDE_EFFECT Queries Superset API for dataset detail and database info.
def get_datasource_columns(
datasource_id: int,
env_id: str,
config_manager: ConfigManager,
) -> DatasourceColumnsResponse:
"""Fetch and return column metadata for a given Superset datasource."""
logger.info(f"[get_datasource_columns] Fetching columns for datasource {datasource_id}")
logger.reason("Fetching datasource columns", {"datasource_id": datasource_id, "env_id": env_id})
# Find environment config
environments = config_manager.get_environments()
env_config = next((e for e in environments if e.id == env_id), None)
if not env_config:
logger.explore("Environment not found", {"env_id": env_id})
raise ValueError(f"Superset environment '{env_id}' not found")
# Create Superset client
@@ -516,6 +540,7 @@ def get_datasource_columns(
db_result = db_record.get("result", db_record) if isinstance(db_record, dict) else db_record
dialect = get_dialect_from_database(db_result)
else:
logger.explore("Could not determine dialect", {"datasource_id": datasource_id})
raise ValueError("Could not determine database dialect for this datasource")
# Extract columns
@@ -533,13 +558,19 @@ def get_datasource_columns(
description=col.get("description", ""),
))
return DatasourceColumnsResponse(
result = DatasourceColumnsResponse(
datasource_id=datasource_id,
datasource_name=dataset_detail.get("table_name"),
schema_name=dataset_detail.get("schema"),
database_dialect=dialect,
columns=columns,
)
# [/DEF:DatasourceColumnsService:Function]
logger.reflect("Datasource columns fetched", {
"datasource_id": datasource_id,
"columns_count": len(columns),
"dialect": dialect,
})
return result
# #endregion get_datasource_columns
# [/DEF:TranslateJobService:Module]
# #endregion TranslateJobService

179
backend/src/plugins/translate/sql_generator.py
View File

@@ -1,18 +1,13 @@
# [DEF:SQLGenerator:Module]
# @COMPLEXITY: 3
# @SEMANTICS: translate, sql, generator, dialect
# @PURPOSE: Dialect-aware safe SQL generation for INSERT/UPSERT operations.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [TranslationJob]
# @RELATION: DEPENDS_ON -> [TranslationRun]
# @PRE: Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS.
# @POST: Returns safe SQL strings for the target dialect.
# @SIDE_EFFECT: None — pure code generation.
# @RATIONALE: Dialect-aware SQL uses ON CONFLICT for PostgreSQL; plain INSERT for ClickHouse with documented limitations.
# @REJECTED: UPDATE statements — source is append-only; UPSERT covers overwrite case.
# @REJECTED: ORM-based insert bypasses Superset's SQL Lab audit trail.
# #region SQLGenerator [C:4] [TYPE Module] [SEMANTICS translate,sql,generator,dialect]
# @BRIEF Dialect-aware safe SQL generation for INSERT/UPSERT operations with identifier quoting and value encoding.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION DEPENDS_ON -> [TranslationRun]
# @PRE Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS.
# @POST Returns safe SQL strings for the target dialect.
from typing import Any, Dict, List, Optional, Tuple
from datetime import datetime, timezone
from ...core.logger import logger, belief_scope
# PostgreSQL/Greenplum dialects that support ON CONFLICT
@@ -21,10 +16,45 @@ POSTGRESQL_DIALECTS = {"postgresql", "redshift", "greenplum"}
CLICKHOUSE_DIALECTS = {"clickhouse"}
# [DEF:_quote_identifier:Function]
# @PURPOSE: Quote an identifier per dialect rules. PostgreSQL uses double quotes; ClickHouse uses backticks.
# @PRE: identifier is a non-empty string.
# @POST: Returns safely quoted identifier.
# #region _normalize_timestamp_value [C:2] [TYPE Function] [SEMANTICS translate,sql,timestamp]
# @BRIEF Detect Unix timestamp strings (seconds or millis) and convert to 'YYYY-MM-DD' for Date columns.
def _normalize_timestamp_value(value: Any) -> Optional[str]:
"""Detect Unix timestamp values and convert to date string.
Handles:
- Integer/float Unix timestamps (seconds or milliseconds)
- String representations of Unix timestamps (e.g. '1726358400000.0')
Returns 'YYYY-MM-DD' if conversion succeeds, None if value is not a timestamp.
"""
# Try numeric conversion first
try:
ts = float(value)
except (ValueError, TypeError):
return None
# Heuristic: Unix timestamps in seconds are ~10 digits (1e9 range for 2001-2033)
# Unix timestamps in milliseconds are ~13 digits (1e12 range for 2001-2033)
if 1e9 <= ts < 1e12:
# Already in seconds
pass
elif 1e12 <= ts < 1e15:
# Milliseconds — convert to seconds
ts = ts / 1000.0
else:
# Not a plausible Unix timestamp
return None
try:
dt = datetime.fromtimestamp(ts, tz=timezone.utc)
return dt.strftime("%Y-%m-%d")
except (OSError, OverflowError, ValueError):
return None
# #endregion _normalize_timestamp_value
# #region _quote_identifier [C:2] [TYPE Function] [SEMANTICS translate,sql,quoting]
# @BRIEF Quote a SQL identifier per dialect rules — double quotes for PostgreSQL, backticks for ClickHouse.
def _quote_identifier(identifier: str, dialect: str) -> str:
"""Quote a SQL identifier per dialect rules."""
if not identifier:
@@ -38,50 +68,67 @@ def _quote_identifier(identifier: str, dialect: str) -> str:
else:
# Generic ANSI double-quote
return f'"{cleaned}"'
# [/DEF:_quote_identifier:Function]
# #endregion _quote_identifier
# [DEF:_encode_sql_value:Function]
# @PURPOSE: Encode a Python value into a SQL-safe literal for INSERT VALUES.
# @PRE: value is a Python primitive or None.
# @POST: Returns SQL-safe string literal representation.
def _encode_sql_value(value: Any) -> str:
"""Encode a Python value into a SQL-safe literal."""
# #region _encode_sql_value [C:2] [TYPE Function] [SEMANTICS translate,sql,encoding]
# @BRIEF Encode a Python value into a SQL-safe literal for INSERT VALUES, with ClickHouse timestamp normalization.
def _encode_sql_value(value: Any, dialect: Optional[str] = None) -> str:
"""Encode a Python value into a SQL-safe literal.
For ClickHouse dialect, attempts to detect Unix timestamp strings
and convert them to 'YYYY-MM-DD' format for Date column compatibility.
"""
if value is None:
return "NULL"
if isinstance(value, bool):
return "TRUE" if value else "FALSE"
# For ClickHouse: try to normalize timestamp-like string values
if dialect in CLICKHOUSE_DIALECTS and isinstance(value, str) and value:
normalized = _normalize_timestamp_value(value)
if normalized:
return f"'{normalized}'"
if isinstance(value, (int, float)):
return str(value)
# String — escape single quotes by doubling them
escaped = str(value).replace("'", "''")
return f"'{escaped}'"
# [/DEF:_encode_sql_value:Function]
# #endregion _encode_sql_value
# [DEF:_build_values_clause:Function]
# @PURPOSE: Build a VALUES clause for multiple rows.
# @PRE: columns list is non-empty; rows is a list of dicts.
# @POST: Returns SQL VALUES clause string.
def _build_values_clause(columns: List[str], rows: List[Dict[str, Any]]) -> str:
"""Build VALUES (...) clause for multiple rows."""
# #region _build_values_clause [C:2] [TYPE Function] [SEMANTICS translate,sql,values]
# @BRIEF Build a VALUES clause for multiple rows with per-column value encoding and dialect-aware quoting.
def _build_values_clause(columns: List[str], rows: List[Dict[str, Any]], dialect: Optional[str] = None) -> str:
"""Build VALUES (...) clause for multiple rows.
NOTE: columns may be quoted (e.g. '"col"' or '`col`') for the SQL column list,
but row dicts have UNQUOTED keys. Strip quotes before value lookup.
"""
value_groups = []
for row in rows:
values = [_encode_sql_value(row.get(col)) for col in columns]
values = []
for col in columns:
# Strip quoting for value lookup — row keys are unquoted
lookup_key = col.strip('"').strip('`').strip('[]')
val = row.get(lookup_key)
values.append(_encode_sql_value(val, dialect=dialect))
value_groups.append(f"({', '.join(values)})")
return ",\n".join(value_groups)
# [/DEF:_build_values_clause:Function]
# #endregion _build_values_clause
# [DEF:generate_insert_sql:Function]
# @PURPOSE: Generate a dialect-aware INSERT SQL statement.
# @PRE: target_table, columns are non-empty.
# @POST: Returns SQL string or raises ValueError on invalid input.
# #region generate_insert_sql [C:3] [TYPE Function] [SEMANTICS translate,sql,insert]
# @BRIEF Generate a dialect-aware plain INSERT SQL statement for the given table, columns, and rows.
# @RELATION DEPENDS_ON -> [_quote_identifier]
# @RELATION DEPENDS_ON -> [_build_values_clause]
def generate_insert_sql(
target_schema: Optional[str],
target_table: str,
columns: List[str],
rows: List[Dict[str, Any]],
dialect: Optional[str] = None,
) -> str:
"""Generate a plain INSERT SQL."""
with belief_scope("generate_insert_sql"):
@@ -93,7 +140,7 @@ def generate_insert_sql(
raise ValueError("At least one row is required for INSERT SQL generation")
col_list = ", ".join(columns)
values = _build_values_clause(columns, rows)
values = _build_values_clause(columns, rows, dialect=dialect)
table_ref = target_table
if target_schema:
@@ -101,19 +148,20 @@ def generate_insert_sql(
sql = f"INSERT INTO {table_ref} ({col_list})\nVALUES\n{values};"
return sql
# [/DEF:generate_insert_sql:Function]
# #endregion generate_insert_sql
# [DEF:generate_upsert_sql:Function]
# @PURPOSE: Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE.
# @PRE: dialect is postgresql-compatible. target_table, columns, key_columns are non-empty.
# @POST: Returns UPSERT SQL string or raises ValueError.
# #region generate_upsert_sql [C:3] [TYPE Function] [SEMANTICS translate,sql,upsert]
# @BRIEF Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE / DO NOTHING.
# @RELATION DEPENDS_ON -> [_quote_identifier]
# @RELATION DEPENDS_ON -> [_build_values_clause]
def generate_upsert_sql(
target_schema: Optional[str],
target_table: str,
columns: List[str],
key_columns: List[str],
rows: List[Dict[str, Any]],
dialect: Optional[str] = None,
) -> str:
"""Generate INSERT ... ON CONFLICT (key_cols) DO UPDATE SET ... SQL."""
with belief_scope("generate_upsert_sql"):
@@ -128,7 +176,7 @@ def generate_upsert_sql(
col_list = ", ".join(columns)
key_list = ", ".join(key_columns)
values = _build_values_clause(columns, rows)
values = _build_values_clause(columns, rows, dialect=dialect)
table_ref = target_table
if target_schema:
@@ -150,21 +198,23 @@ def generate_upsert_sql(
f"ON CONFLICT ({key_list}) {conflict_action};"
)
return sql
# [/DEF:generate_upsert_sql:Function]
# #endregion generate_upsert_sql
# [DEF:SQLGenerator:Class]
# @COMPLEXITY: 3
# @PURPOSE: Generate safe, dialect-appropriate SQL INSERT/UPSERT statements.
# @PRE: Job has target_schema, target_table, key columns configured.
# @POST: Returns generated SQL string for the target dialect.
# #region SQLGenerator [C:4] [TYPE Class] [SEMANTICS translate,sql,generator]
# @BRIEF Generate safe, dialect-appropriate SQL INSERT/UPSERT statements with batching support.
# @PRE Job has target_schema, target_table, key columns configured.
# @POST Returns generated SQL string for the target dialect.
# @SIDE_EFFECT None — pure SQL generation.
# @RELATION DEPENDS_ON -> [generate_insert_sql]
# @RELATION DEPENDS_ON -> [generate_upsert_sql]
class SQLGenerator:
# [DEF:SQLGenerator.generate:Function]
# @PURPOSE: Generate SQL for a set of rows, detecting dialect from the job configuration.
# @PRE: dialect is a supported database dialect. columns list is non-empty. rows is non-empty.
# @POST: Returns tuple of (sql_string, statement_count).
# @SIDE_EFFECT: None — pure SQL generation.
# #region SQLGenerator.generate [C:4] [TYPE Function] [SEMANTICS translate,sql,generate]
# @BRIEF Generate SQL for a set of rows, detecting dialect from the job configuration.
# @PRE dialect is a supported database dialect. columns list is non-empty. rows is non-empty.
# @POST Returns tuple of (sql_string, statement_count).
# @SIDE_EFFECT None — pure SQL generation.
@staticmethod
def generate(
dialect: str,
@@ -236,6 +286,7 @@ class SQLGenerator:
columns=quoted_columns,
key_columns=quoted_key_columns,
rows=rows,
dialect=dialect,
)
else:
sql = generate_insert_sql(
@@ -243,6 +294,7 @@ class SQLGenerator:
target_table=table_ref,
columns=quoted_columns,
rows=rows,
dialect=dialect,
)
elif dialect in CLICKHOUSE_DIALECTS:
# ClickHouse: plain INSERT, no ON CONFLICT support
@@ -251,6 +303,7 @@ class SQLGenerator:
target_table=table_ref,
columns=quoted_columns,
rows=rows,
dialect=dialect,
)
if use_upsert:
logger.reason("ClickHouse UPSERT not supported; using plain INSERT", {
@@ -263,6 +316,7 @@ class SQLGenerator:
target_table=table_ref,
columns=quoted_columns,
rows=rows,
dialect=dialect,
)
logger.reflect("SQL generated", {
@@ -271,12 +325,11 @@ class SQLGenerator:
"sql_length": len(sql),
})
return sql, len(rows)
# [/DEF:SQLGenerator.generate:Function]
# #endregion SQLGenerator.generate
# [DEF:SQLGenerator.generate_batch:Function]
# @PURPOSE: Generate separate INSERT statements for each row (batch-safe version).
# @PRE: Same as generate().
# @POST: Returns list of (sql_string, row_index) tuples.
# #region SQLGenerator.generate_batch [C:3] [TYPE Function] [SEMANTICS translate,sql,batch]
# @BRIEF Generate separate INSERT statements for each row chunk (batch-safe version).
# @RELATION DEPENDS_ON -> [SQLGenerator.generate]
@staticmethod
def generate_batch(
dialect: str,
@@ -312,8 +365,8 @@ class SQLGenerator:
statements.append((sql, count))
return statements
# [/DEF:SQLGenerator.generate_batch:Function]
# #endregion SQLGenerator.generate_batch
# [/DEF:SQLGenerator:Class]
# [/DEF:SQLGenerator:Module]
# #endregion SQLGenerator
# #endregion SQLGenerator

180
backend/src/plugins/translate/superset_executor.py
View File

@@ -1,17 +1,17 @@
# [DEF:SupersetSqlLabExecutor:Module]
# @COMPLEXITY: 4
# @SEMANTICS: translate, superset, sqllab, execute
# @PURPOSE: Submit SQL to Superset SQL Lab API and poll execution status.
# @LAYER: Infra
# @RELATION: DEPENDS_ON -> [SupersetClient]
# @RELATION: DEPENDS_ON -> [ConfigManager]
# @PRE: Valid Superset environment configuration and authenticated client.
# @POST: SQL is submitted to Superset SQL Lab; execution reference is returned.
# @SIDE_EFFECT: Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status.
# @RATIONALE: Direct SQL Lab API submission provides audit trail and execution monitoring within Superset.
# @REJECTED: Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC.
# #region SupersetSqlLabExecutor [C:5] [TYPE Module] [SEMANTICS translate,superset,sqllab,execute]
# @BRIEF Submit SQL to Superset SQL Lab API and poll execution status with retry.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [ConfigManager]
# @PRE Valid Superset environment configuration and authenticated client.
# @POST SQL is submitted to Superset SQL Lab; execution reference is returned.
# @SIDE_EFFECT Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status.
# @DATA_CONTRACT Input[sql:str, database_id:Optional] -> Output[Dict with query_id, status, error_message]
# @INVARIANT Polling respects max_polls limit; timeout returns structured error.
# @RATIONALE Direct SQL Lab API submission provides audit trail and execution monitoring within Superset.
# @REJECTED Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC.
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict, List, Optional, Tuple, Union
from datetime import datetime, timezone
import json
import time
@@ -22,25 +22,26 @@ from ...core.config_manager import ConfigManager
from ...core.superset_client import SupersetClient
# [DEF:SupersetSqlLabExecutor:Class]
# @COMPLEXITY: 4
# @PURPOSE: Submit SQL to Superset SQL Lab API with polling and status tracking.
# @PRE: Valid environment ID and ConfigManager.
# @POST: SQL submitted to Superset; execution reference recorded.
# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status.
# #region SupersetSqlLabExecutor [C:5] [TYPE Class] [SEMANTICS translate,superset,sqllab]
# @BRIEF Submit SQL to Superset SQL Lab API with polling and status tracking.
# @PRE Valid environment ID and ConfigManager.
# @POST SQL submitted to Superset; execution reference recorded; polling completes.
# @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status.
# @DATA_CONTRACT Input[env_id:str, database_id:Optional] -> Output[Dict with query_id, status]
# @INVARIANT database_id resolved lazily; polling capped at max_polls.
class SupersetSqlLabExecutor:
def __init__(self, config_manager: ConfigManager, env_id: str):
def __init__(self, config_manager: ConfigManager, env_id: str, database_id: Optional[int] = None):
self.config_manager = config_manager
self.env_id = env_id
self._client: Optional[SupersetClient] = None
self._database_id: Optional[int] = None
self._database_id: Optional[int] = database_id
# [DEF:_get_client:Function]
# @PURPOSE: Lazy-initialize SupersetClient for the configured environment.
# @PRE: env_id must correspond to a valid environment config.
# @POST: Returns authenticated SupersetClient.
# @SIDE_EFFECT: Authenticates against Superset API.
# #region _get_client [C:4] [TYPE Function] [SEMANTICS translate,superset,client]
# @BRIEF Lazy-initialize SupersetClient for the configured environment.
# @PRE env_id must correspond to a valid environment config.
# @POST Returns authenticated SupersetClient.
# @SIDE_EFFECT Authenticates against Superset API on first call.
def _get_client(self) -> SupersetClient:
with belief_scope("SupersetSqlLabExecutor._get_client"):
if self._client is not None:
@@ -53,13 +54,13 @@ class SupersetSqlLabExecutor:
self._client = SupersetClient(env_config)
return self._client
# [/DEF:_get_client:Function]
# #endregion _get_client
# [DEF:resolve_database_id:Function]
# @PURPOSE: Resolve the target database ID from the environment.
# @PRE: database_name or database_id should be known.
# @POST: Returns database_id integer or raises ValueError.
# @SIDE_EFFECT: Fetches databases list from Superset.
# #region resolve_database_id [C:4] [TYPE Function] [SEMANTICS translate,database,resolve]
# @BRIEF Resolve the target database ID from the environment by name or default.
# @PRE Superset environment is reachable and has databases.
# @POST Returns database_id integer or raises ValueError.
# @SIDE_EFFECT Fetches databases list from Superset API.
def resolve_database_id(self, database_name: Optional[str] = None) -> int:
with belief_scope("SupersetSqlLabExecutor.resolve_database_id"):
client = self._get_client()
@@ -89,37 +90,71 @@ class SupersetSqlLabExecutor:
"database_id": self._database_id,
})
return self._database_id
# [/DEF:resolve_database_id:Function]
# #endregion resolve_database_id
# [DEF:execute_sql:Function]
# @PURPOSE: Submit SQL to Superset SQL Lab and return execution tracking info.
# @PRE: sql is valid SQL string. database_id is a valid Superset DB ID.
# @POST: Returns execution result dict with query_id and status.
# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/.
# #region resolve_database_uuid [C:4] [TYPE Function] [SEMANTICS translate,database,uuid]
# @BRIEF Resolve a database UUID to a numeric database ID from Superset.
# @PRE uuid_str is a valid Superset database UUID.
# @POST Returns numeric database_id or raises ValueError if not found.
# @SIDE_EFFECT Fetches databases list from Superset API.
def resolve_database_uuid(self, uuid_str: str) -> int:
with belief_scope("SupersetSqlLabExecutor.resolve_database_uuid"):
client = self._get_client()
_, databases = client.get_databases(
query={"columns": ["id", "uuid", "database_name"]}
)
if not databases:
raise ValueError("No databases found in Superset environment")
for db in databases:
db_uuid = db.get("uuid") or ""
if db_uuid.lower() == uuid_str.lower():
db_id = db["id"]
self._database_id = db_id
logger.reason("Resolved database ID by UUID", {
"uuid": uuid_str,
"database_id": db_id,
"database_name": db.get("database_name"),
})
return db_id
raise ValueError(
f"Database with UUID '{uuid_str}' not found in Superset environment"
)
# #endregion resolve_database_uuid
# #region execute_sql [C:4] [TYPE Function] [SEMANTICS translate,sql,execute]
# @BRIEF Submit SQL to Superset SQL Lab and return execution tracking info.
# @PRE sql is valid SQL string. database_id is a valid Superset DB ID.
# @POST Returns execution result dict with query_id and status.
# @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/.
def execute_sql(
self,
sql: str,
database_id: Optional[int] = None,
run_async: bool = True,
database_id: Optional[Union[int, str]] = None,
) -> Dict[str, Any]:
with belief_scope("SupersetSqlLabExecutor.execute_sql"):
client = self._get_client()
db_id = database_id or self._database_id
if not db_id:
if db_id is None:
db_id = self.resolve_database_id()
# Handle UUID string passed as database_id — resolve to numeric ID
if isinstance(db_id, str):
try:
db_id = int(db_id)
except (ValueError, TypeError):
db_id = self.resolve_database_uuid(db_id)
logger.reason("Submitting SQL to Superset SQL Lab", {
"database_id": db_id,
"sql_length": len(sql),
"run_async": run_async,
})
payload = {
"database_id": db_id,
"sql": sql,
"runAsync": run_async,
"client_id": uuid.uuid4().hex[:10],
"schema": None,
"tab": "translation-insert",
}
try:
@@ -133,14 +168,22 @@ class SupersetSqlLabExecutor:
logger.explore("SQL Lab execute failed", {"error": str(e)})
raise ValueError(f"Superset SQL Lab execute failed: {e}")
# Parse response
# Parse response — try multiple formats
result = response if isinstance(response, dict) else {}
query_id = result.get("query_id") or result.get("id")
query_id = (result.get("query_id")
or result.get("id")
or result.get("sql_lab_session_ref")
or (result.get("result") or {}).get("query_id")
or (result.get("result") or {}).get("id"))
status = result.get("status", "unknown")
logger.reason("SQL Lab execute response", {
"query_id": query_id,
"status": status,
"has_query_id": query_id is not None,
"has_errors": "errors" in result or "error" in result,
"response_keys": list(result.keys()) if isinstance(result, dict) else type(result).__name__,
"response_preview": str(result)[:1000] if isinstance(result, dict) else str(result)[:1000],
})
return {
@@ -149,13 +192,13 @@ class SupersetSqlLabExecutor:
"raw_response": result,
"database_id": db_id,
}
# [/DEF:execute_sql:Function]
# #endregion execute_sql
# [DEF:poll_execution_status:Function]
# @PURPOSE: Poll Superset for SQL execution status until completion or timeout.
# @PRE: query_id is a valid Superset query ID.
# @POST: Returns final execution status dict.
# @SIDE_EFFECT: Makes HTTP GET requests to Superset.
# #region poll_execution_status [C:4] [TYPE Function] [SEMANTICS translate,poll,status]
# @BRIEF Poll Superset for SQL execution status until completion or timeout.
# @PRE query_id is a valid Superset query ID.
# @POST Returns final execution status dict with success/failure/timeout.
# @SIDE_EFFECT Makes HTTP GET requests to Superset API.
def poll_execution_status(
self,
query_id: str,
@@ -240,17 +283,17 @@ class SupersetSqlLabExecutor:
"error_message": f"Polling timed out after {max_polls} attempts",
"results": None,
}
# [/DEF:poll_execution_status:Function]
# #endregion poll_execution_status
# [DEF:execute_and_poll:Function]
# @PURPOSE: Execute SQL and wait for completion. One-shot convenience method.
# @PRE: sql is valid SQL.
# @POST: Returns final execution result.
# @SIDE_EFFECT: Makes HTTP calls to Superset API.
# #region execute_and_poll [C:4] [TYPE Function] [SEMANTICS translate,sql,execute,poll]
# @BRIEF Execute SQL and wait for completion. One-shot convenience method.
# @PRE sql is valid SQL.
# @POST Returns final execution result.
# @SIDE_EFFECT Makes HTTP calls to Superset API.
def execute_and_poll(
self,
sql: str,
database_id: Optional[int] = None,
database_id: Optional[Union[int, str]] = None,
max_polls: int = 60,
poll_interval_seconds: float = 2.0,
) -> Dict[str, Any]:
@@ -258,7 +301,6 @@ class SupersetSqlLabExecutor:
exec_result = self.execute_sql(
sql=sql,
database_id=database_id,
run_async=True,
)
query_id = exec_result.get("query_id")
@@ -277,13 +319,13 @@ class SupersetSqlLabExecutor:
max_polls=max_polls,
poll_interval_seconds=poll_interval_seconds,
)
# [/DEF:execute_and_poll:Function]
# #endregion execute_and_poll
# [DEF:get_query_results:Function]
# @PURPOSE: Fetch the results of a completed query.
# @PRE: query_id is a valid Superset query ID.
# @POST: Returns query results if available.
# @SIDE_EFFECT: Makes HTTP GET to Superset.
# #region get_query_results [C:4] [TYPE Function] [SEMANTICS translate,query,results]
# @BRIEF Fetch the results of a completed query from Superset.
# @PRE query_id is a valid Superset query ID.
# @POST Returns query results if available, or error dict.
# @SIDE_EFFECT Makes HTTP GET to Superset API.
def get_query_results(self, query_id: str) -> Dict[str, Any]:
with belief_scope("SupersetSqlLabExecutor.get_query_results"):
client = self._get_client()
@@ -309,8 +351,8 @@ class SupersetSqlLabExecutor:
"error_message": str(e),
"results": None,
}
# [/DEF:get_query_results:Function]
# #endregion get_query_results
# [/DEF:SupersetSqlLabExecutor:Class]
# [/DEF:SupersetSqlLabExecutor:Module]
# #endregion SupersetSqlLabExecutor
# #endregion SupersetSqlLabExecutor

5
backend/src/schemas/__init__.py
View File

@@ -1,3 +1,2 @@
# [DEF:SchemasPackage:Package]
# @PURPOSE: API schema package root.
# [/DEF:SchemasPackage:Package]
# #region SchemasPackage [C:1] [TYPE Package]
# #endregion SchemasPackage

172
backend/src/schemas/translate.py
View File

@@ -1,9 +1,6 @@
# [DEF:TranslateSchemas:Module]
# @COMPLEXITY: 2
# @SEMANTICS: translate, schemas, pydantic
# @PURPOSE: Pydantic v2 schemas for translation API request/response serialization.
# @LAYER: API
# @RELATION: DEPENDS_ON -> pydantic
# #region TranslateSchemas [C:2] [TYPE Module]
# @BRIEF Pydantic v2 schemas for translation API request/response serialization.
# @LAYER Domain
from datetime import datetime
from typing import Any, Dict, List, Optional
@@ -11,8 +8,7 @@ from pydantic import BaseModel, Field
import json
# [DEF:TranslateJobCreate:Class]
# @PURPOSE: Schema for creating a new translation job.
# #region TranslateJobCreate [C:1] [TYPE Class]
class TranslateJobCreate(BaseModel):
name: str
description: Optional[str] = None
@@ -33,11 +29,11 @@ class TranslateJobCreate(BaseModel):
upsert_strategy: str = Field("MERGE", description="UPSERT strategy: MERGE, INSERT, UPDATE")
dictionary_ids: Optional[List[str]] = Field(default_factory=list, description="Associated terminology dictionary IDs")
environment_id: Optional[str] = Field(None, description="Superset environment ID")
# [/DEF:TranslateJobCreate:Class]
target_database_id: Optional[str] = Field(None, description="Superset database ID for INSERT via SQL Lab")
# #endregion TranslateJobCreate
# [DEF:TranslateJobUpdate:Class]
# @PURPOSE: Schema for updating an existing translation job.
# #region TranslateJobUpdate [C:1] [TYPE Class]
class TranslateJobUpdate(BaseModel):
name: Optional[str] = None
description: Optional[str] = None
@@ -59,11 +55,11 @@ class TranslateJobUpdate(BaseModel):
status: Optional[str] = None
dictionary_ids: Optional[List[str]] = None
environment_id: Optional[str] = None
# [/DEF:TranslateJobUpdate:Class]
target_database_id: Optional[str] = None
# #endregion TranslateJobUpdate
# [DEF:TranslateJobResponse:Class]
# @PURPOSE: Schema for translation job API responses.
# #region TranslateJobResponse [C:1] [TYPE Class]
class TranslateJobResponse(BaseModel):
id: str
name: str
@@ -89,24 +85,24 @@ class TranslateJobResponse(BaseModel):
updated_at: Optional[datetime] = None
dictionary_ids: Optional[List[str]] = None
environment_id: Optional[str] = None
target_database_id: Optional[str] = None
class Config:
from_attributes = True
# [/DEF:TranslateJobResponse:Class]
# #endregion TranslateJobResponse
# [DEF:DatasourceColumnResponse:Class]
# @PURPOSE: Schema for datasource column metadata response.
# #region DatasourceColumnResponse [C:1] [TYPE Class]
class DatasourceColumnResponse(BaseModel):
name: str
type: Optional[str] = None
is_physical: bool = True
is_dttm: bool = False
description: Optional[str] = None
# #endregion DatasourceColumnResponse
# [DEF:DatasourceColumnsResponse:Class]
# @PURPOSE: Schema for datasource columns endpoint response.
# #region DatasourceColumnsResponse [C:1] [TYPE Class]
class DatasourceColumnsResponse(BaseModel):
datasource_id: int
datasource_name: Optional[str] = None
@@ -116,11 +112,10 @@ class DatasourceColumnsResponse(BaseModel):
class Config:
from_attributes = True
# [/DEF:DatasourceColumnsResponse:Class]
# #endregion DatasourceColumnsResponse
# [DEF:DuplicateJobResponse:Class]
# @PURPOSE: Schema for duplicate job response.
# #region DuplicateJobResponse [C:1] [TYPE Class]
class DuplicateJobResponse(BaseModel):
id: str
name: str
@@ -128,32 +123,29 @@ class DuplicateJobResponse(BaseModel):
class Config:
from_attributes = True
# [/DEF:DuplicateJobResponse:Class]
# #endregion DuplicateJobResponse
# [DEF:DictionaryCreate:Class]
# @PURPOSE: Schema for creating a new terminology dictionary.
# #region DictionaryCreate [C:1] [TYPE Class]
class DictionaryCreate(BaseModel):
name: str
description: Optional[str] = None
source_dialect: str
target_dialect: str
is_active: bool = True
# [/DEF:DictionaryCreate:Class]
# #endregion DictionaryCreate
# [DEF:DictionaryImport:Class]
# @PURPOSE: Schema for importing entries into a terminology dictionary.
# #region DictionaryImport [C:1] [TYPE Class]
class DictionaryImport(BaseModel):
content: str = Field(..., description="CSV or TSV content as raw string")
delimiter: Optional[str] = Field(None, description="Detected or forced delimiter: ',' or '\\t'. Auto-detect if omitted.")
on_conflict: str = Field("overwrite", description="'overwrite' or 'keep_existing' or 'cancel'")
preview_only: bool = Field(False, description="If true, return preview without applying")
# [/DEF:DictionaryImport:Class]
# #endregion DictionaryImport
# [DEF:DictionaryResponse:Class]
# @PURPOSE: Schema for terminology dictionary API responses.
# #region DictionaryResponse [C:1] [TYPE Class]
class DictionaryResponse(BaseModel):
id: str
name: str
@@ -168,11 +160,10 @@ class DictionaryResponse(BaseModel):
class Config:
from_attributes = True
# [/DEF:DictionaryResponse:Class]
# #endregion DictionaryResponse
# [DEF:DictionaryEntryCreate:Class]
# @PURPOSE: Schema for adding/editing a dictionary entry.
# #region DictionaryEntryCreate [C:1] [TYPE Class]
class DictionaryEntryCreate(BaseModel):
source_term: str = Field(..., description="Source term to translate")
target_term: str = Field(..., description="Target/translated term")
@@ -180,11 +171,10 @@ class DictionaryEntryCreate(BaseModel):
class Config:
from_attributes = True
# [/DEF:DictionaryEntryCreate:Class]
# #endregion DictionaryEntryCreate
# [DEF:DictionaryEntryResponse:Class]
# @PURPOSE: Schema for dictionary entry API responses.
# #region DictionaryEntryResponse [C:1] [TYPE Class]
class DictionaryEntryResponse(BaseModel):
id: str
dictionary_id: str
@@ -197,11 +187,10 @@ class DictionaryEntryResponse(BaseModel):
class Config:
from_attributes = True
# [/DEF:DictionaryEntryResponse:Class]
# #endregion DictionaryEntryResponse
# [DEF:DictionaryImportResult:Class]
# @PURPOSE: Schema for dictionary import result.
# #region DictionaryImportResult [C:1] [TYPE Class]
class DictionaryImportResult(BaseModel):
total: int = 0
created: int = 0
@@ -209,28 +198,26 @@ class DictionaryImportResult(BaseModel):
skipped: int = 0
errors: List[Dict[str, Any]] = Field(default_factory=list)
preview: List[Dict[str, Any]] = Field(default_factory=list, description="Preview rows with conflict flags")
# [/DEF:DictionaryImportResult:Class]
# #endregion DictionaryImportResult
# [DEF:PreviewRequest:Class]
# @PURPOSE: Schema for triggering a translation preview.
# #region PreviewRequest [C:1] [TYPE Class]
class PreviewRequest(BaseModel):
sample_size: int = Field(10, ge=1, le=100, description="Number of sample rows to preview")
prompt_template: Optional[str] = Field(None, description="Optional custom prompt template")
env_id: Optional[str] = Field(None, description="Superset environment ID for preview data fetch")
# #endregion PreviewRequest
# [DEF:PreviewRowUpdate:Class]
# @PURPOSE: Schema for approving/editing/rejecting a preview row.
# #region PreviewRowUpdate [C:1] [TYPE Class]
class PreviewRowUpdate(BaseModel):
action: str = Field(..., description="'approve', 'reject', or 'edit'")
translation: Optional[str] = Field(None, description="Edited translation (required for 'edit' action)")
feedback: Optional[str] = Field(None, description="Optional feedback/comment")
# [/DEF:PreviewRowUpdate:Class]
# #endregion PreviewRowUpdate
# [DEF:PreviewAcceptResponse:Class]
# @PURPOSE: Schema for preview accept response.
# #region PreviewAcceptResponse [C:1] [TYPE Class]
class PreviewAcceptResponse(BaseModel):
id: str
job_id: str
@@ -242,11 +229,10 @@ class PreviewAcceptResponse(BaseModel):
class Config:
from_attributes = True
# [/DEF:PreviewAcceptResponse:Class]
# #endregion PreviewAcceptResponse
# [DEF:CostEstimate:Class]
# @PURPOSE: Schema for cost estimation in preview response.
# #region CostEstimate [C:1] [TYPE Class]
class CostEstimate(BaseModel):
sample_size: int = 0
sample_prompt_tokens: int = 0
@@ -256,11 +242,10 @@ class CostEstimate(BaseModel):
estimated_total_rows: int = 0
estimated_tokens: int = 0
estimated_cost: float = 0.0
# [/DEF:CostEstimate:Class]
# #endregion CostEstimate
# [DEF:TermCorrectionSubmit:Class]
# @PURPOSE: Schema for submitting a term correction in a translation preview.
# #region TermCorrectionSubmit [C:1] [TYPE Class]
class TermCorrectionSubmit(BaseModel):
source_term: str
incorrect_target_term: str
@@ -268,29 +253,26 @@ class TermCorrectionSubmit(BaseModel):
dictionary_id: Optional[str] = Field(None, description="Target dictionary ID (language-filtered)")
origin_run_id: Optional[str] = Field(None, description="Run ID from which this correction originated")
origin_row_key: Optional[str] = Field(None, description="Row key within the run")
# [/DEF:TermCorrectionSubmit:Class]
# #endregion TermCorrectionSubmit
# [DEF:TermCorrectionBulkSubmit:Class]
# @PURPOSE: Schema for submitting multiple term corrections atomically.
# #region TermCorrectionBulkSubmit [C:1] [TYPE Class]
class TermCorrectionBulkSubmit(BaseModel):
corrections: List[TermCorrectionSubmit]
dictionary_id: str = Field(..., description="Target dictionary ID for all corrections")
# [/DEF:TermCorrectionBulkSubmit:Class]
# #endregion TermCorrectionBulkSubmit
# [DEF:CorrectionConflictResult:Class]
# @PURPOSE: Schema for reporting conflicts in correction submission.
# #region CorrectionConflictResult [C:1] [TYPE Class]
class CorrectionConflictResult(BaseModel):
source_term: str
existing_target_term: str
submitted_target_term: str
action: str = "keep_existing"
# [/DEF:CorrectionConflictResult:Class]
# #endregion CorrectionConflictResult
# [DEF:CorrectionSubmitResponse:Class]
# @PURPOSE: Schema for correction submission response.
# #region CorrectionSubmitResponse [C:1] [TYPE Class]
class CorrectionSubmitResponse(BaseModel):
entry_id: Optional[str] = None
action: str # "created", "updated", "conflict_detected", "skipped"
@@ -298,20 +280,18 @@ class CorrectionSubmitResponse(BaseModel):
target_term: str
conflict: Optional[CorrectionConflictResult] = None
message: Optional[str] = None
# [/DEF:CorrectionSubmitResponse:Class]
# #endregion CorrectionSubmitResponse
# [DEF:ScheduleConfig:Class]
# @PURPOSE: Schema for configuring a recurring translation schedule.
# #region ScheduleConfig [C:1] [TYPE Class]
class ScheduleConfig(BaseModel):
cron_expression: str = Field(..., description="Cron expression for scheduling (e.g. '0 2 * * *')")
timezone: str = Field("UTC", description="Timezone for the cron schedule (e.g. 'UTC', 'Europe/Moscow')")
is_active: bool = True
# [/DEF:ScheduleConfig:Class]
# #endregion ScheduleConfig
# [DEF:ScheduleResponse:Class]
# @PURPOSE: Schema for schedule API responses.
# #region ScheduleResponse [C:1] [TYPE Class]
class ScheduleResponse(BaseModel):
id: str
job_id: str
@@ -326,21 +306,19 @@ class ScheduleResponse(BaseModel):
class Config:
from_attributes = True
# [/DEF:ScheduleResponse:Class]
# #endregion ScheduleResponse
# [DEF:NextExecutionResponse:Class]
# @PURPOSE: Schema for next execution preview.
# #region NextExecutionResponse [C:1] [TYPE Class]
class NextExecutionResponse(BaseModel):
job_id: str
cron_expression: str
timezone: str
next_executions: List[str] = []
# [/DEF:NextExecutionResponse:Class]
# #endregion NextExecutionResponse
# [DEF:TranslationRunResponse:Class]
# @PURPOSE: Schema for translation run API responses.
# #region TranslationRunResponse [C:1] [TYPE Class]
class TranslationRunResponse(BaseModel):
id: str
job_id: str
@@ -364,22 +342,20 @@ class TranslationRunResponse(BaseModel):
class Config:
from_attributes = True
# [/DEF:TranslationRunResponse:Class]
# #endregion TranslationRunResponse
# [DEF:RunDetailResponse:Class]
# @PURPOSE: Schema for detailed run response including records, events, and config snapshot.
# #region RunDetailResponse [C:1] [TYPE Class]
class RunDetailResponse(BaseModel):
run: TranslationRunResponse
records: List[Dict[str, Any]] = Field(default_factory=list, description="Paginated translation records")
events: List[Dict[str, Any]] = Field(default_factory=list, description="Run events")
event_invariants: Optional[Dict[str, Any]] = None
batch_count: int = 0
# [/DEF:RunDetailResponse:Class>
# #endregion RunDetailResponse
# [DEF:RunHistoryFilter:Class]
# @PURPOSE: Schema for filtering run history.
# #region RunHistoryFilter [C:1] [TYPE Class]
class RunHistoryFilter(BaseModel):
job_id: Optional[str] = None
status: Optional[str] = None
@@ -389,21 +365,19 @@ class RunHistoryFilter(BaseModel):
date_to: Optional[datetime] = None
page: int = 1
page_size: int = 20
# [/DEF:RunHistoryFilter:Class>
# #endregion RunHistoryFilter
# [DEF:RunListResponse:Class]
# @PURPOSE: Schema for paginated run list response.
# #region RunListResponse [C:1] [TYPE Class]
class RunListResponse(BaseModel):
items: List[TranslationRunResponse] = []
total: int = 0
page: int = 1
page_size: int = 20
# [/DEF:RunListResponse:Class>
# #endregion RunListResponse
# [DEF:AggregatedMetricsResponse:Class]
# @PURPOSE: Schema for aggregated per-job metrics.
# #region AggregatedMetricsResponse [C:1] [TYPE Class]
class AggregatedMetricsResponse(BaseModel):
job_id: str
total_runs: int = 0
@@ -419,11 +393,10 @@ class AggregatedMetricsResponse(BaseModel):
avg_duration_ms: Optional[float] = None
last_run_at: Optional[datetime] = None
next_scheduled_run: Optional[datetime] = None
# [/DEF:AggregatedMetricsResponse:Class]
# #endregion AggregatedMetricsResponse
# [DEF:PreviewRow:Class]
# @PURPOSE: A single row in a translation preview showing original and translated content side by side.
# #region PreviewRow [C:1] [TYPE Class]
class PreviewRow(BaseModel):
id: str
source_sql: Optional[str] = None
@@ -433,11 +406,10 @@ class PreviewRow(BaseModel):
source_object_name: Optional[str] = None
status: str = "PENDING"
feedback: Optional[str] = None
# [/DEF:PreviewRow:Class]
# #endregion PreviewRow
# [DEF:TranslationPreviewResponse:Class]
# @PURPOSE: Schema for translation preview session API responses.
# #region TranslationPreviewResponse [C:1] [TYPE Class]
class TranslationPreviewResponse(BaseModel):
id: str
job_id: str
@@ -450,11 +422,10 @@ class TranslationPreviewResponse(BaseModel):
class Config:
from_attributes = True
# [/DEF:TranslationPreviewResponse:Class]
# #endregion TranslationPreviewResponse
# [DEF:TranslationBatchResponse:Class]
# @PURPOSE: Schema for translation batch API responses.
# #region TranslationBatchResponse [C:1] [TYPE Class]
class TranslationBatchResponse(BaseModel):
id: str
run_id: str
@@ -469,11 +440,10 @@ class TranslationBatchResponse(BaseModel):
class Config:
from_attributes = True
# [/DEF:TranslationBatchResponse:Class]
# #endregion TranslationBatchResponse
# [DEF:MetricsResponse:Class]
# @PURPOSE: Schema for translation metrics API responses.
# #region MetricsResponse [C:1] [TYPE Class]
class MetricsResponse(BaseModel):
job_id: str
snapshot_date: datetime
@@ -490,6 +460,6 @@ class MetricsResponse(BaseModel):
class Config:
from_attributes = True
# [/DEF:MetricsResponse:Class]
# #endregion MetricsResponse
# [/DEF:TranslateSchemas:Module]
# #endregion TranslateSchemas