From 9f995f22aeab58dc404292eebc429e87d8f4b4c1 Mon Sep 17 00:00:00 2001 From: busya Date: Tue, 12 May 2026 14:32:28 +0300 Subject: [PATCH] refactor: migrate translate engine to GRACE-Poly v2.6 semantic protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Convert all 84 contracts from legacy [DEF:] to #region/#endregion syntax - Fix complexity tiers: 14 modules re-tiered (6 C4 route modules, 7 C4→C5 plugin services) - Remove forbidden tags: @RATIONALE/@REJECTED stripped from C1–C4 contracts - Add required tags: @PRE/@POST/@SIDE_EFFECT on C4, @RELATION on C3, @DATA_CONTRACT/@INVARIANT on C5 - Add belief runtime markers (reason/reflect/explore) to 7 service.py functions - Fix @LAYER: route files → UI, plugins → Domain, superset_executor → Infra - Fix pre-existing test mock_service fixture in test_orchestrator.py - 196/196 translation tests pass, zero regressions --- backend/src/api/routes/translate/__init__.py | 29 +- .../routes/translate/_correction_routes.py | 40 +-- .../routes/translate/_dictionary_routes.py | 140 +++++---- backend/src/api/routes/translate/_helpers.py | 34 +-- .../src/api/routes/translate/_job_routes.py | 112 +++++--- .../api/routes/translate/_metrics_routes.py | 31 +- .../api/routes/translate/_preview_routes.py | 81 +++--- backend/src/api/routes/translate/_router.py | 14 +- .../api/routes/translate/_run_list_routes.py | 51 ++-- .../src/api/routes/translate/_run_routes.py | 139 +++++---- .../api/routes/translate/_schedule_routes.py | 93 +++--- backend/src/models/__init__.py | 5 +- backend/src/models/translate.py | 78 +++-- backend/src/plugins/translate/__init__.py | 5 +- .../translate/__tests__/test_orchestrator.py | 2 +- backend/src/plugins/translate/_utils.py | 22 +- backend/src/plugins/translate/dictionary.py | 191 ++++++------- backend/src/plugins/translate/events.py | 154 ++++++---- backend/src/plugins/translate/executor.py | 268 +++++++++++++----- backend/src/plugins/translate/metrics.py | 47 ++- backend/src/plugins/translate/orchestrator.py | 236 ++++++++------- backend/src/plugins/translate/plugin.py | 22 +- backend/src/plugins/translate/preview.py | 209 +++++++------- backend/src/plugins/translate/scheduler.py | 128 +++++---- backend/src/plugins/translate/service.py | 219 ++++++++------ .../src/plugins/translate/sql_generator.py | 179 ++++++++---- .../plugins/translate/superset_executor.py | 180 +++++++----- backend/src/schemas/__init__.py | 5 +- backend/src/schemas/translate.py | 172 +++++------ 29 files changed, 1594 insertions(+), 1292 deletions(-) diff --git a/backend/src/api/routes/translate/__init__.py b/backend/src/api/routes/translate/__init__.py index e50ce3b5..f5275bdb 100644 --- a/backend/src/api/routes/translate/__init__.py +++ b/backend/src/api/routes/translate/__init__.py @@ -1,17 +1,16 @@ -# [DEF:TranslateRoutes:Module] -# @COMPLEXITY: 4 -# @SEMANTICS: api, routes, translate -# @PURPOSE: API routes for LLM-based SQL/dashboard translation management including terminology dictionary CRUD and import. -# @LAYER: UI (API) -# @RELATION: DEPENDS_ON -> [TranslateJobResponse] -# @RELATION: DEPENDS_ON -> [DictionaryManager:Class] -# @RELATION: DEPENDS_ON -> [get_current_user] -# @RELATION: DEPENDS_ON -> [get_db] -# @RELATION: DEPENDS_ON -> [has_permission] -# @RELATION: DEPENDS_ON -> [ConfigManager] -# @RELATION: DEPENDS_ON -> [SupersetClient] -# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. -# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity. +# #region TranslateRoutes [C:4] [TYPE Module] [SEMANTICS api,routes,translate] +# @BRIEF API routes for LLM-based SQL/dashboard translation management including terminology dictionary CRUD and import. +# @LAYER UI +# @RELATION DEPENDS_ON -> [TranslateJobResponse] +# @RELATION DEPENDS_ON -> [DictionaryManager] +# @RELATION DEPENDS_ON -> [get_current_user] +# @RELATION DEPENDS_ON -> [get_db] +# @RELATION DEPENDS_ON -> [has_permission] +# @RELATION DEPENDS_ON -> [ConfigManager] +# @RELATION DEPENDS_ON -> [SupersetClient] +# @PRE All sub-modules importable. Router instance available from ._router. +# @POST Package imports all route modules and re-exports router. Sub-routes registered on shared router. +# @SIDE_EFFECT Registers route handlers on shared APIRouter at import time. """ Translate routes package. @@ -34,4 +33,4 @@ __all__ = [ "router", ] -# [/DEF:TranslateRoutes:Module] +# #endregion TranslateRoutes diff --git a/backend/src/api/routes/translate/_correction_routes.py b/backend/src/api/routes/translate/_correction_routes.py index 59593be2..0bf574ae 100644 --- a/backend/src/api/routes/translate/_correction_routes.py +++ b/backend/src/api/routes/translate/_correction_routes.py @@ -1,8 +1,12 @@ -# [DEF:TranslateCorrectionRoutesModule:Module] -# @COMPLEXITY: 2 -# @PURPOSE: Term correction submission endpoints. -# @LAYER: API -# @SEMANTICS: api, routes, translate, corrections +# #region TranslateCorrectionRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,corrections] +# @BRIEF Term correction submission endpoints for applying user feedback to dictionary entries. +# @LAYER UI +# @RELATION DEPENDS_ON -> [DictionaryManager] +# @RELATION DEPENDS_ON -> [get_current_user] +# @RELATION DEPENDS_ON -> [get_db] +# @PRE DB session initialized. User authenticated with translate.dictionary.edit permissions. +# @POST Term corrections submitted and applied to dictionary with conflict detection. +# @SIDE_EFFECT Creates/updates DictionaryEntry records in DB based on correction submissions. from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional @@ -26,10 +30,12 @@ from ._router import router # Corrections # ============================================================ -# [DEF:submit_correction:Function] -# @PURPOSE: Submit a term correction from a run result review. -# @PRE: User has translate.dictionary.edit permission. -# @POST: Correction applied with conflict detection. +# #region submit_correction [C:4] [TYPE Function] [SEMANTICS api,translate,corrections] +# @BRIEF Submit a single term correction from a run result review. +# @PRE User has translate.dictionary.edit permission. Dictionary ID is provided. +# @POST Correction applied with conflict detection. Result returned. +# @SIDE_EFFECT Creates/updates DictionaryEntry record in DB; logs correction origin. +# @RELATION DEPENDS_ON -> [DictionaryManager] @router.post("/corrections") async def submit_correction( payload: TermCorrectionSubmit, @@ -55,13 +61,15 @@ async def submit_correction( return result except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:submit_correction:Function] +# #endregion submit_correction -# [DEF:submit_bulk_corrections:Function] -# @PURPOSE: Submit multiple term corrections atomically. -# @PRE: User has translate.dictionary.edit permission. -# @POST: All corrections applied or none with conflict list. +# #region submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS api,translate,corrections] +# @BRIEF Submit multiple term corrections atomically with full conflict reporting. +# @PRE User has translate.dictionary.edit permission. Dictionary ID is provided. +# @POST All corrections applied or none with conflict list returned. +# @SIDE_EFFECT Creates/updates multiple DictionaryEntry records in DB atomically. +# @RELATION DEPENDS_ON -> [DictionaryManager] @router.post("/corrections/bulk") async def submit_bulk_corrections( payload: TermCorrectionBulkSubmit, @@ -92,6 +100,6 @@ async def submit_bulk_corrections( return result except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:submit_bulk_corrections:Function] +# #endregion submit_bulk_corrections -# [/DEF:TranslateCorrectionRoutesModule:Module] +# #endregion TranslateCorrectionRoutesModule diff --git a/backend/src/api/routes/translate/_dictionary_routes.py b/backend/src/api/routes/translate/_dictionary_routes.py index be3f139d..628426a4 100644 --- a/backend/src/api/routes/translate/_dictionary_routes.py +++ b/backend/src/api/routes/translate/_dictionary_routes.py @@ -1,8 +1,12 @@ -# [DEF:TranslateDictionaryRoutesModule:Module] -# @COMPLEXITY: 3 -# @PURPOSE: Terminology Dictionary CRUD, entries management, and import routes. -# @LAYER: API -# @SEMANTICS: api, routes, translate, dictionaries +# #region TranslateDictionaryRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,dictionaries] +# @BRIEF Terminology Dictionary CRUD, entries management, and import routes. +# @LAYER UI +# @RELATION DEPENDS_ON -> [DictionaryManager] +# @RELATION DEPENDS_ON -> [get_current_user] +# @RELATION DEPENDS_ON -> [get_db] +# @PRE DB session initialized. User authenticated with translate.dictionary permissions. +# @POST Dictionaries and entries CRUD completed. Import executed with conflict resolution. +# @SIDE_EFFECT Reads/writes TerminologyDictionary and DictionaryEntry records to DB. from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional @@ -29,10 +33,12 @@ from ._helpers import _dict_to_response, _get_dictionary_entry_counts # Terminology Dictionaries # ============================================================ -# [DEF:list_dictionaries:Function] -# @PURPOSE: List all terminology dictionaries. -# @PRE: User has translate.dictionary.view permission. -# @POST: Returns list of dictionaries with total count. +# #region list_dictionaries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries] +# @BRIEF List all terminology dictionaries with pagination and entry counts. +# @PRE User has translate.dictionary.view permission. +# @POST Returns paginated list of dictionaries with total count. +# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB. +# @RELATION DEPENDS_ON -> [DictionaryManager] @router.get("/dictionaries") async def list_dictionaries( page: int = Query(1, ge=1), @@ -50,13 +56,15 @@ async def list_dictionaries( items = [_dict_to_response(d, counts.get(d.id, 0)) for d in dicts] logger.reflect("Dictionaries listed", {"total": total, "returned": len(items)}) return {"items": items, "total": total, "page": page, "page_size": page_size} -# [/DEF:list_dictionaries:Function] +# #endregion list_dictionaries -# [DEF:get_dictionary:Function] -# @PURPOSE: Get a single terminology dictionary by ID. -# @PRE: User has translate.dictionary.view permission. -# @POST: Returns the dictionary with entry_count. +# #region get_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries] +# @BRIEF Get a single terminology dictionary by ID with entry count. +# @PRE User has translate.dictionary.view permission. +# @POST Returns the dictionary with entry_count or 404. +# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB. +# @RELATION DEPENDS_ON -> [DictionaryManager] @router.get("/dictionaries/{dictionary_id}") async def get_dictionary( dictionary_id: str, @@ -69,19 +77,21 @@ async def get_dictionary( logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") try: d = DictionaryManager.get_dictionary(db, dictionary_id) - from ...models.translate import DictionaryEntry + from ....models.translate import DictionaryEntry entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count() logger.reflect("Dictionary found", {"id": dictionary_id}) return _dict_to_response(d, entry_count) except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:get_dictionary:Function] +# #endregion get_dictionary -# [DEF:create_dictionary:Function] -# @PURPOSE: Create a new terminology dictionary. -# @PRE: User has translate.dictionary.create permission. -# @POST: Returns the created dictionary. +# #region create_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries] +# @BRIEF Create a new terminology dictionary. +# @PRE User has translate.dictionary.create permission. Payload validated. +# @POST Dictionary created and returned with zero entries. +# @SIDE_EFFECT Creates TerminologyDictionary record in DB. +# @RELATION DEPENDS_ON -> [DictionaryManager] @router.post("/dictionaries", status_code=status.HTTP_201_CREATED) async def create_dictionary( payload: DictionaryCreate, @@ -103,13 +113,15 @@ async def create_dictionary( ) logger.reflect("Dictionary created", {"id": d.id}) return _dict_to_response(d, 0) -# [/DEF:create_dictionary:Function] +# #endregion create_dictionary -# [DEF:update_dictionary:Function] -# @PURPOSE: Update an existing terminology dictionary. -# @PRE: User has translate.dictionary.edit permission. -# @POST: Returns the updated dictionary. +# #region update_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries] +# @BRIEF Update an existing terminology dictionary. +# @PRE User has translate.dictionary.edit permission. Dictionary exists. +# @POST Dictionary updated and returned with current entry count. +# @SIDE_EFFECT Updates TerminologyDictionary record in DB. +# @RELATION DEPENDS_ON -> [DictionaryManager] @router.put("/dictionaries/{dictionary_id}") async def update_dictionary( dictionary_id: str, @@ -131,19 +143,21 @@ async def update_dictionary( target_dialect=payload.target_dialect, is_active=payload.is_active, ) - from ...models.translate import DictionaryEntry + from ....models.translate import DictionaryEntry entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count() logger.reflect("Dictionary updated", {"id": dictionary_id}) return _dict_to_response(d, entry_count) except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:update_dictionary:Function] +# #endregion update_dictionary -# [DEF:delete_dictionary:Function] -# @PURPOSE: Delete a terminology dictionary, blocked if attached to active/scheduled jobs. -# @PRE: User has translate.dictionary.delete permission. -# @POST: Dictionary is deleted. +# #region delete_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries] +# @BRIEF Delete a terminology dictionary, blocked if attached to active/scheduled jobs. +# @PRE User has translate.dictionary.delete permission. Dictionary exists and is not in use. +# @POST Dictionary is deleted or 409 if attached to active jobs. +# @SIDE_EFFECT Deletes TerminologyDictionary record from DB. +# @RELATION DEPENDS_ON -> [DictionaryManager] @router.delete("/dictionaries/{dictionary_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_dictionary( dictionary_id: str, @@ -162,17 +176,19 @@ async def delete_dictionary( if "active/scheduled" in str(e): raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:delete_dictionary:Function] +# #endregion delete_dictionary # ============================================================ # Dictionary Entries CRUD # ============================================================ -# [DEF:list_dictionary_entries:Function] -# @PURPOSE: List entries for a dictionary. -# @PRE: User has translate.dictionary.view permission. -# @POST: Returns paginated list of entries. +# #region list_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries] +# @BRIEF List entries for a dictionary with pagination. +# @PRE User has translate.dictionary.view permission. Dictionary exists. +# @POST Returns paginated list of entries. +# @SIDE_EFFECT Reads DictionaryEntry records from DB. +# @RELATION DEPENDS_ON -> [DictionaryManager] @router.get("/dictionaries/{dictionary_id}/entries") async def list_dictionary_entries( dictionary_id: str, @@ -204,13 +220,15 @@ async def list_dictionary_entries( return {"items": items, "total": total, "page": page, "page_size": page_size} except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:list_dictionary_entries:Function] +# #endregion list_dictionary_entries -# [DEF:add_dictionary_entry:Function] -# @PURPOSE: Add a new entry to a dictionary. -# @PRE: User has translate.dictionary.edit permission. -# @POST: Entry is created. +# #region add_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries] +# @BRIEF Add a new entry to a dictionary. +# @PRE User has translate.dictionary.edit permission. Dictionary exists. +# @POST Entry created and returned. +# @SIDE_EFFECT Creates DictionaryEntry record in DB. +# @RELATION DEPENDS_ON -> [DictionaryManager] @router.post("/dictionaries/{dictionary_id}/entries", status_code=status.HTTP_201_CREATED) async def add_dictionary_entry( dictionary_id: str, @@ -244,13 +262,15 @@ async def add_dictionary_entry( if "already exists" in str(e): raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:add_dictionary_entry:Function] +# #endregion add_dictionary_entry -# [DEF:edit_dictionary_entry:Function] -# @PURPOSE: Update an existing dictionary entry. -# @PRE: User has translate.dictionary.edit permission. -# @POST: Entry is updated. +# #region edit_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries] +# @BRIEF Update an existing dictionary entry. +# @PRE User has translate.dictionary.edit permission. Entry exists. +# @POST Entry updated and returned. +# @SIDE_EFFECT Updates DictionaryEntry record in DB. +# @RELATION DEPENDS_ON -> [DictionaryManager] @router.put("/dictionaries/{dictionary_id}/entries/{entry_id}") async def edit_dictionary_entry( dictionary_id: str, @@ -285,13 +305,15 @@ async def edit_dictionary_entry( if "already exists" in str(e): raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:edit_dictionary_entry:Function] +# #endregion edit_dictionary_entry -# [DEF:delete_dictionary_entry:Function] -# @PURPOSE: Delete a dictionary entry. -# @PRE: User has translate.dictionary.edit permission. -# @POST: Entry is deleted. +# #region delete_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries] +# @BRIEF Delete a dictionary entry. +# @PRE User has translate.dictionary.edit permission. Entry exists. +# @POST Entry is deleted. +# @SIDE_EFFECT Deletes DictionaryEntry record from DB. +# @RELATION DEPENDS_ON -> [DictionaryManager] @router.delete("/dictionaries/{dictionary_id}/entries/{entry_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_dictionary_entry( dictionary_id: str, @@ -309,13 +331,15 @@ async def delete_dictionary_entry( return None except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:delete_dictionary_entry:Function] +# #endregion delete_dictionary_entry -# [DEF:import_dictionary_entries:Function] -# @PURPOSE: Import entries into a terminology dictionary from CSV/TSV content. -# @PRE: User has translate.dictionary.edit permission. -# @POST: Entries are imported per conflict mode. +# #region import_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,import] +# @BRIEF Import entries into a terminology dictionary from CSV/TSV content. +# @PRE User has translate.dictionary.edit permission. Dictionary exists. Content is valid CSV/TSV. +# @POST Entries imported per conflict mode (overwrite/keep_existing/cancel). Import result returned. +# @SIDE_EFFECT Creates/updates DictionaryEntry records in DB. +# @RELATION DEPENDS_ON -> [DictionaryManager] @router.post("/dictionaries/{dictionary_id}/import") async def import_dictionary_entries( dictionary_id: str, @@ -351,6 +375,6 @@ async def import_dictionary_entries( return result except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:import_dictionary_entries:Function] +# #endregion import_dictionary_entries -# [/DEF:TranslateDictionaryRoutesModule:Module] +# #endregion TranslateDictionaryRoutesModule diff --git a/backend/src/api/routes/translate/_helpers.py b/backend/src/api/routes/translate/_helpers.py index 6fb28faa..e9a6380e 100644 --- a/backend/src/api/routes/translate/_helpers.py +++ b/backend/src/api/routes/translate/_helpers.py @@ -1,17 +1,13 @@ -# [DEF:TranslateHelpersModule:Module] -# @COMPLEXITY: 2 -# @PURPOSE: Shared helper functions for translate route handlers. -# @LAYER: API -# @SEMANTICS: api, routes, translate, helpers -# @RELATION: DEPENDS_ON -> [models.translate] +# #region TranslateHelpersModule [C:2] [TYPE Module] [SEMANTICS api,routes,translate,helpers] +# @BRIEF Shared helper functions for translate route handlers. +# @LAYER UI from typing import Any, Dict, List from sqlalchemy.orm import Session -# [DEF:_run_to_response:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Convert TranslationRun ORM to response dict. +# #region _run_to_response [C:2] [TYPE Function] [SEMANTICS helpers,translate] +# @BRIEF Convert TranslationRun ORM to response dict. def _run_to_response(run: Any) -> dict: return { "id": run.id, @@ -34,12 +30,11 @@ def _run_to_response(run: Any) -> dict: "created_by": run.created_by, "created_at": run.created_at.isoformat() if run.created_at else None, } -# [/DEF:_run_to_response:Function] +# #endregion _run_to_response -# [DEF:_dict_to_response:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Convert a TerminologyDictionary ORM model to a response dict with computed entry_count. +# #region _dict_to_response [C:2] [TYPE Function] [SEMANTICS helpers,translate] +# @BRIEF Convert a TerminologyDictionary ORM model to a response dict with computed entry_count. @staticmethod def _dict_to_response(d: Any, entry_count: int = 0) -> dict: return { @@ -54,15 +49,14 @@ def _dict_to_response(d: Any, entry_count: int = 0) -> dict: "updated_at": d.updated_at, "entry_count": entry_count, } -# [/DEF:_dict_to_response:Function] +# #endregion _dict_to_response -# [DEF:_get_dictionary_entry_counts:Function] -# @COMPLEXITY: 2 -# @PURPOSE: Get entry counts for a list of dictionary IDs. +# #region _get_dictionary_entry_counts [C:2] [TYPE Function] [SEMANTICS helpers,translate] +# @BRIEF Get entry counts for a list of dictionary IDs. def _get_dictionary_entry_counts(db: Session, dict_ids: List[str]) -> Dict[str, int]: from sqlalchemy import func - from ...models.translate import DictionaryEntry + from ....models.translate import DictionaryEntry counts = ( db.query(DictionaryEntry.dictionary_id, func.count(DictionaryEntry.id)) .filter(DictionaryEntry.dictionary_id.in_(dict_ids)) @@ -70,6 +64,6 @@ def _get_dictionary_entry_counts(db: Session, dict_ids: List[str]) -> Dict[str, .all() ) return {row[0]: row[1] for row in counts} -# [/DEF:_get_dictionary_entry_counts:Function] +# #endregion _get_dictionary_entry_counts -# [/DEF:TranslateHelpersModule:Module] +# #endregion TranslateHelpersModule diff --git a/backend/src/api/routes/translate/_job_routes.py b/backend/src/api/routes/translate/_job_routes.py index fdeafde5..836577fd 100644 --- a/backend/src/api/routes/translate/_job_routes.py +++ b/backend/src/api/routes/translate/_job_routes.py @@ -1,8 +1,14 @@ -# [DEF:TranslateJobRoutesModule:Module] -# @COMPLEXITY: 3 -# @PURPOSE: Translation Job CRUD and datasource column routes. -# @LAYER: API -# @SEMANTICS: api, routes, translate, jobs +# #region TranslateJobRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,jobs] +# @BRIEF Translation Job CRUD and datasource column routes. +# @LAYER UI +# @RELATION DEPENDS_ON -> [TranslateJobService] +# @RELATION DEPENDS_ON -> [ConfigManager] +# @RELATION DEPENDS_ON -> [SupersetClient] +# @RELATION DEPENDS_ON -> [get_current_user] +# @RELATION DEPENDS_ON -> [get_db] +# @PRE ConfigManager and DB session initialized. User authenticated with translate.job permissions. +# @POST Translation job CRUD completed or datasource columns fetched. +# @SIDE_EFFECT Reads/writes TranslationJob records to DB; fetches datasource metadata from Superset API. from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional @@ -33,10 +39,9 @@ from ._router import router # Translation Job CRUD # ============================================================ -# [DEF:list_jobs:Function] -# @PURPOSE: List all translation jobs. -# @PRE: User has translate.job.view permission. -# @POST: Returns list of translation jobs. +# #region list_jobs [C:3] [TYPE Function] [SEMANTICS api,translate,jobs] +# @BRIEF List all translation jobs with pagination and optional status filter. +# @RELATION DEPENDS_ON -> [TranslateJobService] @router.get("/jobs", response_model=List[TranslateJobResponse]) async def list_jobs( page: int = Query(1, ge=1), @@ -63,13 +68,12 @@ async def list_jobs( return results except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:list_jobs:Function] +# #endregion list_jobs -# [DEF:get_job:Function] -# @PURPOSE: Get a single translation job by ID. -# @PRE: User has translate.job.view permission. -# @POST: Returns the translation job. +# #region get_job [C:3] [TYPE Function] [SEMANTICS api,translate,jobs] +# @BRIEF Get a single translation job by ID. +# @RELATION DEPENDS_ON -> [TranslateJobService] @router.get("/jobs/{job_id}", response_model=TranslateJobResponse) async def get_job( job_id: str, @@ -87,14 +91,15 @@ async def get_job( return job_to_response(job, dict_ids) except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:get_job:Function] +# #endregion get_job -# [DEF:create_job:Function] -# @PURPOSE: Create a new translation job. -# @PRE: User has translate.job.create permission. -# @POST: Returns the created translation job. -# @SIDE_EFFECT: Validates columns via SupersetClient; caches database_dialect. +# #region create_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs] +# @BRIEF Create a new translation job. +# @PRE User has translate.job.create permission. Payload validated. +# @POST Translation job created and returned. +# @SIDE_EFFECT Creates TranslationJob record in DB; validates columns via SupersetClient; caches database_dialect. +# @RELATION DEPENDS_ON -> [TranslateJobService] @router.post("/jobs", response_model=TranslateJobResponse, status_code=status.HTTP_201_CREATED) async def create_job( payload: TranslateJobCreate, @@ -112,14 +117,15 @@ async def create_job( return job_to_response(job, dict_ids) except ValueError as e: raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)) -# [/DEF:create_job:Function] +# #endregion create_job -# [DEF:update_job:Function] -# @PURPOSE: Update an existing translation job. -# @PRE: User has translate.job.edit permission. -# @POST: Returns the updated translation job. -# @SIDE_EFFECT: Re-detects database_dialect if datasource changed. +# #region update_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs] +# @BRIEF Update an existing translation job. +# @PRE User has translate.job.edit permission. Job exists. +# @POST Translation job updated and returned. +# @SIDE_EFFECT Updates TranslationJob record in DB; re-detects database_dialect if datasource changed. +# @RELATION DEPENDS_ON -> [TranslateJobService] @router.put("/jobs/{job_id}", response_model=TranslateJobResponse) async def update_job( job_id: str, @@ -138,13 +144,15 @@ async def update_job( return job_to_response(job, dict_ids) except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:update_job:Function] +# #endregion update_job -# [DEF:delete_job:Function] -# @PURPOSE: Delete a translation job. -# @PRE: User has translate.job.delete permission. -# @POST: Job is deleted. +# #region delete_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs] +# @BRIEF Delete a translation job. +# @PRE User has translate.job.delete permission. Job exists. +# @POST Translation job is deleted from DB. +# @SIDE_EFFECT Deletes TranslationJob and associated records from DB. +# @RELATION DEPENDS_ON -> [TranslateJobService] @router.delete("/jobs/{job_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_job( job_id: str, @@ -160,13 +168,15 @@ async def delete_job( service.delete_job(job_id) except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:delete_job:Function] +# #endregion delete_job -# [DEF:duplicate_job:Function] -# @PURPOSE: Duplicate a translation job. -# @PRE: User has translate.job.create permission. -# @POST: Returns the duplicated job with status DRAFT. +# #region duplicate_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs] +# @BRIEF Duplicate a translation job. +# @PRE User has translate.job.create permission. Source job exists. +# @POST Duplicated job created with status DRAFT and returned. +# @SIDE_EFFECT Creates a new TranslationJob record in DB as a copy of the source. +# @RELATION DEPENDS_ON -> [TranslateJobService] @router.post("/jobs/{job_id}/duplicate", response_model=DuplicateJobResponse, status_code=status.HTTP_201_CREATED) async def duplicate_job( job_id: str, @@ -187,17 +197,16 @@ async def duplicate_job( ) except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:duplicate_job:Function] +# #endregion duplicate_job -# [DEF:get_datasource_columns:Function] -# @PURPOSE: Get column metadata and database dialect for a Superset datasource. -# @PRE: User has translate.job.view permission. -# @POST: Returns column list with metadata and database_dialect. -# @SIDE_EFFECT: Queries Superset API for dataset detail and database info. -# @RATIONALE: Dialect detection from Superset connection cached on TranslationJob at save time. -# @REJECTED: Hardcoding dialect list per database — would drift from actual Superset config. - +# #region list_datasources [C:4] [TYPE Function] [SEMANTICS api,translate,datasources] +# @BRIEF List all Superset datasets available for translation jobs. +# @PRE User has translate.job.view permission. +# @POST Returns list of datasets with metadata. +# @SIDE_EFFECT Queries Superset API for available datasets. +# @RELATION DEPENDS_ON -> [TranslateJobService] +# @RELATION DEPENDS_ON -> [SupersetClient] @router.get("/datasources") async def list_datasources( env_id: str = Query(..., description="Superset environment ID"), @@ -216,7 +225,16 @@ async def list_datasources( except Exception as e: logger.error(f"[translate_routes][list_datasources] Error: {e}") raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e)) +# #endregion list_datasources + +# #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS api,translate,datasources] +# @BRIEF Get column metadata and database dialect for a Superset datasource. +# @PRE User has translate.job.view permission. Datasource exists in Superset. +# @POST Returns column list with metadata and database_dialect. +# @SIDE_EFFECT Queries Superset API for dataset detail and database info. +# @RELATION DEPENDS_ON -> [SupersetClient] +# @RELATION DEPENDS_ON -> [ConfigManager] @router.get("/datasources/{datasource_id}/columns", response_model=DatasourceColumnsResponse) async def get_datasource_columns( datasource_id: int, @@ -245,6 +263,6 @@ async def get_datasource_columns( status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Failed to fetch datasource columns from Superset: {e}", ) -# [/DEF:get_datasource_columns:Function] +# #endregion get_datasource_columns -# [/DEF:TranslateJobRoutesModule:Module] +# #endregion TranslateJobRoutesModule diff --git a/backend/src/api/routes/translate/_metrics_routes.py b/backend/src/api/routes/translate/_metrics_routes.py index 94b39fc6..4bd4fe87 100644 --- a/backend/src/api/routes/translate/_metrics_routes.py +++ b/backend/src/api/routes/translate/_metrics_routes.py @@ -1,8 +1,9 @@ -# [DEF:TranslateMetricsRoutesModule:Module] -# @COMPLEXITY: 2 -# @PURPOSE: Translation Metrics endpoints. -# @LAYER: API -# @SEMANTICS: api, routes, translate, metrics +# #region TranslateMetricsRoutesModule [C:3] [TYPE Module] [SEMANTICS api,routes,translate,metrics] +# @BRIEF Translation Metrics endpoints providing aggregated stats on runs and corrections. +# @LAYER UI +# @RELATION DEPENDS_ON -> [TranslationMetrics] +# @RELATION DEPENDS_ON -> [get_current_user] +# @RELATION DEPENDS_ON -> [get_db] from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional @@ -21,10 +22,9 @@ from ._router import router # Metrics # ============================================================ -# [DEF:get_metrics:Function] -# @PURPOSE: Get translation metrics, optionally filtered by job. -# @PRE: User has translate.metrics.view permission. -# @POST: Returns metrics data. +# #region get_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics] +# @BRIEF Get translation metrics across all jobs, optionally filtered by job. +# @RELATION DEPENDS_ON -> [TranslationMetrics] @router.get("/metrics") async def get_metrics( job_id: Optional[str] = Query(None), @@ -41,13 +41,12 @@ async def get_metrics( return metrics_svc.get_all_metrics() except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:get_metrics:Function] +# #endregion get_metrics -# [DEF:get_job_metrics:Function] -# @PURPOSE: Get aggregated metrics for a specific job. -# @PRE: User has translate.metrics.view permission. -# @POST: Returns metrics dict. +# #region get_job_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics] +# @BRIEF Get aggregated metrics for a specific translation job. +# @RELATION DEPENDS_ON -> [TranslationMetrics] @router.get("/jobs/{job_id}/metrics") async def get_job_metrics( job_id: str, @@ -62,6 +61,6 @@ async def get_job_metrics( return metrics_svc.get_job_metrics(job_id) except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:get_job_metrics:Function] +# #endregion get_job_metrics -# [/DEF:TranslateMetricsRoutesModule:Module] +# #endregion TranslateMetricsRoutesModule diff --git a/backend/src/api/routes/translate/_preview_routes.py b/backend/src/api/routes/translate/_preview_routes.py index d599972b..e3f81ac8 100644 --- a/backend/src/api/routes/translate/_preview_routes.py +++ b/backend/src/api/routes/translate/_preview_routes.py @@ -1,8 +1,13 @@ -# [DEF:TranslatePreviewRoutesModule:Module] -# @COMPLEXITY: 3 -# @PURPOSE: Translation Preview session management routes. -# @LAYER: API -# @SEMANTICS: api, routes, translate, preview +# #region TranslatePreviewRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,preview] +# @BRIEF Translation Preview session management routes. +# @LAYER UI +# @RELATION DEPENDS_ON -> [TranslationPreview] +# @RELATION DEPENDS_ON -> [ConfigManager] +# @RELATION DEPENDS_ON -> [get_current_user] +# @RELATION DEPENDS_ON -> [get_db] +# @PRE ConfigManager and DB session initialized. User authenticated with translate permissions. +# @POST Preview sessions created, rows reviewed/accepted, records queried. +# @SIDE_EFFECT Creates/updates preview sessions and rows in DB; fetches sample data from Superset; calls LLM provider. from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional @@ -28,11 +33,13 @@ from ._router import router # Preview # ============================================================ -# [DEF:preview_translation:Function] -# @PURPOSE: Preview a translation before applying it. -# @PRE: User has translate.job.execute permission. -# @POST: Returns a preview session with records and cost estimation. -# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows. +# #region preview_translation [C:4] [TYPE Function] [SEMANTICS api,translate,preview] +# @BRIEF Preview a translation before applying it. +# @PRE User has translate.job.execute permission. Job exists. +# @POST Preview session created with sample rows and cost estimation. +# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates PreviewSession and PreviewRecord rows in DB. +# @RELATION DEPENDS_ON -> [TranslationPreview] +# @RELATION DEPENDS_ON -> [SupersetClient] @router.post("/jobs/{job_id}/preview", status_code=status.HTTP_201_CREATED) async def preview_translation( job_id: str, @@ -60,13 +67,15 @@ async def preview_translation( except Exception as e: logger.error(f"[translate_routes][preview_translation] Error: {e}") raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Preview failed: {e}") -# [/DEF:preview_translation:Function] +# #endregion preview_translation -# [DEF:update_preview_row:Function] -# @PURPOSE: Approve, edit, or reject a preview row. -# @PRE: User has translate.job.execute permission. -# @POST: Preview row status is updated. +# #region update_preview_row [C:4] [TYPE Function] [SEMANTICS api,translate,preview] +# @BRIEF Approve, edit, or reject a preview row. +# @PRE User has translate.job.execute permission. Preview row exists. +# @POST Preview row status updated to APPROVED/REJECTED/EDITED. +# @SIDE_EFFECT Updates PreviewRecord status in DB. +# @RELATION DEPENDS_ON -> [TranslationPreview] @router.put("/jobs/{job_id}/preview/rows/{row_key}") async def update_preview_row( job_id: str, @@ -91,13 +100,15 @@ async def update_preview_row( return result except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:update_preview_row:Function] +# #endregion update_preview_row -# [DEF:accept_preview_session:Function] -# @PURPOSE: Accept a preview session, marking it as the quality gate for full execution. -# @PRE: User has translate.job.execute permission. Job has an ACTIVE preview session. -# @POST: Preview session is marked as APPLIED; full execution can proceed. +# #region accept_preview_session [C:4] [TYPE Function] [SEMANTICS api,translate,preview] +# @BRIEF Accept a preview session, marking it as the quality gate for full execution. +# @PRE User has translate.job.execute permission. Job has an ACTIVE preview session. +# @POST Preview session status set to APPLIED. Full execution can proceed. +# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB. +# @RELATION DEPENDS_ON -> [TranslationPreview] @router.post("/jobs/{job_id}/preview/accept") async def accept_preview_session( job_id: str, @@ -114,13 +125,15 @@ async def accept_preview_session( return result except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:accept_preview_session:Function] +# #endregion accept_preview_session -# [DEF:apply_preview:Function] -# @PURPOSE: Apply a preview session (alias for accept when accepting at session level). -# @PRE: User has translate.job.execute permission. -# @POST: Preview is applied. +# #region apply_preview [C:4] [TYPE Function] [SEMANTICS api,translate,preview] +# @BRIEF Apply a preview session (alias for accept when accepting at session level). +# @PRE User has translate.job.execute permission. Session exists. +# @POST Preview session applied and marked as quality gate. +# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB. +# @RELATION DEPENDS_ON -> [TranslationPreview] @router.post("/preview/{session_id}/apply") async def apply_preview( session_id: str, @@ -132,7 +145,7 @@ async def apply_preview( """Apply a preview session by session ID.""" logger.info(f"[translate_routes][apply_preview] Session: {session_id}, User: {current_user.username}") # Find job_id from session - from ...models.translate import TranslationPreviewSession + from ....models.translate import TranslationPreviewSession session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first() if not session: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found") @@ -142,17 +155,17 @@ async def apply_preview( return result except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:apply_preview:Function] +# #endregion apply_preview # ============================================================ # Preview Records # ============================================================ -# [DEF:get_preview_records:Function] -# @PURPOSE: Get records for a preview session. -# @PRE: User has translate.job.view permission. -# @POST: Returns list of preview records. +# #region get_preview_records [C:3] [TYPE Function] [SEMANTICS api,translate,preview] +# @BRIEF Get records for a preview session. +# @RELATION DEPENDS_ON -> [TranslationPreviewSession] +# @RELATION DEPENDS_ON -> [TranslationPreviewRecord] @router.get("/preview/{session_id}/records", response_model=List[PreviewRow]) async def get_preview_records( session_id: str, @@ -163,7 +176,7 @@ async def get_preview_records( """Get records for a preview session.""" logger.info(f"[translate_routes][get_preview_records] Session: {session_id}, User: {current_user.username}") try: - from ...models.translate import TranslationPreviewSession, TranslationPreviewRecord + from ....models.translate import TranslationPreviewSession, TranslationPreviewRecord session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first() if not session: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found") @@ -190,6 +203,6 @@ async def get_preview_records( except Exception as e: logger.error(f"[translate_routes][get_preview_records] Error: {e}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:get_preview_records:Function] +# #endregion get_preview_records -# [/DEF:TranslatePreviewRoutesModule:Module] +# #endregion TranslatePreviewRoutesModule diff --git a/backend/src/api/routes/translate/_router.py b/backend/src/api/routes/translate/_router.py index c4eca686..bb01b6b5 100644 --- a/backend/src/api/routes/translate/_router.py +++ b/backend/src/api/routes/translate/_router.py @@ -1,14 +1,10 @@ -# [DEF:TranslateRouterModule:Module] -# @COMPLEXITY: 1 -# @PURPOSE: APIRouter instance for translate routes. -# @LAYER: API -# @SEMANTICS: api, routes, translate, router +# #region TranslateRouterModule [C:1] [TYPE Module] [SEMANTICS api,routes,translate,router] +# @LAYER UI from fastapi import APIRouter -# [DEF:translate_router:Global] -# @PURPOSE: APIRouter instance for all translate sub-routes. +# #region translate_router [C:1] [TYPE Global] router = APIRouter(prefix="/api/translate", tags=["translate"]) -# [/DEF:translate_router:Global] +# #endregion translate_router -# [/DEF:TranslateRouterModule:Module] +# #endregion TranslateRouterModule diff --git a/backend/src/api/routes/translate/_run_list_routes.py b/backend/src/api/routes/translate/_run_list_routes.py index e7f63bff..8fb6432d 100644 --- a/backend/src/api/routes/translate/_run_list_routes.py +++ b/backend/src/api/routes/translate/_run_list_routes.py @@ -1,8 +1,10 @@ -# [DEF:TranslateRunListRoutesModule:Module] -# @COMPLEXITY: 3 -# @PURPOSE: Translation Run listing, detail, and CSV download routes (cross-job). -# @LAYER: API -# @SEMANTICS: api, routes, translate, runs, list, detail +# #region TranslateRunListRoutesModule [C:3] [TYPE Module] [SEMANTICS api,routes,translate,runs,list,detail] +# @BRIEF Translation Run listing, detail, and CSV download routes (cross-job). +# @LAYER UI +# @RELATION DEPENDS_ON -> [TranslationOrchestrator] +# @RELATION DEPENDS_ON -> [TranslationRun] +# @RELATION DEPENDS_ON -> [get_current_user] +# @RELATION DEPENDS_ON -> [get_db] from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional @@ -23,14 +25,13 @@ from ._router import router # List Runs (cross-job) # ============================================================ -# [DEF:list_runs:Function] -# @PURPOSE: List all runs with cross-job filtering and pagination. -# @PRE: User has translate.history.view permission. -# @POST: Returns paginated list of runs. +# #region list_runs [C:3] [TYPE Function] [SEMANTICS api,translate,runs,list] +# @BRIEF List all runs with cross-job filtering and pagination. +# @RELATION DEPENDS_ON -> [TranslationRun] @router.get("/runs") async def list_runs( job_id: Optional[str] = Query(None), - run_status: Optional[str] = Query(None), + run_status: Optional[str] = Query(None, alias="status"), trigger_type: Optional[str] = Query(None), created_by: Optional[str] = Query(None), date_from: Optional[str] = Query(None), @@ -44,7 +45,7 @@ async def list_runs( """List all runs with cross-job filtering and pagination.""" logger.info(f"[translate_routes][list_runs] User: {current_user.username}") try: - from ...models.translate import TranslationRun + from ....models.translate import TranslationRun query = db.query(TranslationRun) if job_id: @@ -104,17 +105,18 @@ async def list_runs( except Exception as e: logger.error(f"[translate_routes][list_runs] Error: {e}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:list_runs:Function] +# #endregion list_runs # ============================================================ # Run Detail # ============================================================ -# [DEF:get_run_detail:Function] -# @PURPOSE: Get detailed run info with config_snapshot, records, events. -# @PRE: User has translate.history.view permission. -# @POST: Returns run detail with records and events. +# #region get_run_detail [C:3] [TYPE Function] [SEMANTICS api,translate,runs,detail] +# @BRIEF Get detailed run info with config_snapshot, records, events. +# @RELATION DEPENDS_ON -> [TranslationOrchestrator] +# @RELATION DEPENDS_ON -> [TranslationEventLog] +# @RELATION DEPENDS_ON -> [TranslationRun] @router.get("/runs/{run_id}/detail") async def get_run_detail( run_id: str, @@ -133,7 +135,7 @@ async def get_run_detail( event_summary = TranslationEventLog(db).get_run_event_summary(run_id) # Get config snapshot from run - from ...models.translate import TranslationRun + from ....models.translate import TranslationRun run = db.query(TranslationRun).filter(TranslationRun.id == run_id).first() config_snapshot = run.config_snapshot if run else None @@ -147,17 +149,16 @@ async def get_run_detail( } except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:get_run_detail:Function] +# #endregion get_run_detail # ============================================================ # Download Skipped CSV # ============================================================ -# [DEF:download_skipped_csv:Function] -# @PURPOSE: Download a CSV of skipped translation records from a run. -# @PRE: User has translate.job.view permission. -# @POST: Returns CSV file of skipped records. +# #region download_skipped_csv [C:3] [TYPE Function] [SEMANTICS api,translate,runs,csv] +# @BRIEF Download a CSV of skipped translation records from a run. +# @RELATION DEPENDS_ON -> [TranslationRecord] @router.get("/runs/{run_id}/skipped.csv") async def download_skipped_csv( run_id: str, @@ -168,7 +169,7 @@ async def download_skipped_csv( """Download skipped translation records as CSV.""" logger.info(f"[translate_routes][download_skipped_csv] Run: {run_id}, User: {current_user.username}") try: - from ...models.translate import TranslationRecord + from ....models.translate import TranslationRecord from fastapi.responses import StreamingResponse import csv import io @@ -201,6 +202,6 @@ async def download_skipped_csv( except Exception as e: logger.error(f"[translate_routes][download_skipped_csv] Error: {e}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:download_skipped_csv:Function] +# #endregion download_skipped_csv -# [/DEF:TranslateRunListRoutesModule:Module] +# #endregion TranslateRunListRoutesModule diff --git a/backend/src/api/routes/translate/_run_routes.py b/backend/src/api/routes/translate/_run_routes.py index a596d1c0..a55bd17b 100644 --- a/backend/src/api/routes/translate/_run_routes.py +++ b/backend/src/api/routes/translate/_run_routes.py @@ -1,20 +1,26 @@ -# [DEF:TranslateRunRoutesModule:Module] -# @COMPLEXITY: 3 -# @PURPOSE: Translation Run execution, history, status, records and batches routes. -# @LAYER: API -# @SEMANTICS: api, routes, translate, runs +# #region TranslateRunRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,runs] +# @BRIEF Translation Run execution, history, status, records and batches routes. +# @LAYER UI +# @RELATION DEPENDS_ON -> [TranslationOrchestrator] +# @RELATION DEPENDS_ON -> [ConfigManager] +# @RELATION DEPENDS_ON -> [get_current_user] +# @RELATION DEPENDS_ON -> [get_db] +# @PRE ConfigManager and DB session initialized. User authenticated with translate permissions. +# @POST Translation run executed, manipulated, or queried. Results returned to caller. +# @SIDE_EFFECT Creates/updates TranslationRun records in DB; spawns background threads for execution; queries Superset API. from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session -from ....core.database import get_db +from ....core.database import get_db, SessionLocal from ....core.logger import logger, belief_scope from ....schemas.auth import User from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager from ....plugins.translate.orchestrator import TranslationOrchestrator from ....plugins.translate.events import TranslationEventLog +from ....models.translate import TranslationRun from ._router import router from ._helpers import _run_to_response @@ -24,43 +30,64 @@ from ._helpers import _run_to_response # Translation Run / Execute # ============================================================ -# [DEF:run_translation:Function] -# @PURPOSE: Execute a translation job (trigger a run). -# @PRE: User has translate.job.execute permission. -# @POST: Returns the created translation run. +# #region run_translation [C:4] [TYPE Function] [SEMANTICS api,translate,runs] +# @BRIEF Execute a translation job (trigger a run). +# @PRE User has translate.job.execute permission. Job exists and preview is accepted. +# @POST Translation run created and started in background. Run object returned. +# @SIDE_EFFECT Creates TranslationRun record in DB; spawns background thread for translation execution. +# @RELATION DEPENDS_ON -> [TranslationOrchestrator] @router.post("/jobs/{job_id}/run", status_code=status.HTTP_201_CREATED) async def run_translation( job_id: str, + full_translation: bool = Query(False, description="If True, fetch ALL rows from Superset dataset instead of preview-only rows"), current_user: User = Depends(get_current_user), _ = Depends(has_permission("translate.job", "EXECUTE")), db: Session = Depends(get_db), config_manager: ConfigManager = Depends(get_config_manager), ): - """Execute a translation job (trigger a run).""" - logger.info(f"[translate_routes][run_translation] Job: {job_id}, User: {current_user.username}") + """Execute a translation job (trigger a run). + + By default runs translation on preview-approved rows only. + Set full_translation=true to fetch ALL rows from the Superset source dataset. + """ + logger.info(f"[translate_routes][run_translation] Job: {job_id}, User: {current_user.username}, full={full_translation}") try: orch = TranslationOrchestrator(db, config_manager, current_user.username) run = orch.start_run(job_id=job_id, is_scheduled=False) - # Execute asynchronously in background + # Execute asynchronously in background with a FRESH session. + # The request-scoped session from Depends(get_db) is closed after the handler returns, + # so the background thread must create its own session to avoid "This transaction is closed". import threading - threading.Thread( - target=orch.execute_run, - args=(run,), - daemon=True, - ).start() + + def _execute_background(): + thread_db = SessionLocal() + try: + thread_orch = TranslationOrchestrator(thread_db, config_manager, current_user.username) + # Reload run from DB with the new session (the passed run object is detached) + thread_run = thread_db.query(TranslationRun).filter(TranslationRun.id == run.id).first() + if thread_run: + thread_orch.execute_run(thread_run, full_translation=full_translation) + except Exception as e: + logger.error(f"[translate_routes][run_translation] Background execution error: {e}") + finally: + thread_db.close() + + threading.Thread(target=_execute_background, daemon=True).start() return _run_to_response(run) except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: logger.error(f"[translate_routes][run_translation] Error: {e}") raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Run failed: {e}") -# [/DEF:run_translation:Function] +# #endregion run_translation -# [DEF:retry_run:Function] -# @PURPOSE: Retry failed batches in a translation run. -# @PRE: User has translate.job.execute permission. -# @POST: Returns the updated translation run. +# #region retry_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs] +# @BRIEF Retry failed batches in a translation run. +# @PRE User has translate.job.execute permission. Run exists and has failed batches. +# @POST Failed batches re-executed. Updated run returned. +# @SIDE_EFFECT Updates TranslationRun and TranslationBatch records in DB. +# @RELATION DEPENDS_ON -> [TranslationOrchestrator] @router.post("/runs/{run_id}/retry") async def retry_run( run_id: str, @@ -80,13 +107,15 @@ async def retry_run( except Exception as e: logger.error(f"[translate_routes][retry_run] Error: {e}") raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry failed: {e}") -# [/DEF:retry_run:Function] +# #endregion retry_run -# [DEF:retry_insert:Function] -# @PURPOSE: Retry the SQL insert phase for a completed run. -# @PRE: User has translate.job.execute permission. -# @POST: Returns the updated run. +# #region retry_insert [C:4] [TYPE Function] [SEMANTICS api,translate,runs] +# @BRIEF Retry the SQL insert phase for a completed run. +# @PRE User has translate.job.execute permission. Run completed with insert_status FAILED. +# @POST SQL insert phase re-executed. Updated run returned. +# @SIDE_EFFECT Re-executes SQL insert into Superset; updates TranslationRun insert_status. +# @RELATION DEPENDS_ON -> [TranslationOrchestrator] @router.post("/runs/{run_id}/retry-insert") async def retry_insert( run_id: str, @@ -106,13 +135,15 @@ async def retry_insert( except Exception as e: logger.error(f"[translate_routes][retry_insert] Error: {e}") raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry insert failed: {e}") -# [/DEF:retry_insert:Function] +# #endregion retry_insert -# [DEF:cancel_run:Function] -# @PURPOSE: Cancel a running translation. -# @PRE: User has translate.job.execute permission. -# @POST: Run is cancelled. +# #region cancel_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs] +# @BRIEF Cancel a running translation. +# @PRE User has translate.job.execute permission. Run is in RUNNING state. +# @POST Run is cancelled. Updated run returned. +# @SIDE_EFFECT Updates TranslationRun status to CANCELLED in DB. +# @RELATION DEPENDS_ON -> [TranslationOrchestrator] @router.post("/runs/{run_id}/cancel") async def cancel_run( run_id: str, @@ -129,13 +160,12 @@ async def cancel_run( return _run_to_response(run) except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:cancel_run:Function] +# #endregion cancel_run -# [DEF:get_run_history:Function] -# @PURPOSE: Get run history for a translation job. -# @PRE: User has translate.history.view permission. -# @POST: Returns list of runs. +# #region get_run_history [C:3] [TYPE Function] [SEMANTICS api,translate,runs] +# @BRIEF Get run history for a translation job. +# @RELATION DEPENDS_ON -> [TranslationOrchestrator] @router.get("/jobs/{job_id}/runs") async def get_run_history( job_id: str, @@ -154,13 +184,12 @@ async def get_run_history( return {"items": runs, "total": total, "page": page, "page_size": page_size} except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:get_run_history:Function] +# #endregion get_run_history -# [DEF:get_run_status:Function] -# @PURPOSE: Get status and statistics for a translation run. -# @PRE: User has translate.history.view permission. -# @POST: Returns run details with statistics. +# #region get_run_status [C:3] [TYPE Function] [SEMANTICS api,translate,runs] +# @BRIEF Get status and statistics for a translation run. +# @RELATION DEPENDS_ON -> [TranslationOrchestrator] @router.get("/runs/{run_id}") async def get_run_status( run_id: str, @@ -176,13 +205,12 @@ async def get_run_status( return orch.get_run_status(run_id) except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:get_run_status:Function] +# #endregion get_run_status -# [DEF:get_run_records:Function] -# @PURPOSE: Get paginated records for a translation run. -# @PRE: User has translate.history.view permission. -# @POST: Returns paginated records. +# #region get_run_records [C:3] [TYPE Function] [SEMANTICS api,translate,runs] +# @BRIEF Get paginated records for a translation run. +# @RELATION DEPENDS_ON -> [TranslationOrchestrator] @router.get("/runs/{run_id}/records") async def get_run_records( run_id: str, @@ -201,17 +229,16 @@ async def get_run_records( return orch.get_run_records(run_id, page=page, page_size=page_size, status_filter=status) except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:get_run_records:Function] +# #endregion get_run_records # ============================================================ # Batches # ============================================================ -# [DEF:get_batches:Function] -# @PURPOSE: Get batches for a translation run. -# @PRE: User has translate.job.view permission. -# @POST: Returns list of batches. +# #region get_batches [C:3] [TYPE Function] [SEMANTICS api,translate,runs] +# @BRIEF Get batches for a translation run. +# @RELATION DEPENDS_ON -> [TranslationBatch] @router.get("/runs/{run_id}/batches") async def get_batches( run_id: str, @@ -222,7 +249,7 @@ async def get_batches( """Get batches for a translation run.""" logger.info(f"[translate_routes][get_batches] Run: {run_id}, User: {current_user.username}") try: - from ...models.translate import TranslationBatch + from ....models.translate import TranslationBatch batches = ( db.query(TranslationBatch) .filter(TranslationBatch.run_id == run_id) @@ -247,6 +274,6 @@ async def get_batches( except Exception as e: logger.error(f"[translate_routes][get_batches] Error: {e}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:get_batches:Function] +# #endregion get_batches -# [/DEF:TranslateRunRoutesModule:Module] +# #endregion TranslateRunRoutesModule diff --git a/backend/src/api/routes/translate/_schedule_routes.py b/backend/src/api/routes/translate/_schedule_routes.py index ccdd9c74..2916930d 100644 --- a/backend/src/api/routes/translate/_schedule_routes.py +++ b/backend/src/api/routes/translate/_schedule_routes.py @@ -1,8 +1,13 @@ -# [DEF:TranslateScheduleRoutesModule:Module] -# @COMPLEXITY: 3 -# @PURPOSE: Translation Schedule management routes. -# @LAYER: API -# @SEMANTICS: api, routes, translate, schedule +# #region TranslateScheduleRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,schedule] +# @BRIEF Translation Schedule management routes. +# @LAYER UI +# @RELATION DEPENDS_ON -> [TranslationScheduler] +# @RELATION DEPENDS_ON -> [ConfigManager] +# @RELATION DEPENDS_ON -> [get_current_user] +# @RELATION DEPENDS_ON -> [get_db] +# @PRE ConfigManager and DB session initialized. User authenticated with translate.schedule permissions. +# @POST Schedule CRUD operations completed. APScheduler jobs registered/unregistered. +# @SIDE_EFFECT Creates/updates/deletes TranslationSchedule records in DB; registers/removes jobs in APScheduler. from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional @@ -23,10 +28,9 @@ from ._router import router # Schedule # ============================================================ -# [DEF:get_schedule:Function] -# @PURPOSE: Get the schedule for a translation job. -# @PRE: User has translate.schedule.view permission. -# @POST: Returns the schedule configuration. +# #region get_schedule [C:3] [TYPE Function] [SEMANTICS api,translate,schedule] +# @BRIEF Get the schedule configuration for a translation job. +# @RELATION DEPENDS_ON -> [TranslationScheduler] @router.get("/jobs/{job_id}/schedule") async def get_schedule( job_id: str, @@ -54,13 +58,15 @@ async def get_schedule( } except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:get_schedule:Function] +# #endregion get_schedule -# [DEF:set_schedule:Function] -# @PURPOSE: Set or update the schedule for a translation job. -# @PRE: User has translate.schedule.manage permission. -# @POST: Schedule is created or updated. +# #region set_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule] +# @BRIEF Set or update the schedule for a translation job. +# @PRE User has translate.schedule.manage permission. Job exists. +# @POST Schedule created or updated. APScheduler job registered. +# @SIDE_EFFECT Creates/updates TranslationSchedule record in DB; registers job in APScheduler. +# @RELATION DEPENDS_ON -> [TranslationScheduler] @router.put("/jobs/{job_id}/schedule") async def set_schedule( job_id: str, @@ -75,7 +81,7 @@ async def set_schedule( try: scheduler = TranslationScheduler(db, config_manager, current_user.username) # Check if schedule already exists - from ...models.translate import TranslationSchedule + from ....models.translate import TranslationSchedule existing = db.query(TranslationSchedule).filter( TranslationSchedule.job_id == job_id ).first() @@ -94,7 +100,7 @@ async def set_schedule( is_active=payload.is_active, ) # Register with APScheduler via SchedulerService - from ...dependencies import get_scheduler_service + from ....dependencies import get_scheduler_service sched_svc = get_scheduler_service() sched_svc.add_translation_job( schedule_id=schedule.id, @@ -115,13 +121,15 @@ async def set_schedule( } except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:set_schedule:Function] +# #endregion set_schedule -# [DEF:enable_schedule:Function] -# @PURPOSE: Enable a schedule for a translation job. -# @PRE: User has translate.schedule.manage permission. -# @POST: Schedule is enabled. +# #region enable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule] +# @BRIEF Enable a schedule for a translation job. +# @PRE User has translate.schedule.manage permission. Schedule exists. +# @POST Schedule is enabled. APScheduler job registered. +# @SIDE_EFFECT Updates TranslationSchedule is_active to True in DB; registers job in APScheduler. +# @RELATION DEPENDS_ON -> [TranslationScheduler] @router.post("/jobs/{job_id}/schedule/enable") async def enable_schedule( job_id: str, @@ -135,7 +143,7 @@ async def enable_schedule( try: scheduler = TranslationScheduler(db, config_manager, current_user.username) schedule = scheduler.set_schedule_active(job_id, is_active=True) - from ...dependencies import get_scheduler_service + from ....dependencies import get_scheduler_service sched_svc = get_scheduler_service() sched_svc.add_translation_job( schedule_id=schedule.id, @@ -146,13 +154,15 @@ async def enable_schedule( return {"status": "enabled", "job_id": job_id} except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:enable_schedule:Function] +# #endregion enable_schedule -# [DEF:disable_schedule:Function] -# @PURPOSE: Disable a schedule for a translation job. -# @PRE: User has translate.schedule.manage permission. -# @POST: Schedule is disabled. +# #region disable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule] +# @BRIEF Disable a schedule for a translation job. +# @PRE User has translate.schedule.manage permission. Schedule exists. +# @POST Schedule is disabled. APScheduler job removed. +# @SIDE_EFFECT Updates TranslationSchedule is_active to False in DB; removes job from APScheduler. +# @RELATION DEPENDS_ON -> [TranslationScheduler] @router.post("/jobs/{job_id}/schedule/disable") async def disable_schedule( job_id: str, @@ -166,19 +176,21 @@ async def disable_schedule( try: scheduler = TranslationScheduler(db, config_manager, current_user.username) scheduler.set_schedule_active(job_id, is_active=False) - from ...dependencies import get_scheduler_service + from ....dependencies import get_scheduler_service sched_svc = get_scheduler_service() sched_svc.remove_translation_job(schedule_id=job_id) return {"status": "disabled", "job_id": job_id} except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:disable_schedule:Function] +# #endregion disable_schedule -# [DEF:delete_schedule:Function] -# @PURPOSE: Delete the schedule for a translation job. -# @PRE: User has translate.schedule.manage permission. -# @POST: Schedule is removed. +# #region delete_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule] +# @BRIEF Delete the schedule for a translation job. +# @PRE User has translate.schedule.manage permission. Schedule exists. +# @POST Schedule is removed. APScheduler job removed. +# @SIDE_EFFECT Deletes TranslationSchedule record from DB; removes job from APScheduler. +# @RELATION DEPENDS_ON -> [TranslationScheduler] @router.delete("/jobs/{job_id}/schedule", status_code=status.HTTP_204_NO_CONTENT) async def delete_schedule( job_id: str, @@ -192,19 +204,18 @@ async def delete_schedule( try: scheduler = TranslationScheduler(db, config_manager, current_user.username) scheduler.delete_schedule(job_id) - from ...dependencies import get_scheduler_service + from ....dependencies import get_scheduler_service sched_svc = get_scheduler_service() sched_svc.remove_translation_job(schedule_id=job_id) return None except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:delete_schedule:Function] +# #endregion delete_schedule -# [DEF:get_next_executions:Function] -# @PURPOSE: Preview next N executions for a job's schedule. -# @PRE: User has translate.schedule.view permission. -# @POST: Returns next execution times. +# #region get_next_executions [C:3] [TYPE Function] [SEMANTICS api,translate,schedule] +# @BRIEF Preview next N execution times for a job's schedule. +# @RELATION DEPENDS_ON -> [TranslationScheduler] @router.get("/jobs/{job_id}/schedule/next-executions") async def get_next_executions( job_id: str, @@ -230,6 +241,6 @@ async def get_next_executions( } except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:get_next_executions:Function] +# #endregion get_next_executions -# [/DEF:TranslateScheduleRoutesModule:Module] +# #endregion TranslateScheduleRoutesModule diff --git a/backend/src/models/__init__.py b/backend/src/models/__init__.py index 4aeb09b4..3c21243d 100644 --- a/backend/src/models/__init__.py +++ b/backend/src/models/__init__.py @@ -1,3 +1,2 @@ -# [DEF:ModelsPackage:Package] -# @PURPOSE: Domain model package root. -# [/DEF:ModelsPackage:Package] +# #region ModelsPackage [C:1] [TYPE Package] +# #endregion ModelsPackage diff --git a/backend/src/models/translate.py b/backend/src/models/translate.py index e2f4a0de..f71b96af 100644 --- a/backend/src/models/translate.py +++ b/backend/src/models/translate.py @@ -1,9 +1,6 @@ -# [DEF:TranslateModels:Module] -# @COMPLEXITY: 3 -# @SEMANTICS: translate, models, sqlalchemy, persistence -# @PURPOSE: SQLAlchemy ORM models for LLM-based SQL/dashboard translation across dialects. -# @LAYER: Domain -# @RELATION: INHERITS_FROM -> MappingModels:Base +# #region TranslateModels [C:2] [TYPE Module] +# @BRIEF SQLAlchemy ORM models for LLM-based SQL/dashboard translation across dialects. +# @LAYER Domain from sqlalchemy import ( Column, String, Boolean, DateTime, JSON, Text, @@ -16,14 +13,13 @@ import hashlib from .mapping import Base +# #region generate_uuid [C:1] [TYPE Function] def generate_uuid(): return str(uuid.uuid4()) +# #endregion generate_uuid -# [DEF:TranslationJob:Class] -# @PURPOSE: A translation job representing a multi-dialect conversion task with column mappings, LLM config, and dictionary attachments. -# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. -# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity. +# #region TranslationJob [C:1] [TYPE Class] class TranslationJob(Base): __tablename__ = "translation_jobs" @@ -55,15 +51,15 @@ class TranslationJob(Base): # Environment association environment_id = Column(String, nullable=True, comment="Superset environment ID for datasource access") + target_database_id = Column(String, nullable=True, comment="Superset database ID for INSERT via SQL Lab") created_by = Column(String, nullable=True) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) -# [/DEF:TranslationJob:Class] +# #endregion TranslationJob -# [DEF:TranslationRun:Class] -# @PURPOSE: Represents a single execution of a translation job, capturing status, timing, and Superset execution metadata. +# #region TranslationRun [C:1] [TYPE Class] class TranslationRun(Base): __tablename__ = "translation_runs" @@ -87,11 +83,10 @@ class TranslationRun(Base): dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at run time") created_by = Column(String, nullable=True) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) -# [/DEF:TranslationRun:Class] +# #endregion TranslationRun -# [DEF:TranslationBatch:Class] -# @PURPOSE: Groups translation records within a run into manageable batches with timing and record counts. +# #region TranslationBatch [C:1] [TYPE Class] class TranslationBatch(Base): __tablename__ = "translation_batches" @@ -105,11 +100,10 @@ class TranslationBatch(Base): started_at = Column(DateTime, nullable=True) completed_at = Column(DateTime, nullable=True) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) -# [/DEF:TranslationBatch:Class] +# #endregion TranslationBatch -# [DEF:TranslationRecord:Class] -# @PURPOSE: Individual translation result for a single SQL statement or dashboard element, tracking source, target, and error state. +# #region TranslationRecord [C:1] [TYPE Class] class TranslationRecord(Base): __tablename__ = "translation_records" @@ -126,16 +120,16 @@ class TranslationRecord(Base): token_count_input = Column(Integer, nullable=True) token_count_output = Column(Integer, nullable=True) translation_duration_ms = Column(Integer, nullable=True) + source_data = Column(JSON, nullable=True, comment="Snapshot of source row key/context column values for INSERT phase") created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) __table_args__ = ( Index("ix_translation_records_run_status", "run_id", "status"), ) -# [/DEF:TranslationRecord:Class] +# #endregion TranslationRecord -# [DEF:TranslationEvent:Class] -# @PURPOSE: Audit/event log for translation operations, with optional run_id for context. +# #region TranslationEvent [C:1] [TYPE Class] class TranslationEvent(Base): __tablename__ = "translation_events" @@ -146,11 +140,10 @@ class TranslationEvent(Base): event_data = Column(JSON, nullable=True) created_by = Column(String, nullable=True) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) -# [/DEF:TranslationEvent:Class] +# #endregion TranslationEvent -# [DEF:TranslationPreviewSession:Class] -# @PURPOSE: A preview session allowing users to review proposed translations before applying them. +# #region TranslationPreviewSession [C:1] [TYPE Class] class TranslationPreviewSession(Base): __tablename__ = "translation_preview_sessions" @@ -161,11 +154,10 @@ class TranslationPreviewSession(Base): created_by = Column(String, nullable=True) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) expires_at = Column(DateTime, nullable=True) -# [/DEF:TranslationPreviewSession:Class] +# #endregion TranslationPreviewSession -# [DEF:TranslationPreviewRecord:Class] -# @PURPOSE: Individual preview entry within a preview session, showing original and translated content side by side. +# #region TranslationPreviewRecord [C:1] [TYPE Class] class TranslationPreviewRecord(Base): __tablename__ = "translation_preview_records" @@ -178,12 +170,12 @@ class TranslationPreviewRecord(Base): source_object_name = Column(String, nullable=True) status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED feedback = Column(Text, nullable=True) + source_data = Column(JSON, nullable=True, comment="Snapshot of source row values for key/context columns") created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) -# [/DEF:TranslationPreviewRecord:Class] +# #endregion TranslationPreviewRecord -# [DEF:TerminologyDictionary:Class] -# @PURPOSE: A named collection of terminology mappings used during translation to ensure consistent term translation. +# #region TerminologyDictionary [C:1] [TYPE Class] class TerminologyDictionary(Base): __tablename__ = "terminology_dictionaries" @@ -196,11 +188,10 @@ class TerminologyDictionary(Base): created_by = Column(String, nullable=True) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) -# [/DEF:TerminologyDictionary:Class] +# #endregion TerminologyDictionary -# [DEF:DictionaryEntry:Class] -# @PURPOSE: A single term mapping entry with case-insensitive unique constraint on source_term_normalized and origin tracking. +# #region DictionaryEntry [C:1] [TYPE Class] class DictionaryEntry(Base): __tablename__ = "dictionary_entries" @@ -222,11 +213,10 @@ class DictionaryEntry(Base): name="uq_dictionary_entry_term" ), ) -# [/DEF:DictionaryEntry:Class] +# #endregion DictionaryEntry -# [DEF:TranslationSchedule:Class] -# @PURPOSE: Defines a cron-based schedule for recurring translation jobs. +# #region TranslationSchedule [C:1] [TYPE Class] class TranslationSchedule(Base): __tablename__ = "translation_schedules" @@ -240,11 +230,10 @@ class TranslationSchedule(Base): created_by = Column(String, nullable=True) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) -# [/DEF:TranslationSchedule:Class] +# #endregion TranslationSchedule -# [DEF:TranslationJobDictionary:Class] -# @PURPOSE: Many-to-many association between translation jobs and terminology dictionaries. +# #region TranslationJobDictionary [C:1] [TYPE Class] class TranslationJobDictionary(Base): __tablename__ = "translation_job_dictionaries" @@ -259,11 +248,10 @@ class TranslationJobDictionary(Base): name="uq_job_dictionary" ), ) -# [/DEF:TranslationJobDictionary:Class] +# #endregion TranslationJobDictionary -# [DEF:MetricSnapshot:Class] -# @PURPOSE: Captures translation performance metrics at a point in time with config/content/dictionary hash keys for aggregation. +# #region MetricSnapshot [C:1] [TYPE Class] class MetricSnapshot(Base): __tablename__ = "translation_metric_snapshots" @@ -290,6 +278,6 @@ class MetricSnapshot(Base): __table_args__ = ( Index("ix_metric_snapshots_job_date", "job_id", "snapshot_date"), ) -# [/DEF:MetricSnapshot:Class] +# #endregion MetricSnapshot -# [/DEF:TranslateModels:Module] +# #endregion TranslateModels diff --git a/backend/src/plugins/translate/__init__.py b/backend/src/plugins/translate/__init__.py index b7837466..eb4efb6c 100644 --- a/backend/src/plugins/translate/__init__.py +++ b/backend/src/plugins/translate/__init__.py @@ -1,3 +1,2 @@ -# [DEF:TranslatePluginPackage:Package] -# @PURPOSE: Translation plugin package for LLM-based cross-Superset SQL/dashboard translation. -# [/DEF:TranslatePluginPackage:Package] +# #region TranslatePluginPackage [C:1] [TYPE Package] +# #endregion TranslatePluginPackage diff --git a/backend/src/plugins/translate/__tests__/test_orchestrator.py b/backend/src/plugins/translate/__tests__/test_orchestrator.py index ecdea175..385d5f2b 100644 --- a/backend/src/plugins/translate/__tests__/test_orchestrator.py +++ b/backend/src/plugins/translate/__tests__/test_orchestrator.py @@ -41,6 +41,7 @@ def mock_job() -> MagicMock: job.source_datasource_id = "42" job.translation_column = "name" job.context_columns = ["category"] + job.source_table = "source_table_name" job.target_schema = "public" job.target_table = "translated_data" job.target_key_cols = ["id"] @@ -152,7 +153,6 @@ class TestTranslationOrchestrator: # @PURPOSE: start_run creates a PENDING run with event. def test_start_run_success( self, - mock_service, mock_job: MagicMock, mock_preview_session: MagicMock, ) -> None: diff --git a/backend/src/plugins/translate/_utils.py b/backend/src/plugins/translate/_utils.py index 728d2e9a..0a86825b 100644 --- a/backend/src/plugins/translate/_utils.py +++ b/backend/src/plugins/translate/_utils.py @@ -1,30 +1,22 @@ -# [DEF:DictionaryUtils:Module] -# @COMPLEXITY: 1 -# @SEMANTICS: dictionary, utils, normalize, delimiter -# @PURPOSE: Utility helpers for dictionary management (term normalization and delimiter detection). -# @LAYER: Domain +# #region DictionaryUtils [C:1] [TYPE Module] +# #endregion DictionaryUtils import re import unicodedata -# [DEF:_normalize_term:Function] -# @PURPOSE: Normalize a term for case-insensitive unique constraint lookup. -# @RATIONALE: NFC normalization is applied before lowercasing to ensure consistent -# comparison of Unicode characters (e.g. precomposed vs decomposed forms). -# @REJECTED: Lowercasing without NFC normalization — would cause duplicate entries -# for semantically identical Unicode strings in different normalization forms. +# #region _normalize_term [C:2] [TYPE Function] [SEMANTICS dictionary,normalize] +# @BRIEF Normalize a term for case-insensitive unique constraint lookup using NFC normalization. def _normalize_term(term: str) -> str: """Normalize a term by NFC, lowercasing, and removing extra whitespace.""" if not term: return "" term = unicodedata.normalize('NFC', term) return " ".join(term.lower().split()) -# [/DEF:_normalize_term:Function] +# #endregion _normalize_term -# [DEF:_detect_delimiter:Function] -# @PURPOSE: Detect the delimiter used in a CSV/TSV header line. +# #region _detect_delimiter [C:1] [TYPE Function] [SEMANTICS dictionary,delimiter] def _detect_delimiter(header_line: str) -> str: """Detect delimiter by counting tabs vs commas in the first line.""" if not header_line: @@ -32,4 +24,4 @@ def _detect_delimiter(header_line: str) -> str: tab_count = header_line.count("\t") comma_count = header_line.count(",") return "\t" if tab_count > comma_count else "," -# [/DEF:_detect_delimiter:Function] +# #endregion _detect_delimiter diff --git a/backend/src/plugins/translate/dictionary.py b/backend/src/plugins/translate/dictionary.py index fa2eb8f8..b8b56baf 100644 --- a/backend/src/plugins/translate/dictionary.py +++ b/backend/src/plugins/translate/dictionary.py @@ -1,15 +1,15 @@ -# [DEF:DictionaryManagerModule:Module] -# @COMPLEXITY: 4 -# @SEMANTICS: dictionary, manager, terminology, crud, import, filter -# @PURPOSE: Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering. -# @LAYER: Domain -# @RELATION: DEPENDS_ON -> [TerminologyDictionary:Class] -# @RELATION: DEPENDS_ON -> [DictionaryEntry:Class] -# @RELATION: DEPENDS_ON -> [TranslationJobDictionary:Class] -# @RELATION: DEPENDS_ON -> [TranslationJob:Class] -# @RATIONALE: C4 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term. -# @REJECTED: Pure C3 CRUD without state guards would allow orphaned job-dictionary links. -# @REJECTED: "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing. +# #region DictionaryManagerModule [C:5] [TYPE Module] [SEMANTICS dictionary,manager,terminology,crud,import,filter] +# @BRIEF Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering. +# @LAYER Domain +# @RELATION DEPENDS_ON -> [TerminologyDictionary] +# @RELATION DEPENDS_ON -> [DictionaryEntry] +# @RELATION DEPENDS_ON -> [TranslationJobDictionary] +# @RELATION DEPENDS_ON -> [TranslationJob] +# @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary] +# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) prohibits duplicate entries within a dictionary. +# @RATIONALE C5 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term. +# @REJECTED Pure C3 CRUD without state guards would allow orphaned job-dictionary links. +# @REJECTED "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing. import csv import io @@ -27,18 +27,17 @@ from ...models.translate import ( from ._utils import _normalize_term, _detect_delimiter -# [DEF:DictionaryManager:Class] -# @COMPLEXITY: 4 -# @PURPOSE: Manages terminology dictionaries and their entries with referential integrity. -# @PRE: Database session is open and valid. -# @POST: Dictionary and entry mutations are persisted with conflict detection. -# @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards. +# #region DictionaryManager [C:5] [TYPE Class] [SEMANTICS dictionary,manager,crud,import] +# @BRIEF Manages terminology dictionaries and their entries with referential integrity, import, and batch filtering. +# @PRE Database session is open and valid. +# @POST Dictionary and entry mutations are persisted with conflict detection. +# @SIDE_EFFECT Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards. +# @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary] +# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) enforces no duplicates within a dictionary. class DictionaryManager: - # [DEF:DictionaryManager.create_dictionary:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Create a new terminology dictionary. - # @PRE: payload contains name, source_dialect, target_dialect. - # @POST: New TerminologyDictionary row is created and returned. + # #region DictionaryManager.create_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,create] + # @BRIEF Create a new terminology dictionary with name, source_dialect, and target_dialect. + # @RELATION DEPENDS_ON -> [TerminologyDictionary] @staticmethod def create_dictionary( db: Session, name: str, source_dialect: str, target_dialect: str, @@ -60,13 +59,11 @@ class DictionaryManager: db.refresh(dictionary) logger.reflect("Dictionary created", {"id": dictionary.id}) return dictionary - # [/DEF:DictionaryManager.create_dictionary:Function] + # #endregion DictionaryManager.create_dictionary - # [DEF:DictionaryManager.update_dictionary:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Update an existing terminology dictionary. - # @PRE: dict_id exists in terminology_dictionaries table. - # @POST: Dictionary metadata is updated and returned. + # #region DictionaryManager.update_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,update] + # @BRIEF Update an existing terminology dictionary's metadata fields. + # @RELATION DEPENDS_ON -> [TerminologyDictionary] @staticmethod def update_dictionary( db: Session, dict_id: str, name: Optional[str] = None, @@ -92,14 +89,13 @@ class DictionaryManager: db.refresh(dictionary) logger.reflect("Dictionary updated", {"id": dictionary.id}) return dictionary - # [/DEF:DictionaryManager.update_dictionary:Function] + # #endregion DictionaryManager.update_dictionary - # [DEF:DictionaryManager.delete_dictionary:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Delete a dictionary, blocked if attached to active/scheduled jobs. - # @PRE: dict_id exists. - # @POST: Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs. - # @SIDE_EFFECT: Deletes TerminologyDictionary row and all DictionaryEntry rows. + # #region DictionaryManager.delete_dictionary [C:4] [TYPE Function] [SEMANTICS dictionary,delete] + # @BRIEF Delete a dictionary, blocked if attached to active/scheduled jobs (referential integrity guard). + # @PRE dict_id exists. + # @POST Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs. + # @SIDE_EFFECT Deletes TerminologyDictionary row and all DictionaryEntry rows; checks job attachments first. @staticmethod def delete_dictionary(db: Session, dict_id: str) -> None: with belief_scope("DictionaryManager.delete_dictionary"): @@ -136,26 +132,20 @@ class DictionaryManager: db.delete(dictionary) db.commit() logger.reflect("Dictionary deleted", {"id": dict_id}) - # [/DEF:DictionaryManager.delete_dictionary:Function] + # #endregion DictionaryManager.delete_dictionary - # [DEF:DictionaryManager.get_dictionary:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Get a single dictionary by ID with entry count. - # @PRE: dict_id exists. - # @POST: Returns dict with dictionary + entry_count or raises ValueError. + # #region DictionaryManager.get_dictionary [C:2] [TYPE Function] [SEMANTICS dictionary,get] + # @BRIEF Get a single dictionary by ID with entry count. @staticmethod def get_dictionary(db: Session, dict_id: str) -> TerminologyDictionary: dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() if not dictionary: raise ValueError(f"Dictionary not found: {dict_id}") return dictionary - # [/DEF:DictionaryManager.get_dictionary:Function] + # #endregion DictionaryManager.get_dictionary - # [DEF:DictionaryManager.list_dictionaries:Function] - # @COMPLEXITY: 2 - # @PURPOSE: List dictionaries with pagination and entry counts. - # @PRE: page >= 1, page_size between 1 and 100. - # @POST: Returns (list of dicts, total_count). + # #region DictionaryManager.list_dictionaries [C:2] [TYPE Function] [SEMANTICS dictionary,list] + # @BRIEF List dictionaries with pagination and total count. @staticmethod def list_dictionaries( db: Session, page: int = 1, page_size: int = 20, @@ -169,13 +159,11 @@ class DictionaryManager: .all() ) return dictionaries, total - # [/DEF:DictionaryManager.list_dictionaries:Function] + # #endregion DictionaryManager.list_dictionaries - # [DEF:DictionaryManager.add_entry:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Add an entry to a dictionary, enforcing unique source_term_normalized. - # @PRE: dict_id exists. source_term and target_term are non-empty. - # @POST: New DictionaryEntry row is created or raises on duplicate. + # #region DictionaryManager.add_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,add] + # @BRIEF Add an entry to a dictionary, enforcing unique source_term_normalized per dictionary. + # @RELATION DEPENDS_ON -> [DictionaryEntry] @staticmethod def add_entry( db: Session, dict_id: str, source_term: str, target_term: str, @@ -210,13 +198,11 @@ class DictionaryManager: db.refresh(entry) logger.reflect("Entry added", {"entry_id": entry.id}) return entry - # [/DEF:DictionaryManager.add_entry:Function] + # #endregion DictionaryManager.add_entry - # [DEF:DictionaryManager.edit_entry:Function] - # @COMPLEXITY: 3 - # @PURPOSE: Edit an existing dictionary entry with duplicate-aware normalization. - # @PRE: entry_id exists. - # @POST: Entry fields are updated. + # #region DictionaryManager.edit_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,edit] + # @BRIEF Edit an existing dictionary entry with duplicate-aware normalization check. + # @RELATION DEPENDS_ON -> [DictionaryEntry] @staticmethod def edit_entry( db: Session, entry_id: str, source_term: Optional[str] = None, @@ -255,13 +241,10 @@ class DictionaryManager: db.refresh(entry) logger.reflect("Entry updated", {"entry_id": entry.id}) return entry - # [/DEF:DictionaryManager.edit_entry:Function] + # #endregion DictionaryManager.edit_entry - # [DEF:DictionaryManager.delete_entry:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Delete a single dictionary entry. - # @PRE: entry_id exists. - # @POST: Entry is deleted. + # #region DictionaryManager.delete_entry [C:2] [TYPE Function] [SEMANTICS dictionary,entry,delete] + # @BRIEF Delete a single dictionary entry. @staticmethod def delete_entry(db: Session, entry_id: str) -> None: with belief_scope("DictionaryManager.delete_entry"): @@ -272,13 +255,10 @@ class DictionaryManager: db.delete(entry) db.commit() logger.reflect("Entry deleted", {"entry_id": entry_id}) - # [/DEF:DictionaryManager.delete_entry:Function] + # #endregion DictionaryManager.delete_entry - # [DEF:DictionaryManager.clear_entries:Function] - # @COMPLEXITY: 2 - # @PURPOSE: Delete all entries for a dictionary. - # @PRE: dict_id exists. - # @POST: All entries for the dictionary are deleted. + # #region DictionaryManager.clear_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,clear] + # @BRIEF Delete all entries for a dictionary. @staticmethod def clear_entries(db: Session, dict_id: str) -> int: with belief_scope("DictionaryManager.clear_entries"): @@ -290,13 +270,10 @@ class DictionaryManager: db.commit() logger.reflect("Entries cleared", {"dict_id": dict_id, "count": deleted}) return deleted - # [/DEF:DictionaryManager.clear_entries:Function] + # #endregion DictionaryManager.clear_entries - # [DEF:DictionaryManager.list_entries:Function] - # @COMPLEXITY: 2 - # @PURPOSE: List entries for a dictionary with pagination. - # @PRE: dict_id exists. - # @POST: Returns (list of entries, total_count). + # #region DictionaryManager.list_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,list] + # @BRIEF List entries for a dictionary with pagination. @staticmethod def list_entries( db: Session, dict_id: str, page: int = 1, page_size: int = 50, @@ -316,14 +293,13 @@ class DictionaryManager: .all() ) return entries, total - # [/DEF:DictionaryManager.list_entries:Function] + # #endregion DictionaryManager.list_entries - # [DEF:DictionaryManager.import_entries:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Import entries from CSV/TSV content with duplicate detection and conflict resolution. - # @PRE: content is valid CSV or TSV. dict_id exists. - # @POST: Entries are created/updated/skipped per conflict mode. Returns result summary. - # @SIDE_EFFECT: Batch-inserts or updates DictionaryEntry rows. + # #region DictionaryManager.import_entries [C:4] [TYPE Function] [SEMANTICS dictionary,import,csv] + # @BRIEF Import entries from CSV/TSV content with duplicate detection and conflict resolution (overwrite/keep_existing/cancel). + # @PRE content is valid CSV or TSV. dict_id exists. + # @POST Entries are created/updated/skipped per conflict mode. Returns result summary with preview support. + # @SIDE_EFFECT Batch-inserts or updates DictionaryEntry rows. @staticmethod def import_entries( db: Session, dict_id: str, content: str, @@ -456,14 +432,13 @@ class DictionaryManager: "errors": len(result["errors"]), }) return result - # [/DEF:DictionaryManager.import_entries:Function] + # #endregion DictionaryManager.import_entries - # [DEF:DictionaryManager.filter_for_batch:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job. - # @PRE: job_id exists and source_texts is a list of strings. - # @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries. - # @SIDE_EFFECT: Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables. + # #region DictionaryManager.filter_for_batch [C:4] [TYPE Function] [SEMANTICS dictionary,filter,batch] + # @BRIEF Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job. + # @PRE job_id exists and source_texts is a list of strings. + # @POST Returns list of matched entries with match info, sorted by dictionary link priority. + # @SIDE_EFFECT Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables. @staticmethod def filter_for_batch( db: Session, source_texts: List[str], job_id: str, @@ -554,15 +529,14 @@ class DictionaryManager: "dictionaries_used": len(dictionaries), }) return matched - # [/DEF:DictionaryManager.filter_for_batch:Function] + # #endregion DictionaryManager.filter_for_batch - # [DEF:DictionaryManager.submit_correction:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Submit a term correction from a run result. Validates language match, detects conflicts. - # @PRE: source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists. - # @POST: Entry created or updated; origin tracking populated. Returns action + conflict info. - # @SIDE_EFFECT: Creates/updates DictionaryEntry row. + # #region DictionaryManager.submit_correction [C:4] [TYPE Function] [SEMANTICS dictionary,correction,submit] + # @BRIEF Submit a term correction from a run result with conflict detection and origin tracking. + # @PRE source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists. + # @POST Entry created or updated; origin tracking populated. Returns action + conflict info. + # @SIDE_EFFECT Creates/updates DictionaryEntry row. @staticmethod def submit_correction( db: Session, @@ -653,14 +627,13 @@ class DictionaryManager: db.commit() logger.reflect("Correction processed", result) return result - # [/DEF:DictionaryManager.submit_correction:Function] + # #endregion DictionaryManager.submit_correction - # [DEF:DictionaryManager.submit_bulk_corrections:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Submit multiple term corrections atomically — all succeed or all fail. - # @PRE: corrections list is non-empty. dict_id exists. - # @POST: All corrections applied or none applied with conflict list. - # @SIDE_EFFECT: Creates/updates DictionaryEntry rows; commits once. + # #region DictionaryManager.submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS dictionary,correction,bulk] + # @BRIEF Submit multiple term corrections atomically — all succeed or all fail on conflict. + # @PRE corrections list is non-empty. dict_id exists. + # @POST All corrections applied or none applied with conflict list (atomic). + # @SIDE_EFFECT Creates/updates DictionaryEntry rows; commits once. @staticmethod def submit_bulk_corrections( db: Session, @@ -774,8 +747,8 @@ class DictionaryManager: "total": len(corrections), "applied": sum(1 for r in results if r["action"] in ("created", "updated")), } - # [/DEF:DictionaryManager.submit_bulk_corrections:Function] + # #endregion DictionaryManager.submit_bulk_corrections -# [/DEF:DictionaryManager:Class] -# [/DEF:DictionaryManagerModule:Module] +# #endregion DictionaryManager +# #endregion DictionaryManagerModule diff --git a/backend/src/plugins/translate/events.py b/backend/src/plugins/translate/events.py index 098197bc..6f58bfa2 100644 --- a/backend/src/plugins/translate/events.py +++ b/backend/src/plugins/translate/events.py @@ -1,18 +1,16 @@ -# [DEF:TranslationEventLog:Module] -# @COMPLEXITY: 5 -# @SEMANTICS: translate, events, audit, logging -# @PURPOSE: Structured event logging for translation operations with terminal event invariant enforcement. -# @LAYER: Domain -# @RELATION: DEPENDS_ON -> [TranslationEvent] -# @RELATION: DEPENDS_ON -> [MetricSnapshot] -# @PRE: Database session is open and valid. -# @POST: Events are persisted immutably; terminal events enforce exactly-one invariant per run. -# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion. -# @DATA_CONTRACT: Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent] -# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id. -# @RATIONALE: Immutable event log with nullable run_id allows job-level events (no run context) and run-level events. -# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant. -# @REJECTED: Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit. +# #region TranslationEventLog [C:5] [TYPE Module] [SEMANTICS translate,events,audit,logging] +# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement. +# @LAYER Domain +# @RELATION DEPENDS_ON -> [TranslationEvent] +# @RELATION DEPENDS_ON -> [MetricSnapshot] +# @PRE Database session is open and valid. +# @POST Events are persisted immutably; terminal events enforce exactly-one invariant per run. +# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion. +# @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent] +# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id. +# @RATIONALE Immutable event log with nullable run_id allows job-level events (no run context) and run-level events. +# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant. +# @REJECTED Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit. from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from datetime import datetime, timezone, timedelta @@ -38,24 +36,48 @@ VALID_EVENT_TYPES = TERMINAL_EVENT_TYPES | { DEFAULT_RETENTION_DAYS = 90 -# [DEF:TranslationEventLog:Class] -# @COMPLEXITY: 5 -# @PURPOSE: Structured event logging for translation operations with terminal event invariant enforcement. -# @PRE: Database session is available. -# @POST: Events are written immutably; terminal events enforced per run. -# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events. -# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id. +# #region TranslationEventLog [C:5] [TYPE Class] [SEMANTICS translate,events,log,invariants] +# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement and pruning. +# @PRE Database session is available. +# @POST Events are written immutably; terminal events enforced per run; expired events pruned with MetricSnapshot. +# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events. +# @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent] +# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id. class TranslationEventLog: def __init__(self, db: Session): self.db = db - # [DEF:log_event:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Write an immutable event. Enforces terminal event invariant for non-null run_id. - # @PRE: event_type must be a known type. If run_id is not None, enforce terminal invariant. - # @POST: TranslationEvent row is created. - # @SIDE_EFFECT: DB write. + # #region _create_event_raw [C:3] [TYPE Function] [SEMANTICS translate,events,create] + # @BRIEF Create a TranslationEvent row without invariant checks. Internal helper for callers that manage invariants themselves. + # @RELATION DEPENDS_ON -> [TranslationEvent] + def _create_event_raw( + self, + job_id: str, + event_type: str, + payload: Optional[Dict[str, Any]] = None, + run_id: Optional[str] = None, + created_by: Optional[str] = None, + ) -> TranslationEvent: + event = TranslationEvent( + id=str(uuid.uuid4()), + job_id=job_id, + run_id=run_id, + event_type=event_type, + event_data=payload or {}, + created_by=created_by, + created_at=datetime.now(timezone.utc), + ) + self.db.add(event) + self.db.flush() + return event + # #endregion _create_event_raw + + # #region log_event [C:4] [TYPE Function] [SEMANTICS translate,events,log] + # @BRIEF Write an immutable event. Enforces terminal event invariant and run_started ordering for non-null run_id. + # @PRE event_type must be a known type. If run_id is not None, enforce terminal + start invariants. + # @POST TranslationEvent row is created; invariants checked before write. + # @SIDE_EFFECT DB write. def log_event( self, job_id: str, @@ -82,9 +104,10 @@ class TranslationEventLog: .first() ) if existing_terminal: + existing_id = existing_terminal[0] if isinstance(existing_terminal, (tuple, list)) else str(existing_terminal.id) raise ValueError( f"Run '{run_id}' already has a terminal event " - f"({existing_terminal[0]}). Cannot add another terminal event." + f"({existing_id}). Cannot add another terminal event." ) # Enforce run_started invariant — must exist before other run events @@ -103,18 +126,13 @@ class TranslationEventLog: f"RUN_STARTED event must precede other run events." ) - event = TranslationEvent( - id=str(uuid.uuid4()), + event = self._create_event_raw( job_id=job_id, - run_id=run_id, event_type=event_type, - event_data=payload or {}, + payload=payload, + run_id=run_id, created_by=created_by, - created_at=datetime.now(timezone.utc), ) - self.db.add(event) - self.db.flush() - logger.reason(f"Event logged: {event_type}", { "event_id": event.id, "job_id": job_id, @@ -122,12 +140,10 @@ class TranslationEventLog: "event_type": event_type, }) return event - # [/DEF:log_event:Function] + # #endregion log_event - # [DEF:query_events:Function] - # @PURPOSE: Query events with optional filters. - # @PRE: None. - # @POST: Returns list of TranslationEvent dicts matching filters. + # #region query_events [C:2] [TYPE Function] [SEMANTICS translate,events,query] + # @BRIEF Query events with optional filters (job_id, run_id, event_type) and pagination. def query_events( self, job_id: Optional[str] = None, @@ -165,13 +181,13 @@ class TranslationEventLog: } for e in events ] - # [/DEF:query_events:Function] + # #endregion query_events - # [DEF:prune_expired:Function] - # @PURPOSE: Delete events older than retention_days. Persists MetricSnapshot before pruning. - # @PRE: None. - # @POST: Expired events are deleted; MetricSnapshot is created before deletion. - # @SIDE_EFFECT: Creates MetricSnapshot row; deletes TranslationEvent rows. + # #region prune_expired [C:4] [TYPE Function] [SEMANTICS translate,events,prune] + # @BRIEF Delete events older than retention_days. Creates MetricSnapshot before pruning for audit trail. + # @PRE None. + # @POST Expired events are deleted; MetricSnapshot is created before deletion in batch. + # @SIDE_EFFECT Creates MetricSnapshot row; deletes TranslationEvent rows in batches. def prune_expired( self, retention_days: int = DEFAULT_RETENTION_DAYS, @@ -233,12 +249,11 @@ class TranslationEventLog: "snapshot_id": snapshot_id, }) return {"pruned": pruned, "snapshot_id": snapshot_id} - # [/DEF:prune_expired:Function] + # #endregion prune_expired - # [DEF:get_run_event_summary:Function] - # @PURPOSE: Get a summary of events for a run, including invariant check. - # @PRE: run_id is not None. - # @POST: Returns dict with event list and invariant validity. + # #region get_run_event_summary [C:3] [TYPE Function] [SEMANTICS translate,events,summary] + # @BRIEF Get a summary of events for a run, including invariant validity check. + # @RELATION DEPENDS_ON -> [TranslationEvent] def get_run_event_summary(self, run_id: str) -> Dict[str, Any]: with belief_scope("TranslationEventLog.get_run_event_summary"): events = self.query_events(run_id=run_id) @@ -255,8 +270,35 @@ class TranslationEventLog: "invariant_valid": has_started and len(terminal_events) <= 1, "events": events, } - # [/DEF:get_run_event_summary:Function] + # #endregion get_run_event_summary + + # #region get_run_event_invariants_lightweight [C:3] [TYPE Function] [SEMANTICS translate,events,invariants] + # @BRIEF Lightweight invariant check for a run — fetches only event_type column, avoids event_data for performance. + # @RELATION DEPENDS_ON -> [TranslationEvent] + def get_run_event_invariants_lightweight(self, run_id: str) -> Dict[str, Any]: + with belief_scope("TranslationEventLog.get_run_event_invariants_lightweight"): + # Lightweight query: only fetch event_type column, not event_data + event_types = [ + row[0] + for row in ( + self.db.query(TranslationEvent.event_type) + .filter(TranslationEvent.run_id == run_id) + .order_by(TranslationEvent.created_at.desc()) + .limit(100) + .all() + ) + ] + + has_run_started = "RUN_STARTED" in event_types + terminal_count = sum(1 for et in event_types if et in TERMINAL_EVENT_TYPES) + + return { + "has_run_started": has_run_started, + "terminal_event_count": terminal_count, + "invariant_valid": has_run_started and terminal_count <= 1, + } + # #endregion get_run_event_invariants_lightweight -# [/DEF:TranslationEventLog:Class] -# [/DEF:TranslationEventLog:Module] +# #endregion TranslationEventLog +# #endregion TranslationEventLog diff --git a/backend/src/plugins/translate/executor.py b/backend/src/plugins/translate/executor.py index 6dc9ceab..972268c8 100644 --- a/backend/src/plugins/translate/executor.py +++ b/backend/src/plugins/translate/executor.py @@ -1,19 +1,19 @@ -# [DEF:TranslationExecutor:Module] -# @COMPLEXITY: 4 -# @SEMANTICS: translate, executor, batch, llm -# @PURPOSE: Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows. -# @LAYER: Domain -# @RELATION: DEPENDS_ON -> [TranslationBatch] -# @RELATION: DEPENDS_ON -> [TranslationRecord] -# @RELATION: DEPENDS_ON -> [TranslationRun] -# @RELATION: DEPENDS_ON -> [LLMProviderService] -# @RELATION: DEPENDS_ON -> [DictionaryManager] -# @RELATION: DEPENDS_ON -> [TranslationPreview] -# @PRE: Valid TranslationRun with job configuration. DB session is available. -# @POST: TranslationBatch and TranslationRecord rows are created. Run status is updated. -# @SIDE_EFFECT: Calls LLM provider; creates DB rows; updates run statistics. -# @RATIONALE: Batch processing with retry — independent batches allow partial recovery. -# @REJECTED: Single monolithic LLM call — would lose all progress on any failure. +# #region TranslationExecutor [C:5] [TYPE Module] [SEMANTICS translate,executor,batch,llm] +# @BRIEF Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows. +# @LAYER Domain +# @RELATION DEPENDS_ON -> [TranslationBatch] +# @RELATION DEPENDS_ON -> [TranslationRecord] +# @RELATION DEPENDS_ON -> [TranslationRun] +# @RELATION DEPENDS_ON -> [LLMProviderService] +# @RELATION DEPENDS_ON -> [DictionaryManager] +# @RELATION DEPENDS_ON -> [TranslationPreview] +# @PRE Valid TranslationRun with job configuration. DB session is available. +# @POST TranslationBatch and TranslationRecord rows are created. Run status is updated. +# @SIDE_EFFECT Calls LLM provider; creates DB rows; updates run statistics. +# @DATA_CONTRACT Input[TranslationRun, job] -> Output[TranslationRun with batches and records] +# @INVARIANT Batch processing with retry — independent batches allow partial recovery; MAX_RETRIES_PER_BATCH = 3. +# @RATIONALE Batch processing with retry — independent batches allow partial recovery. +# @REJECTED Single monolithic LLM call — would lose all progress on any failure. import json import time @@ -35,21 +35,23 @@ from ...models.translate import ( ) from ...services.llm_provider import LLMProviderService from ...services.llm_prompt_templates import render_prompt +from ...core.superset_client import SupersetClient from .dictionary import DictionaryManager from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE -# [DEF:MAX_RETRIES_PER_BATCH:Constant] -# @PURPOSE: Maximum number of retries for a single batch before marking it failed. +# #region MAX_RETRIES_PER_BATCH [C:1] [TYPE Constant] [SEMANTICS translate,executor,retry] MAX_RETRIES_PER_BATCH = 3 +# #endregion MAX_RETRIES_PER_BATCH -# [DEF:TranslationExecutor:Class] -# @COMPLEXITY: 4 -# @PURPOSE: Process translation batches: fetch source rows, filter dict, call LLM, persist results. -# @PRE: DB session and config manager available. -# @POST: Batches and records created with status tracking. -# @SIDE_EFFECT: LLM API calls; DB writes. +# #region TranslationExecutor [C:5] [TYPE Class] [SEMANTICS translate,executor,batch] +# @BRIEF Process translation batches: fetch source rows, filter dict, call LLM, persist results. +# @PRE DB session and config manager available. +# @POST Batches and records created with status tracking; run statistics updated. +# @SIDE_EFFECT LLM API calls; DB writes. +# @DATA_CONTRACT Input[TranslationRun] -> Output[TranslationRun with batches, records, stats] +# @INVARIANT MAX_RETRIES_PER_BATCH limit enforced; partial batch success tracked via COMPLETED_WITH_ERRORS. class TranslationExecutor: def __init__( @@ -65,15 +67,16 @@ class TranslationExecutor: self.on_batch_progress = on_batch_progress self._current_run_id: Optional[str] = None - # [DEF:execute_run:Function] - # @PURPOSE: Run full translation execution for a TranslationRun. - # @PRE: run is in PENDING or RUNNING status with valid job config. - # @POST: Run is populated with batches and records. - # @SIDE_EFFECT: LLM API calls; DB batch writes. + # #region execute_run [C:4] [TYPE Function] [SEMANTICS translate,run,execute] + # @BRIEF Run full translation execution for a TranslationRun: fetch rows, batch, call LLM, persist. + # @PRE run is in PENDING or RUNNING status with valid job config. + # @POST Run is populated with batches and records; run status updated to COMPLETED/FAILED. + # @SIDE_EFFECT LLM API calls; DB batch writes. def execute_run( self, run: TranslationRun, llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None, + full_translation: bool = False, ) -> TranslationRun: with belief_scope("TranslationExecutor.execute_run"): job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first() @@ -84,6 +87,7 @@ class TranslationExecutor: "run_id": run.id, "job_id": job.id, "batch_size": job.batch_size, + "full_translation": full_translation, }) # Mark run as RUNNING @@ -91,8 +95,13 @@ class TranslationExecutor: run.started_at = datetime.now(timezone.utc) self.db.flush() - # Fetch source rows from the accepted preview session - source_rows = self._fetch_source_rows(job.id, run.id) + # Fetch source rows + if full_translation: + # Full translation: fetch ALL rows from Superset dataset + source_rows = self._fetch_all_rows_from_superset(job) + else: + # Preview-based: fetch rows from the accepted preview session + source_rows = self._fetch_source_rows(job.id, run.id) if not source_rows: logger.explore("No source rows to translate", {"run_id": run.id}) run.status = "COMPLETED" @@ -164,12 +173,13 @@ class TranslationExecutor: }) return run - # [/DEF:execute_run:Function] + # #endregion execute_run - # [DEF:_fetch_source_rows:Function] - # @PURPOSE: Fetch source rows from the accepted preview session for this job. - # @PRE: job_id exists. - # @POST: Returns list of dicts with source data. + # #region _fetch_source_rows [C:4] [TYPE Function] [SEMANTICS translate,source,rows] + # @BRIEF Fetch source rows from the accepted preview session for this job. + # @PRE job_id exists. + # @POST Returns list of dicts with source data (row_index, source_text, approved_translation, etc). + # @SIDE_EFFECT Queries preview session and records from DB. def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]: with belief_scope("TranslationExecutor._fetch_source_rows"): # Get the latest APPLIED preview session @@ -203,6 +213,7 @@ class TranslationExecutor: "source_text": rec.source_sql or "", "approved_translation": rec.target_sql if rec.status == "APPROVED" else None, "source_object_name": rec.source_object_name or "", + "source_data": rec.source_data or {}, }) logger.reason(f"Fetched {len(source_rows)} source rows from preview", { @@ -210,13 +221,115 @@ class TranslationExecutor: "session_id": session.id, }) return source_rows - # [/DEF:_fetch_source_rows:Function] + # #endregion _fetch_source_rows - # [DEF:_process_batch:Function] - # @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records. - # @PRE: job and batch_rows are valid. - # @POST: TranslationBatch and TranslationRecord rows are created. - # @SIDE_EFFECT: LLM API call. + # #region _fetch_all_rows_from_superset [C:4] [TYPE Function] [SEMANTICS translate,superset,rows] + # @BRIEF Fetch ALL rows from the Superset dataset for full translation (paginated). + # @PRE job has source_datasource_id and environment_id configured. + # @POST Returns list of row dicts with row_index, source_text, source_data. + # @SIDE_EFFECT Calls Superset chart data endpoint (paginated). + def _fetch_all_rows_from_superset(self, job: TranslationJob) -> List[Dict[str, Any]]: + with belief_scope("TranslationExecutor._fetch_all_rows_from_superset"): + env_id = job.environment_id or job.source_dialect or "" + environments = self.config_manager.get_environments() + env_config = next( + (e for e in environments if e.id == env_id), + None, + ) + if not env_config: + raise ValueError(f"Superset environment '{env_id}' not found") + + client = SupersetClient(env_config) + dataset_detail = client.get_dataset_detail(int(job.source_datasource_id)) + + # Build query context (same as preview, but with large row_limit) + query_context = client.build_dataset_preview_query_context( + dataset_id=int(job.source_datasource_id), + dataset_record=dataset_detail, + template_params={}, + effective_filters=[], + ) + + queries = query_context.get("queries", []) + if queries: + queries[0]["row_limit"] = 100000 # Superset max default + queries[0].pop("result_type", None) + queries[0].pop("columns", None) + queries[0]["metrics"] = [] + queries[0]["row_offset"] = 0 + query_context["result_type"] = "samples" + form_data = query_context.get("form_data", {}) + form_data.pop("query_mode", None) + + all_rows: List[Dict[str, Any]] = [] + batch_size_fetch = 10000 # Fetch 10K rows per pagination request + + while True: + # Set pagination for this batch + if queries: + queries[0]["row_limit"] = min(batch_size_fetch, 100000) + queries[0]["row_offset"] = len(all_rows) + + try: + response = client.network.request( + method="POST", + endpoint="/api/v1/chart/data", + data=json.dumps(query_context), + headers={"Content-Type": "application/json"}, + ) + except Exception as e: + logger.explore("Chart data API failed during full fetch", { + "offset": len(all_rows), + "error": str(e), + }) + if not all_rows: + raise ValueError(f"Failed to fetch data from Superset: {e}") + break # Return what we have + + from .preview import TranslationPreview + rows = TranslationPreview._extract_data_rows(response) + if not rows: + break # No more data + + all_rows.extend(rows) + logger.reason(f"Fetched {len(rows)} rows (total: {len(all_rows)})", { + "offset": len(all_rows) - len(rows), + }) + + if len(rows) < batch_size_fetch: + break # Last page + + # Convert to the format expected by _process_batch + source_rows = [] + for idx, row in enumerate(all_rows): + translation_value = str(row.get(job.translation_column, "") or "") + context_values = {} + if job.context_columns: + for col in job.context_columns: + context_values[col] = str(row.get(col, "") or "") + # Also include target_key_cols values in context_data + if job.target_key_cols: + for col in job.target_key_cols: + context_values[col] = str(row.get(col, "") or "") + source_rows.append({ + "row_index": str(idx), + "source_text": translation_value, + "source_data": context_values, + "source_object_name": f"Row {idx + 1}", + "approved_translation": None, + }) + + logger.reason(f"Prepared {len(source_rows)} source rows for full translation", { + "job_id": job.id, + }) + return source_rows + # #endregion _fetch_all_rows_from_superset + + # #region _process_batch [C:4] [TYPE Function] [SEMANTICS translate,batch,process] + # @BRIEF Process a single batch: filter dictionary, build prompt, call LLM, persist records. + # @PRE job and batch_rows are valid. + # @POST TranslationBatch and TranslationRecord rows are created. + # @SIDE_EFFECT LLM API call; DB writes. def _process_batch( self, job: TranslationJob, @@ -275,6 +388,7 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), + source_data=row.get("source_data", {}), status="SUCCESS", ) self.db.add(record) @@ -309,13 +423,13 @@ class TranslationExecutor: }) return result - # [/DEF:_process_batch:Function] + # #endregion _process_batch - # [DEF:_call_llm_for_batch:Function] - # @PURPOSE: Call LLM for a batch of rows requiring translation. Parse structured JSON response. - # @PRE: job has valid provider_id. batch_rows is non-empty. - # @POST: Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows. - # @SIDE_EFFECT: HTTP call to LLM provider. + # #region _call_llm_for_batch [C:4] [TYPE Function] [SEMANTICS translate,llm,batch] + # @BRIEF Call LLM for a batch of rows requiring translation. Parse structured JSON response. + # @PRE job has valid provider_id. batch_rows is non-empty. + # @POST Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows. + # @SIDE_EFFECT HTTP call to LLM provider. def _call_llm_for_batch( self, job: TranslationJob, @@ -401,6 +515,7 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), + source_data=row.get("source_data", {}), status="FAILED", error_message=f"LLM call failed after {retries} retries: {last_error}", ) @@ -428,6 +543,7 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), + source_data=row.get("source_data", {}), status="SKIPPED", error_message=f"LLM parse failure: {e}", ) @@ -459,6 +575,7 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), + source_data=row.get("source_data", {}), status="SKIPPED", error_message="NULL translation returned by LLM", ) @@ -477,6 +594,7 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), + source_data=row.get("source_data", {}), status="SKIPPED", error_message="Empty translation returned by LLM", ) @@ -493,6 +611,7 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), + source_data=row.get("source_data", {}), status="SUCCESS", ) self.db.add(record) @@ -503,13 +622,13 @@ class TranslationExecutor: "skipped": skipped, "retries": retries, } - # [/DEF:_call_llm_for_batch:Function] + # #endregion _call_llm_for_batch - # [DEF:_call_llm:Function] - # @PURPOSE: Call the configured LLM provider with the batch prompt. - # @PRE: job has valid provider_id. - # @POST: Returns raw LLM response string. - # @SIDE_EFFECT: HTTP call to LLM provider. + # #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call] + # @BRIEF Call the configured LLM provider with the batch prompt, routing by provider type. + # @PRE job has valid provider_id. + # @POST Returns raw LLM response string. + # @SIDE_EFFECT HTTP call to LLM provider. def _call_llm(self, job: TranslationJob, prompt: str) -> str: with belief_scope("TranslationExecutor._call_llm"): if not job.provider_id: @@ -537,13 +656,13 @@ class TranslationExecutor: ) else: raise ValueError(f"Unsupported provider type '{provider_type}'") - # [/DEF:_call_llm:Function] + # #endregion _call_llm - # [DEF:_call_openai_compatible:Function] - # @PURPOSE: Call OpenAI-compatible API for batch translation. - # @PRE: Valid API endpoint, key, model, and prompt. - # @POST: Returns response text. - # @SIDE_EFFECT: HTTP POST to LLM API. + # #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai] + # @BRIEF Call OpenAI-compatible API for batch translation with retry and structured output support. + # @PRE Valid API endpoint, key, model, and prompt. + # @POST Returns response text. + # @SIDE_EFFECT HTTP POST to LLM API. @staticmethod def _call_openai_compatible( base_url: str, @@ -596,15 +715,26 @@ class TranslationExecutor: content = choices[0].get("message", {}).get("content", "") if not content: - raise ValueError("LLM returned empty content") + # Log full response for diagnostics + finish_reason = choices[0].get("finish_reason", "unknown") + logger.explore("LLM returned empty content", { + "finish_reason": finish_reason, + "model": payload.get("model"), + "prompt_len": len(prompt), + "response_keys": list(data.keys()), + "choices_count": len(choices), + }) + raise ValueError( + f"LLM returned empty content (finish_reason={finish_reason}, " + f"model={payload.get('model')}, prompt_len={len(prompt)})" + ) return content - # [/DEF:_call_openai_compatible:Function] + # #endregion _call_openai_compatible - # [DEF:_parse_llm_response:Function] - # @PURPOSE: Parse LLM JSON response into dict of row_id -> translation. - # @PRE: response_text is valid JSON with {"rows": [...]} structure. - # @POST: Returns dict mapping row_id to translation text. + # #region _parse_llm_response [C:3] [TYPE Function] [SEMANTICS translate,llm,parse] + # @BRIEF Parse LLM JSON response into dict of row_id -> translation text. + # @RELATION DEPENDS_ON -> [json] @staticmethod def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]: with belief_scope("TranslationExecutor._parse_llm_response"): @@ -637,8 +767,8 @@ class TranslationExecutor: translations[row_id] = str(translation) return translations - # [/DEF:_parse_llm_response:Function] + # #endregion _parse_llm_response -# [/DEF:TranslationExecutor:Class] -# [/DEF:TranslationExecutor:Module] +# #endregion TranslationExecutor +# #endregion TranslationExecutor diff --git a/backend/src/plugins/translate/metrics.py b/backend/src/plugins/translate/metrics.py index c39d977c..50b53a2f 100644 --- a/backend/src/plugins/translate/metrics.py +++ b/backend/src/plugins/translate/metrics.py @@ -1,16 +1,10 @@ -# [DEF:TranslationMetrics:Module] -# @COMPLEXITY: 3 -# @SEMANTICS: translate, metrics, aggregation -# @PURPOSE: Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting. -# @LAYER: Domain -# @RELATION: DEPENDS_ON -> [TranslationEvent:Class] -# @RELATION: DEPENDS_ON -> [MetricSnapshot:Class] -# @RELATION: DEPENDS_ON -> [TranslationRun:Class] -# @RELATION: DEPENDS_ON -> [TranslationSchedule:Class] -# @PRE: Database session is open. -# @POST: Metrics are aggregated and returned; no side effects. -# @RATIONALE: Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture. -# @REJECTED: Querying all events for all time — would be slow; MetricSnapshot provides historical summary. +# #region TranslationMetrics [C:3] [TYPE Module] [SEMANTICS translate,metrics,aggregation] +# @BRIEF Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting. +# @LAYER Domain +# @RELATION DEPENDS_ON -> [TranslationEvent] +# @RELATION DEPENDS_ON -> [MetricSnapshot] +# @RELATION DEPENDS_ON -> [TranslationRun] +# @RELATION DEPENDS_ON -> [TranslationSchedule] from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Tuple @@ -26,18 +20,18 @@ from ...models.translate import ( ) -# [DEF:TranslationMetrics:Class] -# @COMPLEXITY: 3 -# @PURPOSE: Aggregate translation metrics from live events and MetricSnapshot. +# #region TranslationMetrics [C:3] [TYPE Class] [SEMANTICS translate,metrics] +# @BRIEF Aggregate translation metrics from live events and MetricSnapshot. +# @RELATION DEPENDS_ON -> [TranslationRun] +# @RELATION DEPENDS_ON -> [MetricSnapshot] +# @RELATION DEPENDS_ON -> [TranslationEvent] class TranslationMetrics: def __init__(self, db: Session): self.db = db - # [DEF:get_job_metrics:Function] - # @PURPOSE: Get aggregated metrics for a specific job. - # @PRE: job_id exists. - # @POST: Returns dict with metrics from events + latest snapshot. + # #region get_job_metrics [C:2] [TYPE Function] [SEMANTICS translate,metrics,job] + # @BRIEF Get aggregated metrics for a specific job including run counts, record stats, and duration. def get_job_metrics(self, job_id: str) -> Dict[str, Any]: with belief_scope("TranslationMetrics.get_job_metrics"): # Run counts from TranslationRun @@ -151,11 +145,10 @@ class TranslationMetrics: "last_run_at": last_run.created_at.isoformat() if last_run else None, "next_scheduled_run": next_schedule.last_run_at.isoformat() if next_schedule and next_schedule.last_run_at else None, } - # [/DEF:get_job_metrics:Function] + # #endregion get_job_metrics - # [DEF:get_all_metrics:Function] - # @PURPOSE: Get aggregated metrics for all jobs. - # @POST: Returns list of per-job metrics. + # #region get_all_metrics [C:2] [TYPE Function] [SEMANTICS translate,metrics,all] + # @BRIEF Get aggregated metrics for all jobs. def get_all_metrics(self) -> List[Dict[str, Any]]: with belief_scope("TranslationMetrics.get_all_metrics"): job_ids = ( @@ -164,8 +157,8 @@ class TranslationMetrics: .all() ) return [self.get_job_metrics(jid[0]) for jid in job_ids if jid[0]] - # [/DEF:get_all_metrics:Function] + # #endregion get_all_metrics -# [/DEF:TranslationMetrics:Class] -# [/DEF:TranslationMetrics:Module] +# #endregion TranslationMetrics +# #endregion TranslationMetrics diff --git a/backend/src/plugins/translate/orchestrator.py b/backend/src/plugins/translate/orchestrator.py index fd6bc2e5..a4e8cc7d 100644 --- a/backend/src/plugins/translate/orchestrator.py +++ b/backend/src/plugins/translate/orchestrator.py @@ -1,23 +1,21 @@ -# [DEF:TranslationOrchestrator:Module] -# @COMPLEXITY: 5 -# @SEMANTICS: translate, orchestrator, lifecycle, run -# @PURPOSE: Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events. -# @LAYER: Domain -# @RELATION: DEPENDS_ON -> [TranslationRun] -# @RELATION: DEPENDS_ON -> [TranslationJob] -# @RELATION: DEPENDS_ON -> [TranslationExecutor] -# @RELATION: DEPENDS_ON -> [SQLGenerator] -# @RELATION: DEPENDS_ON -> [SupersetSqlLabExecutor] -# @RELATION: DEPENDS_ON -> [TranslationEventLog] -# @RELATION: DEPENDS_ON -> [TranslationPreviewSession] -# @PRE: Valid job and accepted preview (for manual runs). Superset and LLM are reachable. -# @POST: Translation run is executed, SQL generated and submitted, events recorded. -# @SIDE_EFFECT: Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events. -# @DATA_CONTRACT: Input[db, config_manager, current_user] -> Output[TranslationRun] -# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run. -# @RATIONALE: C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary. -# @REJECTED: Distributed actor model (Celery) — eventual-consistency challenges at current scale. -# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant. +# #region TranslationOrchestrator [C:5] [TYPE Module] [SEMANTICS translate,orchestrator,lifecycle,run] +# @BRIEF Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events. +# @LAYER Domain +# @RELATION DEPENDS_ON -> [TranslationRun] +# @RELATION DEPENDS_ON -> [TranslationJob] +# @RELATION DEPENDS_ON -> [TranslationExecutor] +# @RELATION DEPENDS_ON -> [SQLGenerator] +# @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor] +# @RELATION DEPENDS_ON -> [TranslationEventLog] +# @RELATION DEPENDS_ON -> [TranslationPreviewSession] +# @PRE Valid job and accepted preview (for manual runs). Superset and LLM are reachable. +# @POST Translation run is executed, SQL generated and submitted, events recorded. +# @SIDE_EFFECT Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events. +# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun] +# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run. +# @RATIONALE C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary. +# @REJECTED Distributed actor model (Celery) — eventual-consistency challenges at current scale. +# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant. import json import time @@ -44,13 +42,13 @@ from .events import TranslationEventLog from ..translate.service import TranslateJobService -# [DEF:TranslationOrchestrator:Class] -# @COMPLEXITY: 5 -# @PURPOSE: Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging. -# @PRE: DB session and config manager are available. -# @POST: Runs are created, executed, and finalized with event records. -# @SIDE_EFFECT: Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows. -# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. +# #region TranslationOrchestrator [C:5] [TYPE Class] [SEMANTICS translate,orchestrator] +# @BRIEF Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging. +# @PRE DB session and config manager are available. +# @POST Runs are created, executed, and finalized with event records. +# @SIDE_EFFECT Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows. +# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. +# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun] class TranslationOrchestrator: def __init__( @@ -65,12 +63,11 @@ class TranslationOrchestrator: self.event_log = TranslationEventLog(db) self._job: Optional[TranslationJob] = None - # [DEF:start_run:Function] - # @COMPLEXITY: 5 - # @PURPOSE: Start a new translation run for a job. - # @PRE: job_id exists. For manual runs, there must be an accepted preview session. - # @POST: TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded. - # @SIDE_EFFECT: DB writes. + # #region start_run [C:5] [TYPE Function] [SEMANTICS translate,run,create] + # @BRIEF Start a new translation run for a job with config snapshot and hash computation. + # @PRE job_id exists. For manual runs, there must be an accepted preview session. + # @POST TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded. + # @SIDE_EFFECT DB writes; event logging. def start_run( self, job_id: str, @@ -166,18 +163,19 @@ class TranslationOrchestrator: "trigger_type": run.trigger_type, }) return run - # [/DEF:start_run:Function] + # #endregion start_run - # [DEF:execute_run:Function] - # @PURPOSE: Execute a translation run: dispatch executor, generate SQL, submit to Superset. - # @PRE: run is in PENDING status. - # @POST: Run is executed, SQL generated, Superset submission attempted. - # @SIDE_EFFECT: LLM calls, DB writes, Superset API calls. + # #region execute_run [C:5] [TYPE Function] [SEMANTICS translate,run,execute] + # @BRIEF Execute a translation run: dispatch executor, generate SQL, submit to Superset. + # @PRE run is in PENDING status. + # @POST Run is executed, SQL generated, Superset submission attempted. + # @SIDE_EFFECT LLM calls, DB writes, Superset API calls. def execute_run( self, run: TranslationRun, on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None, skip_insert: bool = False, + full_translation: bool = False, ) -> TranslationRun: with belief_scope("TranslationOrchestrator.execute_run"): job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first() @@ -212,7 +210,7 @@ class TranslationOrchestrator: on_batch_progress=on_batch_progress, ) try: - run = executor.execute_run(run, llm_progress_callback=None) + run = executor.execute_run(run, llm_progress_callback=None, full_translation=full_translation) except Exception as e: logger.explore("Translation execution failed", { "run_id": run.id, @@ -299,13 +297,13 @@ class TranslationOrchestrator: "insert_status": run.insert_status, }) return run - # [/DEF:execute_run:Function] + # #endregion execute_run - # [DEF:_generate_and_insert_sql:Function] - # @PURPOSE: Generate INSERT SQL from successful records and submit to Superset SQL Lab. - # @PRE: job has target table configured. run has successful records. - # @POST: SQL is generated and submitted. Returns execution result. - # @SIDE_EFFECT: Superset API call. + # #region _generate_and_insert_sql [C:5] [TYPE Function] [SEMANTICS translate,sql,insert] + # @BRIEF Generate INSERT SQL from successful records and submit to Superset SQL Lab. + # @PRE job has target table configured. run has successful records. + # @POST SQL is generated and submitted. Returns execution result. + # @SIDE_EFFECT Superset API call; event logging. def _generate_and_insert_sql( self, job: TranslationJob, @@ -333,11 +331,12 @@ class TranslationOrchestrator: }) # Build rows for SQL generation - columns = job.context_columns or [] - if job.translation_column and job.translation_column not in columns: + # Only include the translation column — context columns are for LLM only + columns = [] + if job.translation_column: columns.append(job.translation_column) - # Also include key columns if used for upsert + # Include key columns if present (for UPSERT matching) if job.target_key_cols: for k in job.target_key_cols: if k not in columns: @@ -346,14 +345,13 @@ class TranslationOrchestrator: rows_for_sql = [] for rec in records: row_data = {} - if job.context_columns: - for col in job.context_columns: - row_data[col] = "" if job.translation_column: row_data[job.translation_column] = rec.target_sql or "" if job.target_key_cols: + source_data = rec.source_data or {} for k in job.target_key_cols: - row_data[k] = "" + # Use source_data value if available, fall back to empty string + row_data[k] = source_data.get(k, "") rows_for_sql.append(row_data) if not columns: @@ -377,6 +375,16 @@ class TranslationOrchestrator: return {"status": "failed", "error_message": str(e), "query_id": None} # Log insert phase start + logger.reason("Insert SQL generated", { + "run_id": run.id, + "sql_preview": sql[:500], + "sql_length": len(sql), + "row_count": row_count, + "dialect": job.database_dialect or job.target_dialect or "postgresql", + "columns": columns, + "has_key_cols": bool(job.target_key_cols), + }) + self.event_log.log_event( job_id=job.id, run_id=run.id, @@ -388,7 +396,18 @@ class TranslationOrchestrator: # Submit to Superset try: env_id = job.environment_id or job.source_dialect or "" - executor = SupersetSqlLabExecutor(self.config_manager, env_id) + # Resolve database_id: explicit target_database_id from job (int or UUID), or resolve from environment + target_db_id = None + if job.target_database_id: + try: + target_db_id = int(job.target_database_id) + except (ValueError, TypeError): + # Could be a UUID — pass as-is, executor will resolve it + target_db_id = job.target_database_id + logger.reason("target_database_id is not an integer, will resolve as UUID", { + "target_database_id": job.target_database_id, + }) + executor = SupersetSqlLabExecutor(self.config_manager, env_id, database_id=target_db_id) result = executor.execute_and_poll( sql=sql, max_polls=30, @@ -411,13 +430,13 @@ class TranslationOrchestrator: ) return result - # [/DEF:_generate_and_insert_sql:Function] + # #endregion _generate_and_insert_sql - # [DEF:_validate_preconditions:Function] - # @PURPOSE: Validate preconditions before starting a run. - # @PRE: None. - # @POST: Raises ValueError if preconditions are not met. - # @SIDE_EFFECT: None. + # #region _validate_preconditions [C:5] [TYPE Function] [SEMANTICS translate,validation] + # @BRIEF Validate preconditions before starting a translation run. + # @PRE None. + # @POST Raises ValueError if preconditions are not met. + # @SIDE_EFFECT None. def _validate_preconditions( self, job: TranslationJob, @@ -473,13 +492,13 @@ class TranslationOrchestrator: "job_id": job.id, "is_scheduled": is_scheduled, }) - # [/DEF:_validate_preconditions:Function] + # #endregion _validate_preconditions - # [DEF:retry_failed_batches:Function] - # @PURPOSE: Retry failed batches in a run. - # @PRE: run exists and has failed batches. - # @POST: Failed batches are re-processed. - # @SIDE_EFFECT: LLM calls, DB writes. + # #region retry_failed_batches [C:5] [TYPE Function] [SEMANTICS translate,retry,batch] + # @BRIEF Retry failed batches in a run by re-processing failed records. + # @PRE run exists and has failed batches. + # @POST Failed batches are re-processed; run status and stats updated. + # @SIDE_EFFECT LLM calls; DB writes; event logging. def retry_failed_batches( self, run_id: str, @@ -588,13 +607,13 @@ class TranslationOrchestrator: "status": run.status, }) return run - # [/DEF:retry_failed_batches:Function] + # #endregion retry_failed_batches - # [DEF:retry_insert:Function] - # @PURPOSE: Retry the SQL insert phase for a completed run. - # @PRE: run exists and has successful records. - # @POST: SQL is regenerated and re-submitted to Superset. - # @SIDE_EFFECT: Superset API call. + # #region retry_insert [C:5] [TYPE Function] [SEMANTICS translate,retry,insert] + # @BRIEF Retry the SQL insert phase for a completed run. + # @PRE run exists and has successful records. + # @POST SQL is regenerated and re-submitted to Superset. + # @SIDE_EFFECT Superset API call; event logging. def retry_insert( self, run_id: str, @@ -638,13 +657,13 @@ class TranslationOrchestrator: "insert_status": run.insert_status, }) return run - # [/DEF:retry_insert:Function] + # #endregion retry_insert - # [DEF:cancel_run:Function] - # @PURPOSE: Cancel a running translation. - # @PRE: run is in PENDING or RUNNING status. - # @POST: Run status is set to CANCELLED. - # @SIDE_EFFECT: DB write; records event. + # #region cancel_run [C:5] [TYPE Function] [SEMANTICS translate,run,cancel] + # @BRIEF Cancel a running translation run. + # @PRE run is in PENDING or RUNNING status. + # @POST Run status is set to CANCELLED; event recorded. + # @SIDE_EFFECT DB write; records event. def cancel_run(self, run_id: str) -> TranslationRun: with belief_scope("TranslationOrchestrator.cancel_run"): run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first() @@ -661,7 +680,8 @@ class TranslationOrchestrator: run.completed_at = datetime.now(timezone.utc) self.db.flush() - self.event_log.log_event( + # Record terminal event directly — we are the source of the terminal event + self.event_log._create_event_raw( job_id=run.job_id, run_id=run.id, event_type="RUN_CANCELLED", @@ -676,12 +696,12 @@ class TranslationOrchestrator: "run_id": run_id, }) return run - # [/DEF:cancel_run:Function] + # #endregion cancel_run - # [DEF:get_run_status:Function] - # @PURPOSE: Get run status with statistics. - # @PRE: run_id exists. - # @POST: Returns dict with run details. + # #region get_run_status [C:5] [TYPE Function] [SEMANTICS translate,run,status] + # @BRIEF Get run status with statistics and event invariant checks. + # @PRE run_id exists. + # @POST Returns dict with run details. def get_run_status(self, run_id: str) -> Dict[str, Any]: with belief_scope("TranslationOrchestrator.get_run_status"): run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first() @@ -695,8 +715,8 @@ class TranslationOrchestrator: .count() ) - # Get event summary - event_summary = self.event_log.get_run_event_summary(run_id) + # Get event invariants (lightweight — only fetches event_type column) + invariants = self.event_log.get_run_event_invariants_lightweight(run_id) return { "id": run.id, @@ -712,20 +732,16 @@ class TranslationOrchestrator: "insert_status": run.insert_status, "superset_execution_id": run.superset_execution_id, "batch_count": batch_count, - "event_invariants": { - "has_run_started": event_summary["has_run_started"], - "terminal_event_count": event_summary["terminal_event_count"], - "invariant_valid": event_summary["invariant_valid"], - }, + "event_invariants": invariants, "created_by": run.created_by, "created_at": run.created_at.isoformat() if run.created_at else None, } - # [/DEF:get_run_status:Function] + # #endregion get_run_status - # [DEF:get_run_records:Function] - # @PURPOSE: Get paginated records for a run. - # @PRE: run_id exists. - # @POST: Returns dict with records and pagination info. + # #region get_run_records [C:5] [TYPE Function] [SEMANTICS translate,run,records] + # @BRIEF Get paginated records for a run with optional status filter. + # @PRE run_id exists. + # @POST Returns dict with records and pagination info. def get_run_records( self, run_id: str, @@ -771,12 +787,12 @@ class TranslationOrchestrator: "page_size": page_size, "status_filter": status_filter, } - # [/DEF:get_run_records:Function] + # #endregion get_run_records - # [DEF:get_run_history:Function] - # @PURPOSE: Get run history for a job. - # @PRE: job_id exists. - # @POST: Returns list of runs. + # #region get_run_history [C:5] [TYPE Function] [SEMANTICS translate,run,history] + # @BRIEF Get paginated run history for a job. + # @PRE job_id exists. + # @POST Returns tuple of (total_count, list_of_runs). def get_run_history( self, job_id: str, @@ -814,10 +830,10 @@ class TranslationOrchestrator: } for r in runs ] - # [/DEF:get_run_history:Function] + # #endregion get_run_history - # [DEF:_compute_config_hash:Function] - # @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison. + # #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash] + # @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison. @staticmethod def _compute_config_hash(job: TranslationJob) -> str: import hashlib @@ -833,10 +849,10 @@ class TranslationOrchestrator: "upsert_strategy": job.upsert_strategy, }, sort_keys=True) return hashlib.sha256(config_str.encode()).hexdigest()[:16] - # [/DEF:_compute_config_hash:Function] + # #endregion _compute_config_hash - # [DEF:_compute_dict_snapshot_hash:Function] - # @PURPOSE: Compute a hash of dictionary state for snapshot comparison. + # #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash] + # @BRIEF Compute a SHA-256 hash of dictionary state for snapshot comparison. def _compute_dict_snapshot_hash(self, job_id: str) -> str: import hashlib from ...models.translate import TranslationJobDictionary @@ -848,8 +864,8 @@ class TranslationOrchestrator: dict_ids = sorted([dl.dictionary_id for dl in dict_links]) hash_input = ",".join(dict_ids) return hashlib.sha256(hash_input.encode()).hexdigest()[:16] - # [/DEF:_compute_dict_snapshot_hash:Function] + # #endregion _compute_dict_snapshot_hash -# [/DEF:TranslationOrchestrator:Class] -# [/DEF:TranslationOrchestrator:Module] +# #endregion TranslationOrchestrator +# #endregion TranslationOrchestrator diff --git a/backend/src/plugins/translate/plugin.py b/backend/src/plugins/translate/plugin.py index 9112301c..940468e2 100644 --- a/backend/src/plugins/translate/plugin.py +++ b/backend/src/plugins/translate/plugin.py @@ -1,19 +1,15 @@ -# [DEF:TranslatePlugin:Module] -# @COMPLEXITY: 2 -# @SEMANTICS: plugin, translate, llm, sql, dashboard -# @PURPOSE: TranslatePlugin skeleton for cross-dialect SQL and dashboard translation. -# @LAYER: Domain -# @RELATION: INHERITS -> [PluginBase] -# @RATIONALE: Separate plugin avoids bloating LLMAnalysisPlugin beyond fractal limit (<400 lines). -# @REJECTED: Extending LLMAnalysisPlugin would conflate two distinct feature domains. +# #region TranslatePlugin [C:3] [TYPE Module] [SEMANTICS plugin,translate,llm,sql,dashboard] +# @BRIEF TranslatePlugin skeleton for cross-dialect SQL and dashboard translation. +# @LAYER Domain +# @RELATION INHERITS -> [PluginBase] from typing import Dict, Any, Optional from ...core.plugin_base import PluginBase -# [DEF:TranslatePlugin:Class] -# @PURPOSE: Plugin for translating SQL queries and dashboard definitions across database dialects. -# @RELATION: IMPLEMENTS -> backend.src.core.plugin_base.PluginBase +# #region TranslatePlugin [C:3] [TYPE Class] [SEMANTICS plugin,translate] +# @BRIEF Plugin for translating SQL queries and dashboard definitions across database dialects. +# @RELATION IMPLEMENTS -> [PluginBase] class TranslatePlugin(PluginBase): @property def id(self) -> str: @@ -57,6 +53,6 @@ class TranslatePlugin(PluginBase): async def execute(self, params: Dict[str, Any], context: Optional[Any] = None): """Execute a translation job — implementation deferred to later phases.""" raise NotImplementedError("TranslatePlugin.execute not yet implemented") -# [/DEF:TranslatePlugin:Class] +# #endregion TranslatePlugin -# [/DEF:TranslatePlugin:Module] +# #endregion TranslatePlugin diff --git a/backend/src/plugins/translate/preview.py b/backend/src/plugins/translate/preview.py index 80835ca1..b95b2953 100644 --- a/backend/src/plugins/translate/preview.py +++ b/backend/src/plugins/translate/preview.py @@ -1,21 +1,21 @@ -# [DEF:TranslationPreview:Module] -# @COMPLEXITY: 4 -# @SEMANTICS: translate, preview, llm, session -# @PURPOSE: Preview session management: fetch sample rows from Superset, send to LLM with context + filtered dictionary, return side-by-side results. -# @LAYER: Domain -# @RELATION: DEPENDS_ON -> [TranslationJob:Class] -# @RELATION: DEPENDS_ON -> [TranslationPreviewSession:Class] -# @RELATION: DEPENDS_ON -> [TranslationPreviewRecord:Class] -# @RELATION: DEPENDS_ON -> [LLMProviderService] -# @RELATION: DEPENDS_ON -> [DictionaryManager] -# @RELATION: DEPENDS_ON -> [SupersetClient] -# @RELATION: DEPENDS_ON -> [render_prompt] -# @RELATION: DEPENDS_ON -> [ConfigManager] -# @PRE: Database session and config manager are available. -# @POST: Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected. -# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows. -# @RATIONALE: C4 because preview is stateful with approve/edit/reject lifecycle and LLM API calls. -# @REJECTED: Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably. +# #region TranslationPreview [C:5] [TYPE Module] [SEMANTICS translate,preview,llm,session] +# @BRIEF Preview session management: fetch sample rows from Superset, send to LLM with context + filtered dictionary, return side-by-side results. +# @LAYER Domain +# @RELATION DEPENDS_ON -> [TranslationJob] +# @RELATION DEPENDS_ON -> [TranslationPreviewSession] +# @RELATION DEPENDS_ON -> [TranslationPreviewRecord] +# @RELATION DEPENDS_ON -> [LLMProviderService] +# @RELATION DEPENDS_ON -> [DictionaryManager] +# @RELATION DEPENDS_ON -> [SupersetClient] +# @RELATION DEPENDS_ON -> [render_prompt] +# @RELATION DEPENDS_ON -> [ConfigManager] +# @PRE Database session and config manager are available. +# @POST Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected. +# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates DB rows. +# @DATA_CONTRACT Input[job_id, sample_size] -> Output[Dict with session, records, cost_estimate] +# @INVARIANT Preview sessions expire after 24 hours; exactly one ACTIVE session per job at a time. +# @RATIONALE C5 because preview is stateful with approve/edit/reject lifecycle and LLM API calls requiring structured error handling. +# @REJECTED Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably. from typing import Any, Dict, List, Optional, Tuple from sqlalchemy.orm import Session @@ -38,8 +38,7 @@ from ...services.llm_provider import LLMProviderService from ...services.llm_prompt_templates import render_prompt from .dictionary import DictionaryManager -# [DEF:DEFAULT_EXECUTION_PROMPT_TEMPLATE:Constant] -# @PURPOSE: Default prompt template for batch LLM translation execution (no context columns — faster). +# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [C:1] [TYPE Constant] [SEMANTICS translate,prompt] DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = ( "Translate the following database content from {source_language} to {target_language}.\n\n" "Source dialect: {source_dialect}\n" @@ -52,11 +51,10 @@ DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = ( '{{"rows": [{{"row_id": "", "translation": ""}}]}}\n' "Each row_id must match the index provided. Return exactly {row_count} entries." ) -# [/DEF:DEFAULT_EXECUTION_PROMPT_TEMPLATE:Constant] +# #endregion DEFAULT_EXECUTION_PROMPT_TEMPLATE -# [DEF:DEFAULT_PREVIEW_PROMPT_TEMPLATE:Constant] -# @PURPOSE: Default prompt template for LLM translation preview. +# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [C:1] [TYPE Constant] [SEMANTICS translate,prompt] DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = ( "Translate the following database content from {source_language} to {target_language}.\n\n" "Source dialect: {source_dialect}\n" @@ -70,13 +68,11 @@ DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = ( '{{"rows": [{{"row_id": "", "translation": ""}}]}}\n' "Each row_id must match the index provided. Return exactly {row_count} entries." ) -# [/DEF:DEFAULT_PREVIEW_PROMPT_TEMPLATE:Constant] +# #endregion DEFAULT_PREVIEW_PROMPT_TEMPLATE -# [DEF:TokenEstimator:Class] -# @PURPOSE: Estimate token counts and costs for LLM translation operations. -# @RATIONALE: Token estimation uses a heuristic (chars/token ratio) since exact tokenization depends on the LLM model. -# @REJECTED: Using an external tokenizer library would introduce a heavy dependency for estimation only. +# #region TokenEstimator [C:2] [TYPE Class] [SEMANTICS translate,tokens,estimate] +# @BRIEF Estimate token counts and costs for LLM translation operations using heuristic chars/token ratio. class TokenEstimator: """Estimate token counts and costs for LLM operations.""" @@ -84,46 +80,38 @@ class TokenEstimator: OUTPUT_TOKENS_PER_ROW_ESTIMATE: int = 50 TOKEN_COST_PER_1K: float = 0.002 # Default cost per 1K tokens - # [DEF:estimate_prompt_tokens:Function] - # @PURPOSE: Estimate token count for a prompt string. - # @PRE: prompt is a non-empty string. - # @POST: Returns estimated token count (integer). + # #region estimate_prompt_tokens [C:1] [TYPE Function] [SEMANTICS translate,tokens] @staticmethod def estimate_prompt_tokens(prompt: str) -> int: if not prompt: return 0 return max(1, int(len(prompt) / TokenEstimator.CHARS_PER_TOKEN_ESTIMATE)) - # [/DEF:estimate_prompt_tokens:Function] + # #endregion estimate_prompt_tokens - # [DEF:estimate_output_tokens:Function] - # @PURPOSE: Estimate output token count for translating N rows. - # @PRE: row_count >= 0. - # @POST: Returns estimated output token count. + # #region estimate_output_tokens [C:1] [TYPE Function] [SEMANTICS translate,tokens] @staticmethod def estimate_output_tokens(row_count: int) -> int: return row_count * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE - # [/DEF:estimate_output_tokens:Function] + # #endregion estimate_output_tokens - # [DEF:estimate_cost:Function] - # @PURPOSE: Estimate cost for a given number of tokens. - # @PRE: total_tokens >= 0. - # @POST: Returns estimated cost in USD. + # #region estimate_cost [C:1] [TYPE Function] [SEMANTICS translate,cost] @staticmethod def estimate_cost(total_tokens: int, cost_per_1k: Optional[float] = None) -> float: rate = cost_per_1k if cost_per_1k is not None else TokenEstimator.TOKEN_COST_PER_1K return round((total_tokens / 1000) * rate, 6) - # [/DEF:estimate_cost:Function] + # #endregion estimate_cost -# [/DEF:TokenEstimator:Class] +# #endregion TokenEstimator -# [DEF:TranslationPreview:Class] -# @COMPLEXITY: 4 -# @PURPOSE: Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate. -# @PRE: Database session and config manager are available. -# @POST: Preview sessions created with persisted records; full execution gates on accepted session. -# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows. +# #region TranslationPreview [C:5] [TYPE Class] [SEMANTICS translate,preview,lifecycle] +# @BRIEF Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate. +# @PRE Database session and config manager are available. +# @POST Preview sessions created with persisted records; full execution gates on accepted session. +# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates DB rows. +# @DATA_CONTRACT Input[job_id, sample_size] -> Output[Dict with session, records, cost_estimate] +# @INVARIANT Preview sessions expire after 24 hours; exactly one ACTIVE session per job at a time. class TranslationPreview: def __init__( @@ -136,12 +124,11 @@ class TranslationPreview: self.config_manager = config_manager self.current_user = current_user - # [DEF:preview_rows:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records. - # @PRE: job_id exists and job has source_datasource_id, translation_column configured. - # @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session. - # @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows. + # #region preview_rows [C:4] [TYPE Function] [SEMANTICS translate,preview,rows] + # @BRIEF Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records. + # @PRE job_id exists and job has source_datasource_id, translation_column configured. + # @POST Returns TranslationPreviewResponse with records, cost estimation, and persistent session. + # @SIDE_EFFECT Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows. def preview_rows( self, job_id: str, @@ -303,6 +290,7 @@ class TranslationPreview: source_object_name=f"Row {idx + 1}", status=status, feedback=feedback, + source_data=meta.get("context_data"), created_at=datetime.now(timezone.utc), ) self.db.add(record) @@ -348,12 +336,13 @@ class TranslationPreview: "sample_cost": sample_cost, }) return result - # [/DEF:preview_rows:Function] + # #endregion preview_rows - # [DEF:update_preview_row:Function] - # @PURPOSE: Approve, edit, or reject an individual preview row. - # @PRE: session_id and row_id exist, session is ACTIVE. - # @POST: PreviewRecord status is updated. + # #region update_preview_row [C:4] [TYPE Function] [SEMANTICS translate,preview,row,update] + # @BRIEF Approve, edit, or reject an individual preview row. + # @PRE session_id and row_id exist, session is ACTIVE. + # @POST PreviewRecord status is updated (APPROVED, REJECTED, or APPROVED with edited translation). + # @SIDE_EFFECT DB write. def update_preview_row( self, job_id: str, @@ -417,13 +406,13 @@ class TranslationPreview: "status": record.status, "feedback": record.feedback, } - # [/DEF:update_preview_row:Function] + # #endregion update_preview_row - # [DEF:accept_preview_session:Function] - # @PURPOSE: Mark a preview session as accepted, which gates full execution. - # @PRE: job_id has an ACTIVE preview session. - # @POST: Session status changes to APPLIED. - # @SIDE_EFFECT: Future full execution calls will check for accepted session. + # #region accept_preview_session [C:4] [TYPE Function] [SEMANTICS translate,preview,accept] + # @BRIEF Mark a preview session as accepted (APPLIED), which gates full execution. + # @PRE job_id has an ACTIVE preview session. + # @POST Session status changes to APPLIED; future full execution calls check for accepted session. + # @SIDE_EFFECT DB write. def accept_preview_session(self, job_id: str) -> Dict[str, Any]: with belief_scope("TranslationPreview.accept_preview_session"): session = ( @@ -472,12 +461,11 @@ class TranslationPreview: for r in records ], } - # [/DEF:accept_preview_session:Function] + # #endregion accept_preview_session - # [DEF:get_preview_session:Function] - # @PURPOSE: Get the latest preview session for a job with its records. - # @PRE: job_id exists. - # @POST: Returns session data with records or raises ValueError. + # #region get_preview_session [C:3] [TYPE Function] [SEMANTICS translate,preview,get] + # @BRIEF Get the latest preview session for a job with its records. + # @RELATION DEPENDS_ON -> [TranslationPreviewSession] def get_preview_session(self, job_id: str) -> Dict[str, Any]: with belief_scope("TranslationPreview.get_preview_session"): session = ( @@ -516,14 +504,13 @@ class TranslationPreview: for r in records ], } - # [/DEF:get_preview_session:Function] + # #endregion get_preview_session - # [DEF:_fetch_sample_rows:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Fetch sample rows from the Superset dataset for preview. - # @PRE: job has source_datasource_id and translation_column. - # @POST: Returns list of dicts with row data. - # @SIDE_EFFECT: Calls Superset chart data endpoint. + # #region _fetch_sample_rows [C:4] [TYPE Function] [SEMANTICS translate,superset,sample] + # @BRIEF Fetch sample rows from the Superset dataset for preview. + # @PRE job has source_datasource_id and translation_column. + # @POST Returns list of dicts with row data from Superset chart data API. + # @SIDE_EFFECT Calls Superset chart data endpoint with pagination. def _fetch_sample_rows(self, job: TranslationJob, sample_size: int = 10, env_id: Optional[str] = None) -> List[Dict[str, Any]]: with belief_scope("TranslationPreview._fetch_sample_rows"): # Determine environment: prefer explicit env_id, then job.environment_id, then job.source_dialect (legacy) @@ -602,12 +589,11 @@ class TranslationPreview: ) return rows - # [/DEF:_fetch_sample_rows:Function] + # #endregion _fetch_sample_rows - # [DEF:_extract_data_rows:Function] - # @PURPOSE: Extract data rows from Superset chart data response. - # @PRE: response is a dict from Superset API. - # @POST: Returns list of row dicts. + # #region _extract_data_rows [C:3] [TYPE Function] [SEMANTICS translate,superset,data] + # @BRIEF Extract data rows from Superset chart data response, trying multiple response formats. + # @RELATION DEPENDS_ON -> [SupersetClient] @staticmethod def _extract_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]: with belief_scope("TranslationPreview._extract_data_rows"): @@ -636,14 +622,13 @@ class TranslationPreview: return result return [] - # [/DEF:_extract_data_rows:Function] + # #endregion _extract_data_rows - # [DEF:_call_llm:Function] - # @COMPLEXITY: 4 - # @PURPOSE: Call the configured LLM provider with the preview prompt. - # @PRE: job has a valid provider_id. - # @POST: Returns raw LLM response string. - # @SIDE_EFFECT: Makes HTTP call to LLM provider. + # #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call] + # @BRIEF Call the configured LLM provider with the preview prompt. + # @PRE job has a valid provider_id. + # @POST Returns raw LLM response string. + # @SIDE_EFFECT Makes HTTP call to LLM provider. def _call_llm(self, job: TranslationJob, prompt: str) -> str: with belief_scope("TranslationPreview._call_llm"): if not job.provider_id: @@ -679,13 +664,13 @@ class TranslationPreview: "response_length": len(response_text), }) return response_text - # [/DEF:_call_llm:Function] + # #endregion _call_llm - # [DEF:_call_openai_compatible:Function] - # @PURPOSE: Call an OpenAI-compatible API for translation. - # @PRE: base_url, api_key, model, prompt are valid. - # @POST: Returns response text. - # @SIDE_EFFECT: Makes HTTP POST to LLM API. + # #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai] + # @BRIEF Call an OpenAI-compatible API for translation with structured output support. + # @PRE base_url, api_key, model, prompt are valid. + # @POST Returns response text. + # @SIDE_EFFECT Makes HTTP POST to LLM API. @staticmethod def _call_openai_compatible( base_url: str, @@ -741,12 +726,13 @@ class TranslationPreview: raise ValueError("LLM returned empty content") return content - # [/DEF:_call_openai_compatible:Function] + # #endregion _call_openai_compatible - # [DEF:_parse_llm_response:Function] - # @PURPOSE: Parse the LLM JSON response into a dict of row_id -> translation. - # @PRE: response_text is valid JSON with {"rows": [...]} structure. - # @POST: Returns dict mapping string row_id to translation text. + # #region _parse_llm_response [C:4] [TYPE Function] [SEMANTICS translate,llm,parse] + # @BRIEF Parse the LLM JSON response into a dict of row_id -> translation, with markdown code block fallback. + # @PRE response_text is valid JSON with {"rows": [...]} structure. + # @POST Returns dict mapping string row_id to translation text; warns if fewer translations than expected. + # @SIDE_EFFECT Logs warnings for short responses. @staticmethod def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]: with belief_scope("TranslationPreview._parse_llm_response"): @@ -785,10 +771,10 @@ class TranslationPreview: ) return translations - # [/DEF:_parse_llm_response:Function] + # #endregion _parse_llm_response - # [DEF:_compute_config_hash:Function] - # @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison. + # #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash] + # @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison. @staticmethod def _compute_config_hash(job: TranslationJob) -> str: config_str = json.dumps({ @@ -803,10 +789,10 @@ class TranslationPreview: "upsert_strategy": job.upsert_strategy, }, sort_keys=True) return hashlib.sha256(config_str.encode()).hexdigest()[:16] - # [/DEF:_compute_config_hash:Function] + # #endregion _compute_config_hash - # [DEF:_compute_dict_snapshot_hash:Function] - # @PURPOSE: Compute a hash of the dictionary state for snapshot comparison. + # #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash] + # @BRIEF Compute a SHA-256 hash of the dictionary state for snapshot comparison. def _compute_dict_snapshot_hash(self, job_id: str) -> str: dict_links = ( self.db.query(TranslationJobDictionary) @@ -816,9 +802,8 @@ class TranslationPreview: dict_ids = sorted([dl.dictionary_id for dl in dict_links]) hash_input = ",".join(dict_ids) return hashlib.sha256(hash_input.encode()).hexdigest()[:16] - # [/DEF:_compute_dict_snapshot_hash:Function] + # #endregion _compute_dict_snapshot_hash -# [/DEF:TranslationPreview:Class] - -# [/DEF:TranslationPreview:Module] +# #endregion TranslationPreview +# #endregion TranslationPreview diff --git a/backend/src/plugins/translate/scheduler.py b/backend/src/plugins/translate/scheduler.py index 12a8c911..9f70061e 100644 --- a/backend/src/plugins/translate/scheduler.py +++ b/backend/src/plugins/translate/scheduler.py @@ -1,18 +1,18 @@ -# [DEF:TranslationScheduler:Module] -# @COMPLEXITY: 4 -# @SEMANTICS: translate, scheduler, apscheduler, cron -# @PURPOSE: Manage TranslationSchedule rows and register them with core SchedulerService. -# @LAYER: Domain -# @RELATION: DEPENDS_ON -> [TranslationSchedule:Class] -# @RELATION: DEPENDS_ON -> [SchedulerService:Class] -# @RELATION: DEPENDS_ON -> [TranslationOrchestrator:Class] -# @RELATION: DEPENDS_ON -> [TranslationEventLog:Class] -# @PRE: Database session and SchedulerService are available. -# @POST: TranslationSchedule CRUD persisted; APScheduler jobs registered/updated/removed. -# @SIDE_EFFECT: Registers APScheduler jobs; runs translations on trigger; creates events. -# @RATIONALE: Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance. -# @REJECTED: Separate scheduler instance would create resource contention. -# @REJECTED: Polling-based approach — event-driven APScheduler is more precise. +# #region TranslationScheduler [C:5] [TYPE Module] [SEMANTICS translate,scheduler,apscheduler,cron] +# @BRIEF Manage TranslationSchedule rows and register them with core SchedulerService for cron-based execution. +# @LAYER Domain +# @RELATION DEPENDS_ON -> [TranslationSchedule] +# @RELATION DEPENDS_ON -> [SchedulerService] +# @RELATION DEPENDS_ON -> [TranslationOrchestrator] +# @RELATION DEPENDS_ON -> [TranslationEventLog] +# @PRE Database session and SchedulerService are available. +# @POST TranslationSchedule CRUD persisted; translation runs triggered on schedule. +# @SIDE_EFFECT Registers APScheduler jobs; runs translations on trigger; creates events. +# @DATA_CONTRACT Input[job_id, cron_expression] -> Output[TranslationSchedule] +# @INVARIANT Concurrency guard: max 1 pending/running run per job at scheduled trigger time. +# @RATIONALE Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance. +# @REJECTED Separate scheduler instance would create resource contention. +# @REJECTED Polling-based approach — event-driven APScheduler is more precise. import uuid from datetime import datetime, timezone, timedelta @@ -26,9 +26,13 @@ from ...models.translate import TranslationSchedule, TranslationJob, Translation from .events import TranslationEventLog -# [DEF:TranslationScheduler:Class] -# @COMPLEXITY: 4 -# @PURPOSE: CRUD for TranslationSchedule rows + APScheduler registration wrappers. +# #region TranslationScheduler [C:5] [TYPE Class] [SEMANTICS translate,scheduler,crud] +# @BRIEF CRUD for TranslationSchedule rows + APScheduler registration wrappers. +# @PRE Database session and ConfigManager are available. +# @POST Schedule rows created/updated/deleted with event logging. +# @SIDE_EFFECT Writes TranslationSchedule rows; logs SCHEDULE_CREATED/UPDATED/DELETED events. +# @DATA_CONTRACT Input[job_id, cron_expression, timezone] -> Output[TranslationSchedule] +# @INVARIANT Exactly one schedule per job (upsert pattern). class TranslationScheduler: def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None): @@ -37,10 +41,11 @@ class TranslationScheduler: self.current_user = current_user self.event_log = TranslationEventLog(db) - # [DEF:create_schedule:Function] - # @PURPOSE: Create a new schedule for a job. - # @PRE: job_id exists. cron_expression is valid. - # @POST: TranslationSchedule row created. + # #region create_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,create] + # @BRIEF Create a new schedule for a job with cron expression and event logging. + # @PRE job_id exists. cron_expression is valid. + # @POST TranslationSchedule row created; SCHEDULE_CREATED event logged. + # @SIDE_EFFECT DB write; event logging. def create_schedule( self, job_id: str, @@ -79,12 +84,13 @@ class TranslationScheduler: logger.reflect("Schedule created", {"schedule_id": schedule.id, "job_id": job_id}) return schedule - # [/DEF:create_schedule:Function] + # #endregion create_schedule - # [DEF:update_schedule:Function] - # @PURPOSE: Update an existing schedule. - # @PRE: job_id has an existing schedule. - # @POST: Schedule updated. + # #region update_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,update] + # @BRIEF Update an existing schedule's cron expression, timezone, or active state. + # @PRE job_id has an existing schedule. + # @POST Schedule updated; SCHEDULE_UPDATED event logged. + # @SIDE_EFFECT DB write; event logging. def update_schedule( self, job_id: str, @@ -123,12 +129,13 @@ class TranslationScheduler: logger.reflect("Schedule updated", {"schedule_id": schedule.id, "job_id": job_id}) return schedule - # [/DEF:update_schedule:Function] + # #endregion update_schedule - # [DEF:delete_schedule:Function] - # @PURPOSE: Delete a schedule for a job. - # @PRE: job_id has an existing schedule. - # @POST: Schedule deleted. + # #region delete_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,delete] + # @BRIEF Delete a schedule for a job with event logging. + # @PRE job_id has an existing schedule. + # @POST Schedule deleted; SCHEDULE_DELETED event logged. + # @SIDE_EFFECT DB write; event logging. def delete_schedule(self, job_id: str) -> None: with belief_scope("TranslationScheduler.delete_schedule"): schedule = self.db.query(TranslationSchedule).filter( @@ -149,12 +156,13 @@ class TranslationScheduler: ) logger.reflect("Schedule deleted", {"schedule_id": schedule_id, "job_id": job_id}) - # [/DEF:delete_schedule:Function] + # #endregion delete_schedule - # [DEF:enable_disable_schedule:Function] - # @PURPOSE: Enable or disable a schedule. - # @PRE: job_id has an existing schedule. - # @POST: Schedule is_active updated. + # #region set_schedule_active [C:4] [TYPE Function] [SEMANTICS translate,schedule,activate] + # @BRIEF Enable or disable a schedule by setting is_active flag. + # @PRE job_id has an existing schedule. + # @POST Schedule is_active updated. + # @SIDE_EFFECT DB write. def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule: with belief_scope("TranslationScheduler.set_schedule_active"): schedule = self.db.query(TranslationSchedule).filter( @@ -174,12 +182,10 @@ class TranslationScheduler: "is_active": is_active, }) return schedule - # [/DEF:enable_disable_schedule:Function] + # #endregion set_schedule_active - # [DEF:get_schedule:Function] - # @PURPOSE: Get schedule for a job. - # @PRE: job_id exists. - # @POST: Returns TranslationSchedule or raises ValueError. + # #region get_schedule [C:2] [TYPE Function] [SEMANTICS translate,schedule,get] + # @BRIEF Get the schedule for a job by job_id. def get_schedule(self, job_id: str) -> TranslationSchedule: with belief_scope("TranslationScheduler.get_schedule"): schedule = self.db.query(TranslationSchedule).filter( @@ -188,11 +194,10 @@ class TranslationScheduler: if not schedule: raise ValueError(f"No schedule found for job '{job_id}'") return schedule - # [/DEF:get_schedule:Function] + # #endregion get_schedule - # [DEF:list_active_schedules:Function] - # @PURPOSE: List all active schedules. - # @POST: Returns list of active TranslationSchedule rows. + # #region list_active_schedules [C:2] [TYPE Function] [SEMANTICS translate,schedule,list] + # @BRIEF List all active schedules. @staticmethod def list_active_schedules(db: Session) -> List[TranslationSchedule]: return ( @@ -200,13 +205,10 @@ class TranslationScheduler: .filter(TranslationSchedule.is_active == True) .all() ) - # [/DEF:list_active_schedules:Function] + # #endregion list_active_schedules - # [DEF:get_next_executions:Function] - # @PURPOSE: Compute next N execution times from cron expression. - # @PRE: cron_expression is valid. - # @POST: Returns list of ISO datetime strings. - @staticmethod + # #region get_next_executions [C:2] [TYPE Function] [SEMANTICS translate,schedule,next] + # @BRIEF Compute next N execution times from a cron expression using APScheduler's CronTrigger. def get_next_executions(cron_expression: str, timezone_str: str = "UTC", n: int = 3) -> List[str]: from zoneinfo import ZoneInfo from apscheduler.triggers.cron import CronTrigger @@ -231,18 +233,20 @@ class TranslationScheduler: prev = ft next_time = ft return results - # [/DEF:get_next_executions:Function] + # #endregion get_next_executions -# [/DEF:TranslationScheduler:Class] +# #endregion TranslationScheduler -# [DEF:execute_scheduled_translation:Function] -# @PURPOSE: APScheduler job handler — runs scheduled translation with concurrency check and new-key-only fallback. -# @PRE: schedule_id is valid. -# @POST: Translation run created and executed if no concurrent run exists. -# @SIDE_EFFECT: DB writes; LLM calls; Superset API calls. -# @RATIONALE: New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full. +# #region execute_scheduled_translation [C:5] [TYPE Function] [SEMANTICS translate,schedule,execute] +# @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and baseline expiry detection. +# @PRE schedule_id is valid and job exists. +# @POST Translation run created and executed if no concurrent run exists. +# @SIDE_EFFECT DB writes; LLM calls; Superset API calls; event logging. +# @DATA_CONTRACT Input[schedule_id, job_id, db_session_maker, config_manager] -> Output[None] +# @INVARIANT Concurrency guard: max 1 pending/running run per job at trigger time. +# @RATIONALE New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full. def execute_scheduled_translation( schedule_id: str, job_id: str, @@ -357,5 +361,5 @@ def execute_scheduled_translation( logger.error(f"[scheduled_translation] Unexpected error: {e}") finally: db.close() -# [/DEF:execute_scheduled_translation:Function] -# [/DEF:TranslationScheduler:Module] +# #endregion execute_scheduled_translation +# #endregion TranslationScheduler diff --git a/backend/src/plugins/translate/service.py b/backend/src/plugins/translate/service.py index dd9cef77..fb0441bb 100644 --- a/backend/src/plugins/translate/service.py +++ b/backend/src/plugins/translate/service.py @@ -1,16 +1,16 @@ -# [DEF:TranslateJobService:Module] -# @COMPLEXITY: 4 -# @SEMANTICS: translate, service, crud, validation -# @PURPOSE: Service layer for translation job CRUD with datasource column validation and database dialect detection. -# @LAYER: Domain -# @RELATION: DEPENDS_ON -> [TranslationJob] -# @RELATION: DEPENDS_ON -> [SupersetClient] -# @RELATION: DEPENDS_ON -> [ConfigManager] -# @PRE: Database session and config manager are available. -# @POST: Translation jobs are created/updated/deleted with column validation and dialect caching. -# @SIDE_EFFECT: Queries Superset for column metadata and database dialect at save time. -# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. -# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity. +# #region TranslateJobService [C:5] [TYPE Module] [SEMANTICS translate,service,crud,validation] +# @BRIEF Service layer for translation job CRUD with datasource column validation and database dialect detection. +# @LAYER Domain +# @RELATION DEPENDS_ON -> [TranslationJob] +# @RELATION DEPENDS_ON -> [SupersetClient] +# @RELATION DEPENDS_ON -> [ConfigManager] +# @PRE Database session and config manager are available. +# @POST Translation jobs are created/updated/deleted with column validation and dialect caching. +# @SIDE_EFFECT Queries Superset for column metadata and database dialect at save time. +# @DATA_CONTRACT Input[TranslateJobCreate/Update] -> Output[TranslationJob/TranslateJobResponse] +# @INVARIANT Snapshot isolation: in-progress runs use config snapshot; config edits affect future runs only. +# @RATIONALE Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. +# @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity. from typing import Any, Dict, List, Optional, Tuple from sqlalchemy.orm import Session @@ -37,12 +37,16 @@ SUPPORTED_DIALECTS = { } -# [DEF:get_dialect_from_database:Function] -# @PURPOSE: Extract normalized dialect string from a Superset database record. -# @PRE: database_record is a dict from Superset API. -# @POST: Returns normalized dialect string or raises ValueError if unsupported. +# #region get_dialect_from_database [C:4] [TYPE Function] [SEMANTICS translate,dialect,validation] +# @BRIEF Extract normalized dialect string from a Superset database record and validate it is supported. +# @PRE database_record is a dict from Superset API with 'backend' or 'engine' key. +# @POST Returns normalized dialect string or raises ValueError if unsupported. +# @SIDE_EFFECT None — pure data extraction. def get_dialect_from_database(database_record: Dict[str, Any]) -> str: """Extract and validate dialect from Superset database record.""" + logger.reason("Extracting dialect from database record", { + "has_backend": bool(database_record.get("backend") or database_record.get("engine")), + }) backend = ( database_record.get("backend") or database_record.get("engine") @@ -50,6 +54,9 @@ def get_dialect_from_database(database_record: Dict[str, Any]) -> str: ).lower().strip() if not backend: + logger.explore("Could not determine database dialect", { + "record_keys": list(database_record.keys()), + }) raise ValueError("Could not determine database dialect from connection") # Map Superset backend names to normalized dialect @@ -75,28 +82,36 @@ def get_dialect_from_database(database_record: Dict[str, Any]) -> str: normalized = dialect_map.get(backend, backend) if normalized not in SUPPORTED_DIALECTS: + logger.explore("Unsupported dialect", {"backend": backend, "normalized": normalized}) raise ValueError( f"Unsupported database dialect: '{backend}'. " f"Supported dialects: {', '.join(sorted(SUPPORTED_DIALECTS))}" ) + logger.reflect("Dialect validated", {"dialect": normalized}) return normalized -# [/DEF:get_dialect_from_database:Function] +# #endregion get_dialect_from_database -# [DEF:fetch_datasource_metadata:Function] -# @PURPOSE: Fetch datasource columns and database dialect from Superset. -# @PRE: dataset_id is a valid Superset dataset ID and environment has valid credentials. -# @POST: Returns (columns_list, dialect_string) or raises on failure. +# #region fetch_datasource_metadata [C:4] [TYPE Function] [SEMANTICS translate,datasource,metadata] +# @BRIEF Fetch datasource columns and database dialect from Superset for a given dataset. +# @PRE dataset_id is a valid Superset dataset ID and environment has valid credentials. +# @POST Returns (columns_list, dialect_string) or raises on failure. +# @SIDE_EFFECT Queries Superset API for dataset detail and database info. def fetch_datasource_metadata( dataset_id: int, env_id: str, config_manager: ConfigManager, ) -> Tuple[List[Dict[str, Any]], str]: """Fetch column metadata and database dialect for a datasource from Superset.""" + logger.reason("Fetching datasource metadata", { + "dataset_id": dataset_id, + "env_id": env_id, + }) # Find environment config environments = config_manager.get_environments() env_config = next((e for e in environments if e.id == env_id), None) if not env_config: + logger.explore("Environment not found", {"env_id": env_id}) raise ValueError(f"Superset environment '{env_id}' not found in configuration") # Create Superset client and fetch dataset detail @@ -133,25 +148,29 @@ def fetch_datasource_metadata( else: raise ValueError("No database information available for this datasource") except Exception as e: - logger.warning(f"[translate_service] Could not fetch database dialect: {e}") + logger.explore("Could not fetch database dialect", {"error": str(e)}) raise ValueError(f"Could not determine database dialect: {e}") + logger.reflect("Metadata fetched", {"columns": len(columns), "dialect": dialect}) return columns, dialect -# [/DEF:fetch_datasource_metadata:Function] +# #endregion fetch_datasource_metadata -# [DEF:detect_virtual_columns:Function] -# @PURPOSE: Identify virtual (calculated) columns from column metadata. -# @PRE: columns is a list of dicts with 'is_physical' key. -# @POST: Returns list of virtual column names. +# #region detect_virtual_columns [C:2] [TYPE Function] [SEMANTICS translate,columns] +# @BRIEF Identify virtual (calculated) columns from column metadata by filtering non-physical entries. def detect_virtual_columns(columns: List[Dict[str, Any]]) -> List[str]: """Return names of columns that are virtual (not physical).""" return [col["name"] for col in columns if not col.get("is_physical", True)] -# [/DEF:detect_virtual_columns:Function] +# #endregion detect_virtual_columns -# [DEF:TranslateJobService:Class] -# @PURPOSE: Service for translation job CRUD with validation and Superset integration. +# #region TranslateJobService [C:5] [TYPE Class] [SEMANTICS translate,service,crud] +# @BRIEF Service for translation job CRUD with validation and Superset integration. +# @PRE Database session and config manager are available. +# @POST Translation jobs are managed with full lifecycle (create/update/delete/duplicate). +# @SIDE_EFFECT Queries Superset for dialect detection and column validation. +# @DATA_CONTRACT Input[TranslateJobCreate/Update] -> Output[TranslationJob] +# @INVARIANT Snapshot isolation: in-progress runs unaffected by config edits. class TranslateJobService: def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None): @@ -159,9 +178,8 @@ class TranslateJobService: self.config_manager = config_manager self.current_user = current_user - # [DEF:list_jobs:Function] - # @PURPOSE: List translation jobs with optional status filter and pagination. - # @POST: Returns tuple of (total_count, list_of_jobs). + # #region list_jobs [C:2] [TYPE Function] [SEMANTICS translate,jobs,query] + # @BRIEF List translation jobs with optional status filter and pagination. def list_jobs( self, page: int = 1, @@ -178,29 +196,30 @@ class TranslateJobService: jobs = query.order_by(TranslationJob.created_at.desc()).offset(offset).limit(page_size).all() return total, jobs - # [/DEF:list_jobs:Function] + # #endregion list_jobs - # [DEF:get_job:Function] - # @PURPOSE: Get a single translation job by ID. - # @POST: Returns TranslationJob or raises ValueError. + # #region get_job [C:2] [TYPE Function] [SEMANTICS translate,jobs,query] + # @BRIEF Get a single translation job by ID, raising ValueError if not found. def get_job(self, job_id: str) -> TranslationJob: job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() if not job: raise ValueError(f"Translation job '{job_id}' not found") return job - # [/DEF:get_job:Function] + # #endregion get_job - # [DEF:create_job:Function] - # @PURPOSE: Create a new translation job with column validation. - # @PRE: payload contains valid job configuration. - # @POST: Returns the created TranslationJob with database_dialect cached. - # @SIDE_EFFECT: Validates columns via SupersetClient if source_datasource_id is provided. - # @SIDE_EFFECT: Caches database_dialect from Superset connection. + # #region create_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,create] + # @BRIEF Create a new translation job with column validation and dialect detection. + # @PRE payload contains valid job configuration (name, upsert_strategy, etc). + # @POST Returns the created TranslationJob with database_dialect cached. + # @SIDE_EFFECT Validates columns via SupersetClient if source_datasource_id is provided; caches database_dialect. def create_job(self, payload: TranslateJobCreate) -> TranslationJob: - logger.info(f"[TranslateJobService] Creating job '{payload.name}'") + logger.reason("Creating translation job", {"name": payload.name}) # Validate: must have a translation column if datasource is configured if payload.source_datasource_id and not payload.translation_column: + logger.explore("Missing translation column", { + "source_datasource_id": payload.source_datasource_id, + }) raise ValueError("A translation column is required when a datasource is selected") # Validate upsert strategy @@ -225,7 +244,7 @@ class TranslateJobService: ) dialect = detected_dialect except Exception as e: - logger.warning(f"[TranslateJobService] Dialect detection failed: {e}") + logger.explore("Dialect detection failed", {"error": str(e)}) dialect = payload.source_dialect # Build job instance @@ -249,6 +268,7 @@ class TranslateJobService: batch_size=payload.batch_size, upsert_strategy=payload.upsert_strategy, environment_id=payload.environment_id, + target_database_id=payload.target_database_id, status="DRAFT", created_by=self.current_user, ) @@ -267,17 +287,17 @@ class TranslateJobService: self.db.commit() self.db.refresh(job) - logger.info(f"[TranslateJobService] Created job '{job.id}'") + logger.reflect("Job created", {"job_id": job.id}) return job - # [/DEF:create_job:Function] + # #endregion create_job - # [DEF:update_job:Function] - # @PURPOSE: Update an existing translation job. - # @PRE: payload contains fields to update. - # @POST: Returns the updated TranslationJob. - # @SIDE_EFFECT: Re-detects database_dialect if source_datasource_id changed. + # #region update_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,update] + # @BRIEF Update an existing translation job with optional dialect re-detection. + # @PRE payload contains fields to update; job_id exists. + # @POST Returns the updated TranslationJob with re-detected dialect if datasource changed. + # @SIDE_EFFECT Re-detects database_dialect if source_datasource_id changed. def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob: - logger.info(f"[TranslateJobService] Updating job '{job_id}'") + logger.reason("Updating translation job", {"job_id": job_id}) job = self.get_job(job_id) update_data = payload.model_dump(exclude_unset=True) @@ -298,7 +318,7 @@ class TranslateJobService: ) job.database_dialect = detected_dialect except Exception as e: - logger.warning(f"[TranslateJobService] Dialect re-detection failed: {e}") + logger.explore("Dialect re-detection failed", {"error": str(e)}) job.updated_at = datetime.now(timezone.utc) @@ -317,16 +337,17 @@ class TranslateJobService: self.db.commit() self.db.refresh(job) - logger.info(f"[TranslateJobService] Updated job '{job_id}'") + logger.reflect("Job updated", {"job_id": job_id}) return job - # [/DEF:update_job:Function] + # #endregion update_job - # [DEF:delete_job:Function] - # @PURPOSE: Delete a translation job and its associations. - # @PRE: job_id must exist. - # @POST: Job and all related records are deleted. + # #region delete_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,delete] + # @BRIEF Delete a translation job and its dictionary associations. + # @PRE job_id must exist. + # @POST Job and all related TranslationJobDictionary records are deleted. + # @SIDE_EFFECT Deletes TranslationJob and TranslationJobDictionary rows. def delete_job(self, job_id: str) -> None: - logger.info(f"[TranslateJobService] Deleting job '{job_id}'") + logger.reason("Deleting translation job", {"job_id": job_id}) job = self.get_job(job_id) # Delete dictionary associations @@ -336,15 +357,16 @@ class TranslateJobService: self.db.delete(job) self.db.commit() - logger.info(f"[TranslateJobService] Deleted job '{job_id}'") - # [/DEF:delete_job:Function] + logger.reflect("Job deleted", {"job_id": job_id}) + # #endregion delete_job - # [DEF:duplicate_job:Function] - # @PURPOSE: Duplicate a translation job with a new name. - # @PRE: job_id must exist. - # @POST: Returns the duplicated TranslationJob with status DRAFT. + # #region duplicate_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,duplicate] + # @BRIEF Duplicate a translation job with a new name, copying all fields and dictionary associations. + # @PRE job_id must exist. + # @POST Returns the duplicated TranslationJob with status DRAFT. + # @SIDE_EFFECT Creates new TranslationJob and TranslationJobDictionary rows. def duplicate_job(self, job_id: str, new_name: Optional[str] = None) -> TranslationJob: - logger.info(f"[TranslateJobService] Duplicating job '{job_id}'") + logger.reason("Duplicating translation job", {"job_id": job_id, "new_name": new_name}) source = self.get_job(job_id) # Copy all fields except id, created_at, updated_at, status @@ -367,6 +389,8 @@ class TranslateJobService: provider_id=source.provider_id, batch_size=source.batch_size, upsert_strategy=source.upsert_strategy, + environment_id=source.environment_id, + target_database_id=source.target_database_id, status="DRAFT", created_by=self.current_user, ) @@ -387,22 +411,20 @@ class TranslateJobService: self.db.commit() self.db.refresh(new_job) - logger.info(f"[TranslateJobService] Duplicated job '{job_id}' -> '{new_job.id}'") + logger.reflect("Job duplicated", {"new_job_id": new_job.id, "source_job_id": job_id}) return new_job - # [/DEF:duplicate_job:Function] + # #endregion duplicate_job - # [DEF:get_job_dictionary_ids:Function] - # @PURPOSE: Get dictionary IDs attached to a job. - # @POST: Returns list of dictionary IDs. + # #region get_job_dictionary_ids [C:1] [TYPE Function] [SEMANTICS translate,jobs,dictionaries] def get_job_dictionary_ids(self, job_id: str) -> List[str]: assocs = self.db.query(TranslationJobDictionary).filter( TranslationJobDictionary.job_id == job_id ).all() return [a.dictionary_id for a in assocs] - # [/DEF:get_job_dictionary_ids:Function] + # #endregion get_job_dictionary_ids - # [DEF:fetch_available_datasources:Function] - # @PURPOSE: List available Superset datasets for translation job creation. + # #region fetch_available_datasources [C:2] [TYPE Function] [SEMANTICS translate,datasources,query] + # @BRIEF List available Superset datasets for translation job creation with optional search filter. def fetch_available_datasources(self, env_id: str, search: Optional[str] = None) -> list: """List Superset datasets available for translation.""" from ...core.superset_client import SupersetClient @@ -428,12 +450,11 @@ class TranslateJobService: "description": ds.get("description", ""), }) return result - # [/DEF:fetch_available_datasources:Function] -# [/DEF:TranslateJobService:Class] + # #endregion fetch_available_datasources +# #endregion TranslateJobService -# [DEF:_extract_dialect:Function] -# @PURPOSE: Extract database dialect from Superset backend URI. +# #region _extract_dialect [C:1] [TYPE Function] [SEMANTICS translate,dialect] def _extract_dialect(backend: str) -> str: """Extract dialect name from a Superset database backend URI (e.g. 'postgresql+psycopg2://...').""" if not backend: @@ -444,10 +465,11 @@ def _extract_dialect(backend: str) -> str: return dialect.lower() except Exception: return "unknown" +# #endregion _extract_dialect -# [DEF:job_to_response:Function] -# @PURPOSE: Convert a TranslationJob ORM model to a TranslateJobResponse schema with dictionary_ids. +# #region job_to_response [C:2] [TYPE Function] [SEMANTICS translate,serialization] +# @BRIEF Convert a TranslationJob ORM model to a TranslateJobResponse schema with dictionary_ids. def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -> TranslateJobResponse: return TranslateJobResponse( id=job.id, @@ -474,27 +496,29 @@ def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) - updated_at=job.updated_at, dictionary_ids=dict_ids or [], environment_id=job.environment_id, + target_database_id=job.target_database_id, ) -# [/DEF:job_to_response:Function] +# #endregion job_to_response -# [DEF:DatasourceColumnsService:Function] -# @PURPOSE: Fetch datasource column metadata from Superset and return structured response. -# @PRE: datasource_id is a valid Superset dataset ID. -# @POST: Returns DatasourceColumnsResponse with column metadata and database dialect. -# @SIDE_EFFECT: Queries Superset API for dataset detail and database info. +# #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS translate,datasource,columns] +# @BRIEF Fetch datasource column metadata from Superset and return structured DatasourceColumnsResponse. +# @PRE datasource_id is a valid Superset dataset ID. +# @POST Returns DatasourceColumnsResponse with column metadata and database dialect. +# @SIDE_EFFECT Queries Superset API for dataset detail and database info. def get_datasource_columns( datasource_id: int, env_id: str, config_manager: ConfigManager, ) -> DatasourceColumnsResponse: """Fetch and return column metadata for a given Superset datasource.""" - logger.info(f"[get_datasource_columns] Fetching columns for datasource {datasource_id}") + logger.reason("Fetching datasource columns", {"datasource_id": datasource_id, "env_id": env_id}) # Find environment config environments = config_manager.get_environments() env_config = next((e for e in environments if e.id == env_id), None) if not env_config: + logger.explore("Environment not found", {"env_id": env_id}) raise ValueError(f"Superset environment '{env_id}' not found") # Create Superset client @@ -516,6 +540,7 @@ def get_datasource_columns( db_result = db_record.get("result", db_record) if isinstance(db_record, dict) else db_record dialect = get_dialect_from_database(db_result) else: + logger.explore("Could not determine dialect", {"datasource_id": datasource_id}) raise ValueError("Could not determine database dialect for this datasource") # Extract columns @@ -533,13 +558,19 @@ def get_datasource_columns( description=col.get("description", ""), )) - return DatasourceColumnsResponse( + result = DatasourceColumnsResponse( datasource_id=datasource_id, datasource_name=dataset_detail.get("table_name"), schema_name=dataset_detail.get("schema"), database_dialect=dialect, columns=columns, ) -# [/DEF:DatasourceColumnsService:Function] + logger.reflect("Datasource columns fetched", { + "datasource_id": datasource_id, + "columns_count": len(columns), + "dialect": dialect, + }) + return result +# #endregion get_datasource_columns -# [/DEF:TranslateJobService:Module] +# #endregion TranslateJobService diff --git a/backend/src/plugins/translate/sql_generator.py b/backend/src/plugins/translate/sql_generator.py index 760c12f1..3e8e46b9 100644 --- a/backend/src/plugins/translate/sql_generator.py +++ b/backend/src/plugins/translate/sql_generator.py @@ -1,18 +1,13 @@ -# [DEF:SQLGenerator:Module] -# @COMPLEXITY: 3 -# @SEMANTICS: translate, sql, generator, dialect -# @PURPOSE: Dialect-aware safe SQL generation for INSERT/UPSERT operations. -# @LAYER: Domain -# @RELATION: DEPENDS_ON -> [TranslationJob] -# @RELATION: DEPENDS_ON -> [TranslationRun] -# @PRE: Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS. -# @POST: Returns safe SQL strings for the target dialect. -# @SIDE_EFFECT: None — pure code generation. -# @RATIONALE: Dialect-aware SQL uses ON CONFLICT for PostgreSQL; plain INSERT for ClickHouse with documented limitations. -# @REJECTED: UPDATE statements — source is append-only; UPSERT covers overwrite case. -# @REJECTED: ORM-based insert bypasses Superset's SQL Lab audit trail. +# #region SQLGenerator [C:4] [TYPE Module] [SEMANTICS translate,sql,generator,dialect] +# @BRIEF Dialect-aware safe SQL generation for INSERT/UPSERT operations with identifier quoting and value encoding. +# @LAYER Domain +# @RELATION DEPENDS_ON -> [TranslationJob] +# @RELATION DEPENDS_ON -> [TranslationRun] +# @PRE Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS. +# @POST Returns safe SQL strings for the target dialect. from typing import Any, Dict, List, Optional, Tuple +from datetime import datetime, timezone from ...core.logger import logger, belief_scope # PostgreSQL/Greenplum dialects that support ON CONFLICT @@ -21,10 +16,45 @@ POSTGRESQL_DIALECTS = {"postgresql", "redshift", "greenplum"} CLICKHOUSE_DIALECTS = {"clickhouse"} -# [DEF:_quote_identifier:Function] -# @PURPOSE: Quote an identifier per dialect rules. PostgreSQL uses double quotes; ClickHouse uses backticks. -# @PRE: identifier is a non-empty string. -# @POST: Returns safely quoted identifier. +# #region _normalize_timestamp_value [C:2] [TYPE Function] [SEMANTICS translate,sql,timestamp] +# @BRIEF Detect Unix timestamp strings (seconds or millis) and convert to 'YYYY-MM-DD' for Date columns. +def _normalize_timestamp_value(value: Any) -> Optional[str]: + """Detect Unix timestamp values and convert to date string. + + Handles: + - Integer/float Unix timestamps (seconds or milliseconds) + - String representations of Unix timestamps (e.g. '1726358400000.0') + + Returns 'YYYY-MM-DD' if conversion succeeds, None if value is not a timestamp. + """ + # Try numeric conversion first + try: + ts = float(value) + except (ValueError, TypeError): + return None + + # Heuristic: Unix timestamps in seconds are ~10 digits (1e9 range for 2001-2033) + # Unix timestamps in milliseconds are ~13 digits (1e12 range for 2001-2033) + if 1e9 <= ts < 1e12: + # Already in seconds + pass + elif 1e12 <= ts < 1e15: + # Milliseconds — convert to seconds + ts = ts / 1000.0 + else: + # Not a plausible Unix timestamp + return None + + try: + dt = datetime.fromtimestamp(ts, tz=timezone.utc) + return dt.strftime("%Y-%m-%d") + except (OSError, OverflowError, ValueError): + return None +# #endregion _normalize_timestamp_value + + +# #region _quote_identifier [C:2] [TYPE Function] [SEMANTICS translate,sql,quoting] +# @BRIEF Quote a SQL identifier per dialect rules — double quotes for PostgreSQL, backticks for ClickHouse. def _quote_identifier(identifier: str, dialect: str) -> str: """Quote a SQL identifier per dialect rules.""" if not identifier: @@ -38,50 +68,67 @@ def _quote_identifier(identifier: str, dialect: str) -> str: else: # Generic ANSI double-quote return f'"{cleaned}"' -# [/DEF:_quote_identifier:Function] +# #endregion _quote_identifier -# [DEF:_encode_sql_value:Function] -# @PURPOSE: Encode a Python value into a SQL-safe literal for INSERT VALUES. -# @PRE: value is a Python primitive or None. -# @POST: Returns SQL-safe string literal representation. -def _encode_sql_value(value: Any) -> str: - """Encode a Python value into a SQL-safe literal.""" +# #region _encode_sql_value [C:2] [TYPE Function] [SEMANTICS translate,sql,encoding] +# @BRIEF Encode a Python value into a SQL-safe literal for INSERT VALUES, with ClickHouse timestamp normalization. +def _encode_sql_value(value: Any, dialect: Optional[str] = None) -> str: + """Encode a Python value into a SQL-safe literal. + + For ClickHouse dialect, attempts to detect Unix timestamp strings + and convert them to 'YYYY-MM-DD' format for Date column compatibility. + """ if value is None: return "NULL" if isinstance(value, bool): return "TRUE" if value else "FALSE" + + # For ClickHouse: try to normalize timestamp-like string values + if dialect in CLICKHOUSE_DIALECTS and isinstance(value, str) and value: + normalized = _normalize_timestamp_value(value) + if normalized: + return f"'{normalized}'" + if isinstance(value, (int, float)): return str(value) # String — escape single quotes by doubling them escaped = str(value).replace("'", "''") return f"'{escaped}'" -# [/DEF:_encode_sql_value:Function] +# #endregion _encode_sql_value -# [DEF:_build_values_clause:Function] -# @PURPOSE: Build a VALUES clause for multiple rows. -# @PRE: columns list is non-empty; rows is a list of dicts. -# @POST: Returns SQL VALUES clause string. -def _build_values_clause(columns: List[str], rows: List[Dict[str, Any]]) -> str: - """Build VALUES (...) clause for multiple rows.""" +# #region _build_values_clause [C:2] [TYPE Function] [SEMANTICS translate,sql,values] +# @BRIEF Build a VALUES clause for multiple rows with per-column value encoding and dialect-aware quoting. +def _build_values_clause(columns: List[str], rows: List[Dict[str, Any]], dialect: Optional[str] = None) -> str: + """Build VALUES (...) clause for multiple rows. + + NOTE: columns may be quoted (e.g. '"col"' or '`col`') for the SQL column list, + but row dicts have UNQUOTED keys. Strip quotes before value lookup. + """ value_groups = [] for row in rows: - values = [_encode_sql_value(row.get(col)) for col in columns] + values = [] + for col in columns: + # Strip quoting for value lookup — row keys are unquoted + lookup_key = col.strip('"').strip('`').strip('[]') + val = row.get(lookup_key) + values.append(_encode_sql_value(val, dialect=dialect)) value_groups.append(f"({', '.join(values)})") return ",\n".join(value_groups) -# [/DEF:_build_values_clause:Function] +# #endregion _build_values_clause -# [DEF:generate_insert_sql:Function] -# @PURPOSE: Generate a dialect-aware INSERT SQL statement. -# @PRE: target_table, columns are non-empty. -# @POST: Returns SQL string or raises ValueError on invalid input. +# #region generate_insert_sql [C:3] [TYPE Function] [SEMANTICS translate,sql,insert] +# @BRIEF Generate a dialect-aware plain INSERT SQL statement for the given table, columns, and rows. +# @RELATION DEPENDS_ON -> [_quote_identifier] +# @RELATION DEPENDS_ON -> [_build_values_clause] def generate_insert_sql( target_schema: Optional[str], target_table: str, columns: List[str], rows: List[Dict[str, Any]], + dialect: Optional[str] = None, ) -> str: """Generate a plain INSERT SQL.""" with belief_scope("generate_insert_sql"): @@ -93,7 +140,7 @@ def generate_insert_sql( raise ValueError("At least one row is required for INSERT SQL generation") col_list = ", ".join(columns) - values = _build_values_clause(columns, rows) + values = _build_values_clause(columns, rows, dialect=dialect) table_ref = target_table if target_schema: @@ -101,19 +148,20 @@ def generate_insert_sql( sql = f"INSERT INTO {table_ref} ({col_list})\nVALUES\n{values};" return sql -# [/DEF:generate_insert_sql:Function] +# #endregion generate_insert_sql -# [DEF:generate_upsert_sql:Function] -# @PURPOSE: Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE. -# @PRE: dialect is postgresql-compatible. target_table, columns, key_columns are non-empty. -# @POST: Returns UPSERT SQL string or raises ValueError. +# #region generate_upsert_sql [C:3] [TYPE Function] [SEMANTICS translate,sql,upsert] +# @BRIEF Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE / DO NOTHING. +# @RELATION DEPENDS_ON -> [_quote_identifier] +# @RELATION DEPENDS_ON -> [_build_values_clause] def generate_upsert_sql( target_schema: Optional[str], target_table: str, columns: List[str], key_columns: List[str], rows: List[Dict[str, Any]], + dialect: Optional[str] = None, ) -> str: """Generate INSERT ... ON CONFLICT (key_cols) DO UPDATE SET ... SQL.""" with belief_scope("generate_upsert_sql"): @@ -128,7 +176,7 @@ def generate_upsert_sql( col_list = ", ".join(columns) key_list = ", ".join(key_columns) - values = _build_values_clause(columns, rows) + values = _build_values_clause(columns, rows, dialect=dialect) table_ref = target_table if target_schema: @@ -150,21 +198,23 @@ def generate_upsert_sql( f"ON CONFLICT ({key_list}) {conflict_action};" ) return sql -# [/DEF:generate_upsert_sql:Function] +# #endregion generate_upsert_sql -# [DEF:SQLGenerator:Class] -# @COMPLEXITY: 3 -# @PURPOSE: Generate safe, dialect-appropriate SQL INSERT/UPSERT statements. -# @PRE: Job has target_schema, target_table, key columns configured. -# @POST: Returns generated SQL string for the target dialect. +# #region SQLGenerator [C:4] [TYPE Class] [SEMANTICS translate,sql,generator] +# @BRIEF Generate safe, dialect-appropriate SQL INSERT/UPSERT statements with batching support. +# @PRE Job has target_schema, target_table, key columns configured. +# @POST Returns generated SQL string for the target dialect. +# @SIDE_EFFECT None — pure SQL generation. +# @RELATION DEPENDS_ON -> [generate_insert_sql] +# @RELATION DEPENDS_ON -> [generate_upsert_sql] class SQLGenerator: - # [DEF:SQLGenerator.generate:Function] - # @PURPOSE: Generate SQL for a set of rows, detecting dialect from the job configuration. - # @PRE: dialect is a supported database dialect. columns list is non-empty. rows is non-empty. - # @POST: Returns tuple of (sql_string, statement_count). - # @SIDE_EFFECT: None — pure SQL generation. + # #region SQLGenerator.generate [C:4] [TYPE Function] [SEMANTICS translate,sql,generate] + # @BRIEF Generate SQL for a set of rows, detecting dialect from the job configuration. + # @PRE dialect is a supported database dialect. columns list is non-empty. rows is non-empty. + # @POST Returns tuple of (sql_string, statement_count). + # @SIDE_EFFECT None — pure SQL generation. @staticmethod def generate( dialect: str, @@ -236,6 +286,7 @@ class SQLGenerator: columns=quoted_columns, key_columns=quoted_key_columns, rows=rows, + dialect=dialect, ) else: sql = generate_insert_sql( @@ -243,6 +294,7 @@ class SQLGenerator: target_table=table_ref, columns=quoted_columns, rows=rows, + dialect=dialect, ) elif dialect in CLICKHOUSE_DIALECTS: # ClickHouse: plain INSERT, no ON CONFLICT support @@ -251,6 +303,7 @@ class SQLGenerator: target_table=table_ref, columns=quoted_columns, rows=rows, + dialect=dialect, ) if use_upsert: logger.reason("ClickHouse UPSERT not supported; using plain INSERT", { @@ -263,6 +316,7 @@ class SQLGenerator: target_table=table_ref, columns=quoted_columns, rows=rows, + dialect=dialect, ) logger.reflect("SQL generated", { @@ -271,12 +325,11 @@ class SQLGenerator: "sql_length": len(sql), }) return sql, len(rows) - # [/DEF:SQLGenerator.generate:Function] + # #endregion SQLGenerator.generate - # [DEF:SQLGenerator.generate_batch:Function] - # @PURPOSE: Generate separate INSERT statements for each row (batch-safe version). - # @PRE: Same as generate(). - # @POST: Returns list of (sql_string, row_index) tuples. + # #region SQLGenerator.generate_batch [C:3] [TYPE Function] [SEMANTICS translate,sql,batch] + # @BRIEF Generate separate INSERT statements for each row chunk (batch-safe version). + # @RELATION DEPENDS_ON -> [SQLGenerator.generate] @staticmethod def generate_batch( dialect: str, @@ -312,8 +365,8 @@ class SQLGenerator: statements.append((sql, count)) return statements - # [/DEF:SQLGenerator.generate_batch:Function] + # #endregion SQLGenerator.generate_batch -# [/DEF:SQLGenerator:Class] -# [/DEF:SQLGenerator:Module] +# #endregion SQLGenerator +# #endregion SQLGenerator diff --git a/backend/src/plugins/translate/superset_executor.py b/backend/src/plugins/translate/superset_executor.py index b653ca3c..b1fbc9df 100644 --- a/backend/src/plugins/translate/superset_executor.py +++ b/backend/src/plugins/translate/superset_executor.py @@ -1,17 +1,17 @@ -# [DEF:SupersetSqlLabExecutor:Module] -# @COMPLEXITY: 4 -# @SEMANTICS: translate, superset, sqllab, execute -# @PURPOSE: Submit SQL to Superset SQL Lab API and poll execution status. -# @LAYER: Infra -# @RELATION: DEPENDS_ON -> [SupersetClient] -# @RELATION: DEPENDS_ON -> [ConfigManager] -# @PRE: Valid Superset environment configuration and authenticated client. -# @POST: SQL is submitted to Superset SQL Lab; execution reference is returned. -# @SIDE_EFFECT: Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status. -# @RATIONALE: Direct SQL Lab API submission provides audit trail and execution monitoring within Superset. -# @REJECTED: Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC. +# #region SupersetSqlLabExecutor [C:5] [TYPE Module] [SEMANTICS translate,superset,sqllab,execute] +# @BRIEF Submit SQL to Superset SQL Lab API and poll execution status with retry. +# @LAYER Infra +# @RELATION DEPENDS_ON -> [SupersetClient] +# @RELATION DEPENDS_ON -> [ConfigManager] +# @PRE Valid Superset environment configuration and authenticated client. +# @POST SQL is submitted to Superset SQL Lab; execution reference is returned. +# @SIDE_EFFECT Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status. +# @DATA_CONTRACT Input[sql:str, database_id:Optional] -> Output[Dict with query_id, status, error_message] +# @INVARIANT Polling respects max_polls limit; timeout returns structured error. +# @RATIONALE Direct SQL Lab API submission provides audit trail and execution monitoring within Superset. +# @REJECTED Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC. -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, Union from datetime import datetime, timezone import json import time @@ -22,25 +22,26 @@ from ...core.config_manager import ConfigManager from ...core.superset_client import SupersetClient -# [DEF:SupersetSqlLabExecutor:Class] -# @COMPLEXITY: 4 -# @PURPOSE: Submit SQL to Superset SQL Lab API with polling and status tracking. -# @PRE: Valid environment ID and ConfigManager. -# @POST: SQL submitted to Superset; execution reference recorded. -# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status. +# #region SupersetSqlLabExecutor [C:5] [TYPE Class] [SEMANTICS translate,superset,sqllab] +# @BRIEF Submit SQL to Superset SQL Lab API with polling and status tracking. +# @PRE Valid environment ID and ConfigManager. +# @POST SQL submitted to Superset; execution reference recorded; polling completes. +# @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status. +# @DATA_CONTRACT Input[env_id:str, database_id:Optional] -> Output[Dict with query_id, status] +# @INVARIANT database_id resolved lazily; polling capped at max_polls. class SupersetSqlLabExecutor: - def __init__(self, config_manager: ConfigManager, env_id: str): + def __init__(self, config_manager: ConfigManager, env_id: str, database_id: Optional[int] = None): self.config_manager = config_manager self.env_id = env_id self._client: Optional[SupersetClient] = None - self._database_id: Optional[int] = None + self._database_id: Optional[int] = database_id - # [DEF:_get_client:Function] - # @PURPOSE: Lazy-initialize SupersetClient for the configured environment. - # @PRE: env_id must correspond to a valid environment config. - # @POST: Returns authenticated SupersetClient. - # @SIDE_EFFECT: Authenticates against Superset API. + # #region _get_client [C:4] [TYPE Function] [SEMANTICS translate,superset,client] + # @BRIEF Lazy-initialize SupersetClient for the configured environment. + # @PRE env_id must correspond to a valid environment config. + # @POST Returns authenticated SupersetClient. + # @SIDE_EFFECT Authenticates against Superset API on first call. def _get_client(self) -> SupersetClient: with belief_scope("SupersetSqlLabExecutor._get_client"): if self._client is not None: @@ -53,13 +54,13 @@ class SupersetSqlLabExecutor: self._client = SupersetClient(env_config) return self._client - # [/DEF:_get_client:Function] + # #endregion _get_client - # [DEF:resolve_database_id:Function] - # @PURPOSE: Resolve the target database ID from the environment. - # @PRE: database_name or database_id should be known. - # @POST: Returns database_id integer or raises ValueError. - # @SIDE_EFFECT: Fetches databases list from Superset. + # #region resolve_database_id [C:4] [TYPE Function] [SEMANTICS translate,database,resolve] + # @BRIEF Resolve the target database ID from the environment by name or default. + # @PRE Superset environment is reachable and has databases. + # @POST Returns database_id integer or raises ValueError. + # @SIDE_EFFECT Fetches databases list from Superset API. def resolve_database_id(self, database_name: Optional[str] = None) -> int: with belief_scope("SupersetSqlLabExecutor.resolve_database_id"): client = self._get_client() @@ -89,37 +90,71 @@ class SupersetSqlLabExecutor: "database_id": self._database_id, }) return self._database_id - # [/DEF:resolve_database_id:Function] + # #endregion resolve_database_id - # [DEF:execute_sql:Function] - # @PURPOSE: Submit SQL to Superset SQL Lab and return execution tracking info. - # @PRE: sql is valid SQL string. database_id is a valid Superset DB ID. - # @POST: Returns execution result dict with query_id and status. - # @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/. + # #region resolve_database_uuid [C:4] [TYPE Function] [SEMANTICS translate,database,uuid] + # @BRIEF Resolve a database UUID to a numeric database ID from Superset. + # @PRE uuid_str is a valid Superset database UUID. + # @POST Returns numeric database_id or raises ValueError if not found. + # @SIDE_EFFECT Fetches databases list from Superset API. + def resolve_database_uuid(self, uuid_str: str) -> int: + with belief_scope("SupersetSqlLabExecutor.resolve_database_uuid"): + client = self._get_client() + _, databases = client.get_databases( + query={"columns": ["id", "uuid", "database_name"]} + ) + if not databases: + raise ValueError("No databases found in Superset environment") + + for db in databases: + db_uuid = db.get("uuid") or "" + if db_uuid.lower() == uuid_str.lower(): + db_id = db["id"] + self._database_id = db_id + logger.reason("Resolved database ID by UUID", { + "uuid": uuid_str, + "database_id": db_id, + "database_name": db.get("database_name"), + }) + return db_id + + raise ValueError( + f"Database with UUID '{uuid_str}' not found in Superset environment" + ) + # #endregion resolve_database_uuid + + # #region execute_sql [C:4] [TYPE Function] [SEMANTICS translate,sql,execute] + # @BRIEF Submit SQL to Superset SQL Lab and return execution tracking info. + # @PRE sql is valid SQL string. database_id is a valid Superset DB ID. + # @POST Returns execution result dict with query_id and status. + # @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/. def execute_sql( self, sql: str, - database_id: Optional[int] = None, - run_async: bool = True, + database_id: Optional[Union[int, str]] = None, ) -> Dict[str, Any]: with belief_scope("SupersetSqlLabExecutor.execute_sql"): client = self._get_client() db_id = database_id or self._database_id - if not db_id: + if db_id is None: db_id = self.resolve_database_id() + # Handle UUID string passed as database_id — resolve to numeric ID + if isinstance(db_id, str): + try: + db_id = int(db_id) + except (ValueError, TypeError): + db_id = self.resolve_database_uuid(db_id) logger.reason("Submitting SQL to Superset SQL Lab", { "database_id": db_id, "sql_length": len(sql), - "run_async": run_async, }) payload = { "database_id": db_id, "sql": sql, - "runAsync": run_async, + "client_id": uuid.uuid4().hex[:10], "schema": None, - "tab": "translation-insert", } try: @@ -133,14 +168,22 @@ class SupersetSqlLabExecutor: logger.explore("SQL Lab execute failed", {"error": str(e)}) raise ValueError(f"Superset SQL Lab execute failed: {e}") - # Parse response + # Parse response — try multiple formats result = response if isinstance(response, dict) else {} - query_id = result.get("query_id") or result.get("id") + query_id = (result.get("query_id") + or result.get("id") + or result.get("sql_lab_session_ref") + or (result.get("result") or {}).get("query_id") + or (result.get("result") or {}).get("id")) status = result.get("status", "unknown") logger.reason("SQL Lab execute response", { "query_id": query_id, "status": status, + "has_query_id": query_id is not None, + "has_errors": "errors" in result or "error" in result, + "response_keys": list(result.keys()) if isinstance(result, dict) else type(result).__name__, + "response_preview": str(result)[:1000] if isinstance(result, dict) else str(result)[:1000], }) return { @@ -149,13 +192,13 @@ class SupersetSqlLabExecutor: "raw_response": result, "database_id": db_id, } - # [/DEF:execute_sql:Function] + # #endregion execute_sql - # [DEF:poll_execution_status:Function] - # @PURPOSE: Poll Superset for SQL execution status until completion or timeout. - # @PRE: query_id is a valid Superset query ID. - # @POST: Returns final execution status dict. - # @SIDE_EFFECT: Makes HTTP GET requests to Superset. + # #region poll_execution_status [C:4] [TYPE Function] [SEMANTICS translate,poll,status] + # @BRIEF Poll Superset for SQL execution status until completion or timeout. + # @PRE query_id is a valid Superset query ID. + # @POST Returns final execution status dict with success/failure/timeout. + # @SIDE_EFFECT Makes HTTP GET requests to Superset API. def poll_execution_status( self, query_id: str, @@ -240,17 +283,17 @@ class SupersetSqlLabExecutor: "error_message": f"Polling timed out after {max_polls} attempts", "results": None, } - # [/DEF:poll_execution_status:Function] + # #endregion poll_execution_status - # [DEF:execute_and_poll:Function] - # @PURPOSE: Execute SQL and wait for completion. One-shot convenience method. - # @PRE: sql is valid SQL. - # @POST: Returns final execution result. - # @SIDE_EFFECT: Makes HTTP calls to Superset API. + # #region execute_and_poll [C:4] [TYPE Function] [SEMANTICS translate,sql,execute,poll] + # @BRIEF Execute SQL and wait for completion. One-shot convenience method. + # @PRE sql is valid SQL. + # @POST Returns final execution result. + # @SIDE_EFFECT Makes HTTP calls to Superset API. def execute_and_poll( self, sql: str, - database_id: Optional[int] = None, + database_id: Optional[Union[int, str]] = None, max_polls: int = 60, poll_interval_seconds: float = 2.0, ) -> Dict[str, Any]: @@ -258,7 +301,6 @@ class SupersetSqlLabExecutor: exec_result = self.execute_sql( sql=sql, database_id=database_id, - run_async=True, ) query_id = exec_result.get("query_id") @@ -277,13 +319,13 @@ class SupersetSqlLabExecutor: max_polls=max_polls, poll_interval_seconds=poll_interval_seconds, ) - # [/DEF:execute_and_poll:Function] + # #endregion execute_and_poll - # [DEF:get_query_results:Function] - # @PURPOSE: Fetch the results of a completed query. - # @PRE: query_id is a valid Superset query ID. - # @POST: Returns query results if available. - # @SIDE_EFFECT: Makes HTTP GET to Superset. + # #region get_query_results [C:4] [TYPE Function] [SEMANTICS translate,query,results] + # @BRIEF Fetch the results of a completed query from Superset. + # @PRE query_id is a valid Superset query ID. + # @POST Returns query results if available, or error dict. + # @SIDE_EFFECT Makes HTTP GET to Superset API. def get_query_results(self, query_id: str) -> Dict[str, Any]: with belief_scope("SupersetSqlLabExecutor.get_query_results"): client = self._get_client() @@ -309,8 +351,8 @@ class SupersetSqlLabExecutor: "error_message": str(e), "results": None, } - # [/DEF:get_query_results:Function] + # #endregion get_query_results -# [/DEF:SupersetSqlLabExecutor:Class] -# [/DEF:SupersetSqlLabExecutor:Module] +# #endregion SupersetSqlLabExecutor +# #endregion SupersetSqlLabExecutor diff --git a/backend/src/schemas/__init__.py b/backend/src/schemas/__init__.py index 0fddfa24..1fc9bb5f 100644 --- a/backend/src/schemas/__init__.py +++ b/backend/src/schemas/__init__.py @@ -1,3 +1,2 @@ -# [DEF:SchemasPackage:Package] -# @PURPOSE: API schema package root. -# [/DEF:SchemasPackage:Package] +# #region SchemasPackage [C:1] [TYPE Package] +# #endregion SchemasPackage diff --git a/backend/src/schemas/translate.py b/backend/src/schemas/translate.py index 9c3c0324..67ed6b2a 100644 --- a/backend/src/schemas/translate.py +++ b/backend/src/schemas/translate.py @@ -1,9 +1,6 @@ -# [DEF:TranslateSchemas:Module] -# @COMPLEXITY: 2 -# @SEMANTICS: translate, schemas, pydantic -# @PURPOSE: Pydantic v2 schemas for translation API request/response serialization. -# @LAYER: API -# @RELATION: DEPENDS_ON -> pydantic +# #region TranslateSchemas [C:2] [TYPE Module] +# @BRIEF Pydantic v2 schemas for translation API request/response serialization. +# @LAYER Domain from datetime import datetime from typing import Any, Dict, List, Optional @@ -11,8 +8,7 @@ from pydantic import BaseModel, Field import json -# [DEF:TranslateJobCreate:Class] -# @PURPOSE: Schema for creating a new translation job. +# #region TranslateJobCreate [C:1] [TYPE Class] class TranslateJobCreate(BaseModel): name: str description: Optional[str] = None @@ -33,11 +29,11 @@ class TranslateJobCreate(BaseModel): upsert_strategy: str = Field("MERGE", description="UPSERT strategy: MERGE, INSERT, UPDATE") dictionary_ids: Optional[List[str]] = Field(default_factory=list, description="Associated terminology dictionary IDs") environment_id: Optional[str] = Field(None, description="Superset environment ID") -# [/DEF:TranslateJobCreate:Class] + target_database_id: Optional[str] = Field(None, description="Superset database ID for INSERT via SQL Lab") +# #endregion TranslateJobCreate -# [DEF:TranslateJobUpdate:Class] -# @PURPOSE: Schema for updating an existing translation job. +# #region TranslateJobUpdate [C:1] [TYPE Class] class TranslateJobUpdate(BaseModel): name: Optional[str] = None description: Optional[str] = None @@ -59,11 +55,11 @@ class TranslateJobUpdate(BaseModel): status: Optional[str] = None dictionary_ids: Optional[List[str]] = None environment_id: Optional[str] = None -# [/DEF:TranslateJobUpdate:Class] + target_database_id: Optional[str] = None +# #endregion TranslateJobUpdate -# [DEF:TranslateJobResponse:Class] -# @PURPOSE: Schema for translation job API responses. +# #region TranslateJobResponse [C:1] [TYPE Class] class TranslateJobResponse(BaseModel): id: str name: str @@ -89,24 +85,24 @@ class TranslateJobResponse(BaseModel): updated_at: Optional[datetime] = None dictionary_ids: Optional[List[str]] = None environment_id: Optional[str] = None + target_database_id: Optional[str] = None class Config: from_attributes = True -# [/DEF:TranslateJobResponse:Class] +# #endregion TranslateJobResponse -# [DEF:DatasourceColumnResponse:Class] -# @PURPOSE: Schema for datasource column metadata response. +# #region DatasourceColumnResponse [C:1] [TYPE Class] class DatasourceColumnResponse(BaseModel): name: str type: Optional[str] = None is_physical: bool = True is_dttm: bool = False description: Optional[str] = None +# #endregion DatasourceColumnResponse -# [DEF:DatasourceColumnsResponse:Class] -# @PURPOSE: Schema for datasource columns endpoint response. +# #region DatasourceColumnsResponse [C:1] [TYPE Class] class DatasourceColumnsResponse(BaseModel): datasource_id: int datasource_name: Optional[str] = None @@ -116,11 +112,10 @@ class DatasourceColumnsResponse(BaseModel): class Config: from_attributes = True -# [/DEF:DatasourceColumnsResponse:Class] +# #endregion DatasourceColumnsResponse -# [DEF:DuplicateJobResponse:Class] -# @PURPOSE: Schema for duplicate job response. +# #region DuplicateJobResponse [C:1] [TYPE Class] class DuplicateJobResponse(BaseModel): id: str name: str @@ -128,32 +123,29 @@ class DuplicateJobResponse(BaseModel): class Config: from_attributes = True -# [/DEF:DuplicateJobResponse:Class] +# #endregion DuplicateJobResponse -# [DEF:DictionaryCreate:Class] -# @PURPOSE: Schema for creating a new terminology dictionary. +# #region DictionaryCreate [C:1] [TYPE Class] class DictionaryCreate(BaseModel): name: str description: Optional[str] = None source_dialect: str target_dialect: str is_active: bool = True -# [/DEF:DictionaryCreate:Class] +# #endregion DictionaryCreate -# [DEF:DictionaryImport:Class] -# @PURPOSE: Schema for importing entries into a terminology dictionary. +# #region DictionaryImport [C:1] [TYPE Class] class DictionaryImport(BaseModel): content: str = Field(..., description="CSV or TSV content as raw string") delimiter: Optional[str] = Field(None, description="Detected or forced delimiter: ',' or '\\t'. Auto-detect if omitted.") on_conflict: str = Field("overwrite", description="'overwrite' or 'keep_existing' or 'cancel'") preview_only: bool = Field(False, description="If true, return preview without applying") -# [/DEF:DictionaryImport:Class] +# #endregion DictionaryImport -# [DEF:DictionaryResponse:Class] -# @PURPOSE: Schema for terminology dictionary API responses. +# #region DictionaryResponse [C:1] [TYPE Class] class DictionaryResponse(BaseModel): id: str name: str @@ -168,11 +160,10 @@ class DictionaryResponse(BaseModel): class Config: from_attributes = True -# [/DEF:DictionaryResponse:Class] +# #endregion DictionaryResponse -# [DEF:DictionaryEntryCreate:Class] -# @PURPOSE: Schema for adding/editing a dictionary entry. +# #region DictionaryEntryCreate [C:1] [TYPE Class] class DictionaryEntryCreate(BaseModel): source_term: str = Field(..., description="Source term to translate") target_term: str = Field(..., description="Target/translated term") @@ -180,11 +171,10 @@ class DictionaryEntryCreate(BaseModel): class Config: from_attributes = True -# [/DEF:DictionaryEntryCreate:Class] +# #endregion DictionaryEntryCreate -# [DEF:DictionaryEntryResponse:Class] -# @PURPOSE: Schema for dictionary entry API responses. +# #region DictionaryEntryResponse [C:1] [TYPE Class] class DictionaryEntryResponse(BaseModel): id: str dictionary_id: str @@ -197,11 +187,10 @@ class DictionaryEntryResponse(BaseModel): class Config: from_attributes = True -# [/DEF:DictionaryEntryResponse:Class] +# #endregion DictionaryEntryResponse -# [DEF:DictionaryImportResult:Class] -# @PURPOSE: Schema for dictionary import result. +# #region DictionaryImportResult [C:1] [TYPE Class] class DictionaryImportResult(BaseModel): total: int = 0 created: int = 0 @@ -209,28 +198,26 @@ class DictionaryImportResult(BaseModel): skipped: int = 0 errors: List[Dict[str, Any]] = Field(default_factory=list) preview: List[Dict[str, Any]] = Field(default_factory=list, description="Preview rows with conflict flags") -# [/DEF:DictionaryImportResult:Class] +# #endregion DictionaryImportResult -# [DEF:PreviewRequest:Class] -# @PURPOSE: Schema for triggering a translation preview. +# #region PreviewRequest [C:1] [TYPE Class] class PreviewRequest(BaseModel): sample_size: int = Field(10, ge=1, le=100, description="Number of sample rows to preview") prompt_template: Optional[str] = Field(None, description="Optional custom prompt template") env_id: Optional[str] = Field(None, description="Superset environment ID for preview data fetch") +# #endregion PreviewRequest -# [DEF:PreviewRowUpdate:Class] -# @PURPOSE: Schema for approving/editing/rejecting a preview row. +# #region PreviewRowUpdate [C:1] [TYPE Class] class PreviewRowUpdate(BaseModel): action: str = Field(..., description="'approve', 'reject', or 'edit'") translation: Optional[str] = Field(None, description="Edited translation (required for 'edit' action)") feedback: Optional[str] = Field(None, description="Optional feedback/comment") -# [/DEF:PreviewRowUpdate:Class] +# #endregion PreviewRowUpdate -# [DEF:PreviewAcceptResponse:Class] -# @PURPOSE: Schema for preview accept response. +# #region PreviewAcceptResponse [C:1] [TYPE Class] class PreviewAcceptResponse(BaseModel): id: str job_id: str @@ -242,11 +229,10 @@ class PreviewAcceptResponse(BaseModel): class Config: from_attributes = True -# [/DEF:PreviewAcceptResponse:Class] +# #endregion PreviewAcceptResponse -# [DEF:CostEstimate:Class] -# @PURPOSE: Schema for cost estimation in preview response. +# #region CostEstimate [C:1] [TYPE Class] class CostEstimate(BaseModel): sample_size: int = 0 sample_prompt_tokens: int = 0 @@ -256,11 +242,10 @@ class CostEstimate(BaseModel): estimated_total_rows: int = 0 estimated_tokens: int = 0 estimated_cost: float = 0.0 -# [/DEF:CostEstimate:Class] +# #endregion CostEstimate -# [DEF:TermCorrectionSubmit:Class] -# @PURPOSE: Schema for submitting a term correction in a translation preview. +# #region TermCorrectionSubmit [C:1] [TYPE Class] class TermCorrectionSubmit(BaseModel): source_term: str incorrect_target_term: str @@ -268,29 +253,26 @@ class TermCorrectionSubmit(BaseModel): dictionary_id: Optional[str] = Field(None, description="Target dictionary ID (language-filtered)") origin_run_id: Optional[str] = Field(None, description="Run ID from which this correction originated") origin_row_key: Optional[str] = Field(None, description="Row key within the run") -# [/DEF:TermCorrectionSubmit:Class] +# #endregion TermCorrectionSubmit -# [DEF:TermCorrectionBulkSubmit:Class] -# @PURPOSE: Schema for submitting multiple term corrections atomically. +# #region TermCorrectionBulkSubmit [C:1] [TYPE Class] class TermCorrectionBulkSubmit(BaseModel): corrections: List[TermCorrectionSubmit] dictionary_id: str = Field(..., description="Target dictionary ID for all corrections") -# [/DEF:TermCorrectionBulkSubmit:Class] +# #endregion TermCorrectionBulkSubmit -# [DEF:CorrectionConflictResult:Class] -# @PURPOSE: Schema for reporting conflicts in correction submission. +# #region CorrectionConflictResult [C:1] [TYPE Class] class CorrectionConflictResult(BaseModel): source_term: str existing_target_term: str submitted_target_term: str action: str = "keep_existing" -# [/DEF:CorrectionConflictResult:Class] +# #endregion CorrectionConflictResult -# [DEF:CorrectionSubmitResponse:Class] -# @PURPOSE: Schema for correction submission response. +# #region CorrectionSubmitResponse [C:1] [TYPE Class] class CorrectionSubmitResponse(BaseModel): entry_id: Optional[str] = None action: str # "created", "updated", "conflict_detected", "skipped" @@ -298,20 +280,18 @@ class CorrectionSubmitResponse(BaseModel): target_term: str conflict: Optional[CorrectionConflictResult] = None message: Optional[str] = None -# [/DEF:CorrectionSubmitResponse:Class] +# #endregion CorrectionSubmitResponse -# [DEF:ScheduleConfig:Class] -# @PURPOSE: Schema for configuring a recurring translation schedule. +# #region ScheduleConfig [C:1] [TYPE Class] class ScheduleConfig(BaseModel): cron_expression: str = Field(..., description="Cron expression for scheduling (e.g. '0 2 * * *')") timezone: str = Field("UTC", description="Timezone for the cron schedule (e.g. 'UTC', 'Europe/Moscow')") is_active: bool = True -# [/DEF:ScheduleConfig:Class] +# #endregion ScheduleConfig -# [DEF:ScheduleResponse:Class] -# @PURPOSE: Schema for schedule API responses. +# #region ScheduleResponse [C:1] [TYPE Class] class ScheduleResponse(BaseModel): id: str job_id: str @@ -326,21 +306,19 @@ class ScheduleResponse(BaseModel): class Config: from_attributes = True -# [/DEF:ScheduleResponse:Class] +# #endregion ScheduleResponse -# [DEF:NextExecutionResponse:Class] -# @PURPOSE: Schema for next execution preview. +# #region NextExecutionResponse [C:1] [TYPE Class] class NextExecutionResponse(BaseModel): job_id: str cron_expression: str timezone: str next_executions: List[str] = [] -# [/DEF:NextExecutionResponse:Class] +# #endregion NextExecutionResponse -# [DEF:TranslationRunResponse:Class] -# @PURPOSE: Schema for translation run API responses. +# #region TranslationRunResponse [C:1] [TYPE Class] class TranslationRunResponse(BaseModel): id: str job_id: str @@ -364,22 +342,20 @@ class TranslationRunResponse(BaseModel): class Config: from_attributes = True -# [/DEF:TranslationRunResponse:Class] +# #endregion TranslationRunResponse -# [DEF:RunDetailResponse:Class] -# @PURPOSE: Schema for detailed run response including records, events, and config snapshot. +# #region RunDetailResponse [C:1] [TYPE Class] class RunDetailResponse(BaseModel): run: TranslationRunResponse records: List[Dict[str, Any]] = Field(default_factory=list, description="Paginated translation records") events: List[Dict[str, Any]] = Field(default_factory=list, description="Run events") event_invariants: Optional[Dict[str, Any]] = None batch_count: int = 0 -# [/DEF:RunDetailResponse:Class> +# #endregion RunDetailResponse -# [DEF:RunHistoryFilter:Class] -# @PURPOSE: Schema for filtering run history. +# #region RunHistoryFilter [C:1] [TYPE Class] class RunHistoryFilter(BaseModel): job_id: Optional[str] = None status: Optional[str] = None @@ -389,21 +365,19 @@ class RunHistoryFilter(BaseModel): date_to: Optional[datetime] = None page: int = 1 page_size: int = 20 -# [/DEF:RunHistoryFilter:Class> +# #endregion RunHistoryFilter -# [DEF:RunListResponse:Class] -# @PURPOSE: Schema for paginated run list response. +# #region RunListResponse [C:1] [TYPE Class] class RunListResponse(BaseModel): items: List[TranslationRunResponse] = [] total: int = 0 page: int = 1 page_size: int = 20 -# [/DEF:RunListResponse:Class> +# #endregion RunListResponse -# [DEF:AggregatedMetricsResponse:Class] -# @PURPOSE: Schema for aggregated per-job metrics. +# #region AggregatedMetricsResponse [C:1] [TYPE Class] class AggregatedMetricsResponse(BaseModel): job_id: str total_runs: int = 0 @@ -419,11 +393,10 @@ class AggregatedMetricsResponse(BaseModel): avg_duration_ms: Optional[float] = None last_run_at: Optional[datetime] = None next_scheduled_run: Optional[datetime] = None -# [/DEF:AggregatedMetricsResponse:Class] +# #endregion AggregatedMetricsResponse -# [DEF:PreviewRow:Class] -# @PURPOSE: A single row in a translation preview showing original and translated content side by side. +# #region PreviewRow [C:1] [TYPE Class] class PreviewRow(BaseModel): id: str source_sql: Optional[str] = None @@ -433,11 +406,10 @@ class PreviewRow(BaseModel): source_object_name: Optional[str] = None status: str = "PENDING" feedback: Optional[str] = None -# [/DEF:PreviewRow:Class] +# #endregion PreviewRow -# [DEF:TranslationPreviewResponse:Class] -# @PURPOSE: Schema for translation preview session API responses. +# #region TranslationPreviewResponse [C:1] [TYPE Class] class TranslationPreviewResponse(BaseModel): id: str job_id: str @@ -450,11 +422,10 @@ class TranslationPreviewResponse(BaseModel): class Config: from_attributes = True -# [/DEF:TranslationPreviewResponse:Class] +# #endregion TranslationPreviewResponse -# [DEF:TranslationBatchResponse:Class] -# @PURPOSE: Schema for translation batch API responses. +# #region TranslationBatchResponse [C:1] [TYPE Class] class TranslationBatchResponse(BaseModel): id: str run_id: str @@ -469,11 +440,10 @@ class TranslationBatchResponse(BaseModel): class Config: from_attributes = True -# [/DEF:TranslationBatchResponse:Class] +# #endregion TranslationBatchResponse -# [DEF:MetricsResponse:Class] -# @PURPOSE: Schema for translation metrics API responses. +# #region MetricsResponse [C:1] [TYPE Class] class MetricsResponse(BaseModel): job_id: str snapshot_date: datetime @@ -490,6 +460,6 @@ class MetricsResponse(BaseModel): class Config: from_attributes = True -# [/DEF:MetricsResponse:Class] +# #endregion MetricsResponse -# [/DEF:TranslateSchemas:Module] +# #endregion TranslateSchemas