busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

refactor: migrate translate engine to GRACE-Poly v2.6 semantic protocol

Browse Source 
- Convert all 84 contracts from legacy [DEF:] to #region/#endregion syntax
- Fix complexity tiers: 14 modules re-tiered (6 C4 route modules, 7 C4→C5 plugin services)
- Remove forbidden tags: @RATIONALE/@REJECTED stripped from C1–C4 contracts
- Add required tags: @PRE/@POST/@SIDE_EFFECT on C4, @RELATION on C3, @DATA_CONTRACT/@INVARIANT on C5
- Add belief runtime markers (reason/reflect/explore) to 7 service.py functions
- Fix @LAYER: route files → UI, plugins → Domain, superset_executor → Infra
- Fix pre-existing test mock_service fixture in test_orchestrator.py
- 196/196 translation tests pass, zero regressions
This commit is contained in:
busya
2026-05-12 14:32:28 +03:00
parent 72d034746d
commit 316d97ae25
29 changed files with 1594 additions and 1292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
backend/src/api/routes/translate/__init__.py
View File

@@ -1,17 +1,16 @@
# [DEF:TranslateRoutes:Module] # #region TranslateRoutes [C:4] [TYPE Module] [SEMANTICS api,routes,translate]
# @COMPLEXITY: 4 # @BRIEF API routes for LLM-based SQL/dashboard translation management including terminology dictionary CRUD and import.
# @SEMANTICS: api, routes, translate # @LAYER UI
# @PURPOSE: API routes for LLM-based SQL/dashboard translation management including terminology dictionary CRUD and import. # @RELATION DEPENDS_ON -> [TranslateJobResponse]
# @LAYER: UI (API) # @RELATION DEPENDS_ON -> [DictionaryManager]
# @RELATION: DEPENDS_ON -> [TranslateJobResponse] # @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION: DEPENDS_ON -> [DictionaryManager:Class] # @RELATION DEPENDS_ON -> [get_db]
# @RELATION: DEPENDS_ON -> [get_current_user] # @RELATION DEPENDS_ON -> [has_permission]
# @RELATION: DEPENDS_ON -> [get_db] # @RELATION DEPENDS_ON -> [ConfigManager]
# @RELATION: DEPENDS_ON -> [has_permission] # @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION: DEPENDS_ON -> [ConfigManager] # @PRE All sub-modules importable. Router instance available from ._router.
# @RELATION: DEPENDS_ON -> [SupersetClient] # @POST Package imports all route modules and re-exports router. Sub-routes registered on shared router.
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. # @SIDE_EFFECT Registers route handlers on shared APIRouter at import time.
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity.
""" """
Translate routes package. Translate routes package.
@@ -34,4 +33,4 @@ __all__ = [
"router", "router",
] ]
# [/DEF:TranslateRoutes:Module] # #endregion TranslateRoutes

40
backend/src/api/routes/translate/_correction_routes.py
View File

@@ -1,8 +1,12 @@
# [DEF:TranslateCorrectionRoutesModule:Module] # #region TranslateCorrectionRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,corrections]
# @COMPLEXITY: 2 # @BRIEF Term correction submission endpoints for applying user feedback to dictionary entries.
# @PURPOSE: Term correction submission endpoints. # @LAYER UI
# @LAYER: API # @RELATION DEPENDS_ON -> [DictionaryManager]
# @SEMANTICS: api, routes, translate, corrections # @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
# @PRE DB session initialized. User authenticated with translate.dictionary.edit permissions.
# @POST Term corrections submitted and applied to dictionary with conflict detection.
# @SIDE_EFFECT Creates/updates DictionaryEntry records in DB based on correction submissions.
from fastapi import APIRouter, Depends, HTTPException, status, Query from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
@@ -26,10 +30,12 @@ from ._router import router
# Corrections # Corrections
# ============================================================ # ============================================================
# [DEF:submit_correction:Function] # #region submit_correction [C:4] [TYPE Function] [SEMANTICS api,translate,corrections]
# @PURPOSE: Submit a term correction from a run result review. # @BRIEF Submit a single term correction from a run result review.
# @PRE: User has translate.dictionary.edit permission. # @PRE User has translate.dictionary.edit permission. Dictionary ID is provided.
# @POST: Correction applied with conflict detection. # @POST Correction applied with conflict detection. Result returned.
# @SIDE_EFFECT Creates/updates DictionaryEntry record in DB; logs correction origin.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.post("/corrections") @router.post("/corrections")
async def submit_correction( async def submit_correction(
payload: TermCorrectionSubmit, payload: TermCorrectionSubmit,
@@ -55,13 +61,15 @@ async def submit_correction(
return result return result
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:submit_correction:Function] # #endregion submit_correction
# [DEF:submit_bulk_corrections:Function] # #region submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS api,translate,corrections]
# @PURPOSE: Submit multiple term corrections atomically. # @BRIEF Submit multiple term corrections atomically with full conflict reporting.
# @PRE: User has translate.dictionary.edit permission. # @PRE User has translate.dictionary.edit permission. Dictionary ID is provided.
# @POST: All corrections applied or none with conflict list. # @POST All corrections applied or none with conflict list returned.
# @SIDE_EFFECT Creates/updates multiple DictionaryEntry records in DB atomically.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.post("/corrections/bulk") @router.post("/corrections/bulk")
async def submit_bulk_corrections( async def submit_bulk_corrections(
payload: TermCorrectionBulkSubmit, payload: TermCorrectionBulkSubmit,
@@ -92,6 +100,6 @@ async def submit_bulk_corrections(
return result return result
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:submit_bulk_corrections:Function] # #endregion submit_bulk_corrections
# [/DEF:TranslateCorrectionRoutesModule:Module] # #endregion TranslateCorrectionRoutesModule

140
backend/src/api/routes/translate/_dictionary_routes.py
View File

@@ -1,8 +1,12 @@
# [DEF:TranslateDictionaryRoutesModule:Module] # #region TranslateDictionaryRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,dictionaries]
# @COMPLEXITY: 3 # @BRIEF Terminology Dictionary CRUD, entries management, and import routes.
# @PURPOSE: Terminology Dictionary CRUD, entries management, and import routes. # @LAYER UI
# @LAYER: API # @RELATION DEPENDS_ON -> [DictionaryManager]
# @SEMANTICS: api, routes, translate, dictionaries # @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
# @PRE DB session initialized. User authenticated with translate.dictionary permissions.
# @POST Dictionaries and entries CRUD completed. Import executed with conflict resolution.
# @SIDE_EFFECT Reads/writes TerminologyDictionary and DictionaryEntry records to DB.
from fastapi import APIRouter, Depends, HTTPException, status, Query from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
@@ -29,10 +33,12 @@ from ._helpers import _dict_to_response, _get_dictionary_entry_counts
# Terminology Dictionaries # Terminology Dictionaries
# ============================================================ # ============================================================
# [DEF:list_dictionaries:Function] # #region list_dictionaries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]
# @PURPOSE: List all terminology dictionaries. # @BRIEF List all terminology dictionaries with pagination and entry counts.
# @PRE: User has translate.dictionary.view permission. # @PRE User has translate.dictionary.view permission.
# @POST: Returns list of dictionaries with total count. # @POST Returns paginated list of dictionaries with total count.
# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.get("/dictionaries") @router.get("/dictionaries")
async def list_dictionaries( async def list_dictionaries(
page: int = Query(1, ge=1), page: int = Query(1, ge=1),
@@ -50,13 +56,15 @@ async def list_dictionaries(
items = [_dict_to_response(d, counts.get(d.id, 0)) for d in dicts] items = [_dict_to_response(d, counts.get(d.id, 0)) for d in dicts]
logger.reflect("Dictionaries listed", {"total": total, "returned": len(items)}) logger.reflect("Dictionaries listed", {"total": total, "returned": len(items)})
return {"items": items, "total": total, "page": page, "page_size": page_size} return {"items": items, "total": total, "page": page, "page_size": page_size}
# [/DEF:list_dictionaries:Function] # #endregion list_dictionaries
# [DEF:get_dictionary:Function] # #region get_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]
# @PURPOSE: Get a single terminology dictionary by ID. # @BRIEF Get a single terminology dictionary by ID with entry count.
# @PRE: User has translate.dictionary.view permission. # @PRE User has translate.dictionary.view permission.
# @POST: Returns the dictionary with entry_count. # @POST Returns the dictionary with entry_count or 404.
# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.get("/dictionaries/{dictionary_id}") @router.get("/dictionaries/{dictionary_id}")
async def get_dictionary( async def get_dictionary(
dictionary_id: str, dictionary_id: str,
@@ -69,19 +77,21 @@ async def get_dictionary(
logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}")
try: try:
d = DictionaryManager.get_dictionary(db, dictionary_id) d = DictionaryManager.get_dictionary(db, dictionary_id)
from ...models.translate import DictionaryEntry from ....models.translate import DictionaryEntry
entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count() entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count()
logger.reflect("Dictionary found", {"id": dictionary_id}) logger.reflect("Dictionary found", {"id": dictionary_id})
return _dict_to_response(d, entry_count) return _dict_to_response(d, entry_count)
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_dictionary:Function] # #endregion get_dictionary
# [DEF:create_dictionary:Function] # #region create_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]
# @PURPOSE: Create a new terminology dictionary. # @BRIEF Create a new terminology dictionary.
# @PRE: User has translate.dictionary.create permission. # @PRE User has translate.dictionary.create permission. Payload validated.
# @POST: Returns the created dictionary. # @POST Dictionary created and returned with zero entries.
# @SIDE_EFFECT Creates TerminologyDictionary record in DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.post("/dictionaries", status_code=status.HTTP_201_CREATED) @router.post("/dictionaries", status_code=status.HTTP_201_CREATED)
async def create_dictionary( async def create_dictionary(
payload: DictionaryCreate, payload: DictionaryCreate,
@@ -103,13 +113,15 @@ async def create_dictionary(
) )
logger.reflect("Dictionary created", {"id": d.id}) logger.reflect("Dictionary created", {"id": d.id})
return _dict_to_response(d, 0) return _dict_to_response(d, 0)
# [/DEF:create_dictionary:Function] # #endregion create_dictionary
# [DEF:update_dictionary:Function] # #region update_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]
# @PURPOSE: Update an existing terminology dictionary. # @BRIEF Update an existing terminology dictionary.
# @PRE: User has translate.dictionary.edit permission. # @PRE User has translate.dictionary.edit permission. Dictionary exists.
# @POST: Returns the updated dictionary. # @POST Dictionary updated and returned with current entry count.
# @SIDE_EFFECT Updates TerminologyDictionary record in DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.put("/dictionaries/{dictionary_id}") @router.put("/dictionaries/{dictionary_id}")
async def update_dictionary( async def update_dictionary(
dictionary_id: str, dictionary_id: str,
@@ -131,19 +143,21 @@ async def update_dictionary(
target_dialect=payload.target_dialect, target_dialect=payload.target_dialect,
is_active=payload.is_active, is_active=payload.is_active,
) )
from ...models.translate import DictionaryEntry from ....models.translate import DictionaryEntry
entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count() entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count()
logger.reflect("Dictionary updated", {"id": dictionary_id}) logger.reflect("Dictionary updated", {"id": dictionary_id})
return _dict_to_response(d, entry_count) return _dict_to_response(d, entry_count)
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:update_dictionary:Function] # #endregion update_dictionary
# [DEF:delete_dictionary:Function] # #region delete_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]
# @PURPOSE: Delete a terminology dictionary, blocked if attached to active/scheduled jobs. # @BRIEF Delete a terminology dictionary, blocked if attached to active/scheduled jobs.
# @PRE: User has translate.dictionary.delete permission. # @PRE User has translate.dictionary.delete permission. Dictionary exists and is not in use.
# @POST: Dictionary is deleted. # @POST Dictionary is deleted or 409 if attached to active jobs.
# @SIDE_EFFECT Deletes TerminologyDictionary record from DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.delete("/dictionaries/{dictionary_id}", status_code=status.HTTP_204_NO_CONTENT) @router.delete("/dictionaries/{dictionary_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_dictionary( async def delete_dictionary(
dictionary_id: str, dictionary_id: str,
@@ -162,17 +176,19 @@ async def delete_dictionary(
if "active/scheduled" in str(e): if "active/scheduled" in str(e):
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e)) raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:delete_dictionary:Function] # #endregion delete_dictionary
# ============================================================ # ============================================================
# Dictionary Entries CRUD # Dictionary Entries CRUD
# ============================================================ # ============================================================
# [DEF:list_dictionary_entries:Function] # #region list_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]
# @PURPOSE: List entries for a dictionary. # @BRIEF List entries for a dictionary with pagination.
# @PRE: User has translate.dictionary.view permission. # @PRE User has translate.dictionary.view permission. Dictionary exists.
# @POST: Returns paginated list of entries. # @POST Returns paginated list of entries.
# @SIDE_EFFECT Reads DictionaryEntry records from DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.get("/dictionaries/{dictionary_id}/entries") @router.get("/dictionaries/{dictionary_id}/entries")
async def list_dictionary_entries( async def list_dictionary_entries(
dictionary_id: str, dictionary_id: str,
@@ -204,13 +220,15 @@ async def list_dictionary_entries(
return {"items": items, "total": total, "page": page, "page_size": page_size} return {"items": items, "total": total, "page": page, "page_size": page_size}
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:list_dictionary_entries:Function] # #endregion list_dictionary_entries
# [DEF:add_dictionary_entry:Function] # #region add_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]
# @PURPOSE: Add a new entry to a dictionary. # @BRIEF Add a new entry to a dictionary.
# @PRE: User has translate.dictionary.edit permission. # @PRE User has translate.dictionary.edit permission. Dictionary exists.
# @POST: Entry is created. # @POST Entry created and returned.
# @SIDE_EFFECT Creates DictionaryEntry record in DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.post("/dictionaries/{dictionary_id}/entries", status_code=status.HTTP_201_CREATED) @router.post("/dictionaries/{dictionary_id}/entries", status_code=status.HTTP_201_CREATED)
async def add_dictionary_entry( async def add_dictionary_entry(
dictionary_id: str, dictionary_id: str,
@@ -244,13 +262,15 @@ async def add_dictionary_entry(
if "already exists" in str(e): if "already exists" in str(e):
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e)) raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:add_dictionary_entry:Function] # #endregion add_dictionary_entry
# [DEF:edit_dictionary_entry:Function] # #region edit_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]
# @PURPOSE: Update an existing dictionary entry. # @BRIEF Update an existing dictionary entry.
# @PRE: User has translate.dictionary.edit permission. # @PRE User has translate.dictionary.edit permission. Entry exists.
# @POST: Entry is updated. # @POST Entry updated and returned.
# @SIDE_EFFECT Updates DictionaryEntry record in DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.put("/dictionaries/{dictionary_id}/entries/{entry_id}") @router.put("/dictionaries/{dictionary_id}/entries/{entry_id}")
async def edit_dictionary_entry( async def edit_dictionary_entry(
dictionary_id: str, dictionary_id: str,
@@ -285,13 +305,15 @@ async def edit_dictionary_entry(
if "already exists" in str(e): if "already exists" in str(e):
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e)) raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:edit_dictionary_entry:Function] # #endregion edit_dictionary_entry
# [DEF:delete_dictionary_entry:Function] # #region delete_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]
# @PURPOSE: Delete a dictionary entry. # @BRIEF Delete a dictionary entry.
# @PRE: User has translate.dictionary.edit permission. # @PRE User has translate.dictionary.edit permission. Entry exists.
# @POST: Entry is deleted. # @POST Entry is deleted.
# @SIDE_EFFECT Deletes DictionaryEntry record from DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.delete("/dictionaries/{dictionary_id}/entries/{entry_id}", status_code=status.HTTP_204_NO_CONTENT) @router.delete("/dictionaries/{dictionary_id}/entries/{entry_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_dictionary_entry( async def delete_dictionary_entry(
dictionary_id: str, dictionary_id: str,
@@ -309,13 +331,15 @@ async def delete_dictionary_entry(
return None return None
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:delete_dictionary_entry:Function] # #endregion delete_dictionary_entry
# [DEF:import_dictionary_entries:Function] # #region import_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,import]
# @PURPOSE: Import entries into a terminology dictionary from CSV/TSV content. # @BRIEF Import entries into a terminology dictionary from CSV/TSV content.
# @PRE: User has translate.dictionary.edit permission. # @PRE User has translate.dictionary.edit permission. Dictionary exists. Content is valid CSV/TSV.
# @POST: Entries are imported per conflict mode. # @POST Entries imported per conflict mode (overwrite/keep_existing/cancel). Import result returned.
# @SIDE_EFFECT Creates/updates DictionaryEntry records in DB.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.post("/dictionaries/{dictionary_id}/import") @router.post("/dictionaries/{dictionary_id}/import")
async def import_dictionary_entries( async def import_dictionary_entries(
dictionary_id: str, dictionary_id: str,
@@ -351,6 +375,6 @@ async def import_dictionary_entries(
return result return result
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:import_dictionary_entries:Function] # #endregion import_dictionary_entries
# [/DEF:TranslateDictionaryRoutesModule:Module] # #endregion TranslateDictionaryRoutesModule

34
backend/src/api/routes/translate/_helpers.py
View File

@@ -1,17 +1,13 @@
# [DEF:TranslateHelpersModule:Module] # #region TranslateHelpersModule [C:2] [TYPE Module] [SEMANTICS api,routes,translate,helpers]
# @COMPLEXITY: 2 # @BRIEF Shared helper functions for translate route handlers.
# @PURPOSE: Shared helper functions for translate route handlers. # @LAYER UI
# @LAYER: API
# @SEMANTICS: api, routes, translate, helpers
# @RELATION: DEPENDS_ON -> [models.translate]
from typing import Any, Dict, List from typing import Any, Dict, List
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
# [DEF:_run_to_response:Function] # #region _run_to_response [C:2] [TYPE Function] [SEMANTICS helpers,translate]
# @COMPLEXITY: 2 # @BRIEF Convert TranslationRun ORM to response dict.
# @PURPOSE: Convert TranslationRun ORM to response dict.
def _run_to_response(run: Any) -> dict: def _run_to_response(run: Any) -> dict:
return { return {
"id": run.id, "id": run.id,
@@ -34,12 +30,11 @@ def _run_to_response(run: Any) -> dict:
"created_by": run.created_by, "created_by": run.created_by,
"created_at": run.created_at.isoformat() if run.created_at else None, "created_at": run.created_at.isoformat() if run.created_at else None,
} }
# [/DEF:_run_to_response:Function] # #endregion _run_to_response
# [DEF:_dict_to_response:Function] # #region _dict_to_response [C:2] [TYPE Function] [SEMANTICS helpers,translate]
# @COMPLEXITY: 2 # @BRIEF Convert a TerminologyDictionary ORM model to a response dict with computed entry_count.
# @PURPOSE: Convert a TerminologyDictionary ORM model to a response dict with computed entry_count.
@staticmethod @staticmethod
def _dict_to_response(d: Any, entry_count: int = 0) -> dict: def _dict_to_response(d: Any, entry_count: int = 0) -> dict:
return { return {
@@ -54,15 +49,14 @@ def _dict_to_response(d: Any, entry_count: int = 0) -> dict:
"updated_at": d.updated_at, "updated_at": d.updated_at,
"entry_count": entry_count, "entry_count": entry_count,
} }
# [/DEF:_dict_to_response:Function] # #endregion _dict_to_response
# [DEF:_get_dictionary_entry_counts:Function] # #region _get_dictionary_entry_counts [C:2] [TYPE Function] [SEMANTICS helpers,translate]
# @COMPLEXITY: 2 # @BRIEF Get entry counts for a list of dictionary IDs.
# @PURPOSE: Get entry counts for a list of dictionary IDs.
def _get_dictionary_entry_counts(db: Session, dict_ids: List[str]) -> Dict[str, int]: def _get_dictionary_entry_counts(db: Session, dict_ids: List[str]) -> Dict[str, int]:
from sqlalchemy import func from sqlalchemy import func
from ...models.translate import DictionaryEntry from ....models.translate import DictionaryEntry
counts = ( counts = (
db.query(DictionaryEntry.dictionary_id, func.count(DictionaryEntry.id)) db.query(DictionaryEntry.dictionary_id, func.count(DictionaryEntry.id))
.filter(DictionaryEntry.dictionary_id.in_(dict_ids)) .filter(DictionaryEntry.dictionary_id.in_(dict_ids))
@@ -70,6 +64,6 @@ def _get_dictionary_entry_counts(db: Session, dict_ids: List[str]) -> Dict[str,
.all() .all()
) )
return {row[0]: row[1] for row in counts} return {row[0]: row[1] for row in counts}
# [/DEF:_get_dictionary_entry_counts:Function] # #endregion _get_dictionary_entry_counts
# [/DEF:TranslateHelpersModule:Module] # #endregion TranslateHelpersModule

112
backend/src/api/routes/translate/_job_routes.py
View File

@@ -1,8 +1,14 @@
# [DEF:TranslateJobRoutesModule:Module] # #region TranslateJobRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,jobs]
# @COMPLEXITY: 3 # @BRIEF Translation Job CRUD and datasource column routes.
# @PURPOSE: Translation Job CRUD and datasource column routes. # @LAYER UI
# @LAYER: API # @RELATION DEPENDS_ON -> [TranslateJobService]
# @SEMANTICS: api, routes, translate, jobs # @RELATION DEPENDS_ON -> [ConfigManager]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
# @PRE ConfigManager and DB session initialized. User authenticated with translate.job permissions.
# @POST Translation job CRUD completed or datasource columns fetched.
# @SIDE_EFFECT Reads/writes TranslationJob records to DB; fetches datasource metadata from Superset API.
from fastapi import APIRouter, Depends, HTTPException, status, Query from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
@@ -33,10 +39,9 @@ from ._router import router
# Translation Job CRUD # Translation Job CRUD
# ============================================================ # ============================================================
# [DEF:list_jobs:Function] # #region list_jobs [C:3] [TYPE Function] [SEMANTICS api,translate,jobs]
# @PURPOSE: List all translation jobs. # @BRIEF List all translation jobs with pagination and optional status filter.
# @PRE: User has translate.job.view permission. # @RELATION DEPENDS_ON -> [TranslateJobService]
# @POST: Returns list of translation jobs.
@router.get("/jobs", response_model=List[TranslateJobResponse]) @router.get("/jobs", response_model=List[TranslateJobResponse])
async def list_jobs( async def list_jobs(
page: int = Query(1, ge=1), page: int = Query(1, ge=1),
@@ -63,13 +68,12 @@ async def list_jobs(
return results return results
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:list_jobs:Function] # #endregion list_jobs
# [DEF:get_job:Function] # #region get_job [C:3] [TYPE Function] [SEMANTICS api,translate,jobs]
# @PURPOSE: Get a single translation job by ID. # @BRIEF Get a single translation job by ID.
# @PRE: User has translate.job.view permission. # @RELATION DEPENDS_ON -> [TranslateJobService]
# @POST: Returns the translation job.
@router.get("/jobs/{job_id}", response_model=TranslateJobResponse) @router.get("/jobs/{job_id}", response_model=TranslateJobResponse)
async def get_job( async def get_job(
job_id: str, job_id: str,
@@ -87,14 +91,15 @@ async def get_job(
return job_to_response(job, dict_ids) return job_to_response(job, dict_ids)
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_job:Function] # #endregion get_job
# [DEF:create_job:Function] # #region create_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]
# @PURPOSE: Create a new translation job. # @BRIEF Create a new translation job.
# @PRE: User has translate.job.create permission. # @PRE User has translate.job.create permission. Payload validated.
# @POST: Returns the created translation job. # @POST Translation job created and returned.
# @SIDE_EFFECT: Validates columns via SupersetClient; caches database_dialect. # @SIDE_EFFECT Creates TranslationJob record in DB; validates columns via SupersetClient; caches database_dialect.
# @RELATION DEPENDS_ON -> [TranslateJobService]
@router.post("/jobs", response_model=TranslateJobResponse, status_code=status.HTTP_201_CREATED) @router.post("/jobs", response_model=TranslateJobResponse, status_code=status.HTTP_201_CREATED)
async def create_job( async def create_job(
payload: TranslateJobCreate, payload: TranslateJobCreate,
@@ -112,14 +117,15 @@ async def create_job(
return job_to_response(job, dict_ids) return job_to_response(job, dict_ids)
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)) raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))
# [/DEF:create_job:Function] # #endregion create_job
# [DEF:update_job:Function] # #region update_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]
# @PURPOSE: Update an existing translation job. # @BRIEF Update an existing translation job.
# @PRE: User has translate.job.edit permission. # @PRE User has translate.job.edit permission. Job exists.
# @POST: Returns the updated translation job. # @POST Translation job updated and returned.
# @SIDE_EFFECT: Re-detects database_dialect if datasource changed. # @SIDE_EFFECT Updates TranslationJob record in DB; re-detects database_dialect if datasource changed.
# @RELATION DEPENDS_ON -> [TranslateJobService]
@router.put("/jobs/{job_id}", response_model=TranslateJobResponse) @router.put("/jobs/{job_id}", response_model=TranslateJobResponse)
async def update_job( async def update_job(
job_id: str, job_id: str,
@@ -138,13 +144,15 @@ async def update_job(
return job_to_response(job, dict_ids) return job_to_response(job, dict_ids)
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:update_job:Function] # #endregion update_job
# [DEF:delete_job:Function] # #region delete_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]
# @PURPOSE: Delete a translation job. # @BRIEF Delete a translation job.
# @PRE: User has translate.job.delete permission. # @PRE User has translate.job.delete permission. Job exists.
# @POST: Job is deleted. # @POST Translation job is deleted from DB.
# @SIDE_EFFECT Deletes TranslationJob and associated records from DB.
# @RELATION DEPENDS_ON -> [TranslateJobService]
@router.delete("/jobs/{job_id}", status_code=status.HTTP_204_NO_CONTENT) @router.delete("/jobs/{job_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_job( async def delete_job(
job_id: str, job_id: str,
@@ -160,13 +168,15 @@ async def delete_job(
service.delete_job(job_id) service.delete_job(job_id)
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:delete_job:Function] # #endregion delete_job
# [DEF:duplicate_job:Function] # #region duplicate_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]
# @PURPOSE: Duplicate a translation job. # @BRIEF Duplicate a translation job.
# @PRE: User has translate.job.create permission. # @PRE User has translate.job.create permission. Source job exists.
# @POST: Returns the duplicated job with status DRAFT. # @POST Duplicated job created with status DRAFT and returned.
# @SIDE_EFFECT Creates a new TranslationJob record in DB as a copy of the source.
# @RELATION DEPENDS_ON -> [TranslateJobService]
@router.post("/jobs/{job_id}/duplicate", response_model=DuplicateJobResponse, status_code=status.HTTP_201_CREATED) @router.post("/jobs/{job_id}/duplicate", response_model=DuplicateJobResponse, status_code=status.HTTP_201_CREATED)
async def duplicate_job( async def duplicate_job(
job_id: str, job_id: str,
@@ -187,17 +197,16 @@ async def duplicate_job(
) )
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:duplicate_job:Function] # #endregion duplicate_job
# [DEF:get_datasource_columns:Function] # #region list_datasources [C:4] [TYPE Function] [SEMANTICS api,translate,datasources]
# @PURPOSE: Get column metadata and database dialect for a Superset datasource. # @BRIEF List all Superset datasets available for translation jobs.
# @PRE: User has translate.job.view permission. # @PRE User has translate.job.view permission.
# @POST: Returns column list with metadata and database_dialect. # @POST Returns list of datasets with metadata.
# @SIDE_EFFECT: Queries Superset API for dataset detail and database info. # @SIDE_EFFECT Queries Superset API for available datasets.
# @RATIONALE: Dialect detection from Superset connection cached on TranslationJob at save time. # @RELATION DEPENDS_ON -> [TranslateJobService]
# @REJECTED: Hardcoding dialect list per database — would drift from actual Superset config. # @RELATION DEPENDS_ON -> [SupersetClient]
@router.get("/datasources") @router.get("/datasources")
async def list_datasources( async def list_datasources(
env_id: str = Query(..., description="Superset environment ID"), env_id: str = Query(..., description="Superset environment ID"),
@@ -216,7 +225,16 @@ async def list_datasources(
except Exception as e: except Exception as e:
logger.error(f"[translate_routes][list_datasources] Error: {e}") logger.error(f"[translate_routes][list_datasources] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e)) raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e))
# #endregion list_datasources
# #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS api,translate,datasources]
# @BRIEF Get column metadata and database dialect for a Superset datasource.
# @PRE User has translate.job.view permission. Datasource exists in Superset.
# @POST Returns column list with metadata and database_dialect.
# @SIDE_EFFECT Queries Superset API for dataset detail and database info.
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [ConfigManager]
@router.get("/datasources/{datasource_id}/columns", response_model=DatasourceColumnsResponse) @router.get("/datasources/{datasource_id}/columns", response_model=DatasourceColumnsResponse)
async def get_datasource_columns( async def get_datasource_columns(
datasource_id: int, datasource_id: int,
@@ -245,6 +263,6 @@ async def get_datasource_columns(
status_code=status.HTTP_502_BAD_GATEWAY, status_code=status.HTTP_502_BAD_GATEWAY,
detail=f"Failed to fetch datasource columns from Superset: {e}", detail=f"Failed to fetch datasource columns from Superset: {e}",
) )
# [/DEF:get_datasource_columns:Function] # #endregion get_datasource_columns
# [/DEF:TranslateJobRoutesModule:Module] # #endregion TranslateJobRoutesModule

31
backend/src/api/routes/translate/_metrics_routes.py
View File

@@ -1,8 +1,9 @@
# [DEF:TranslateMetricsRoutesModule:Module] # #region TranslateMetricsRoutesModule [C:3] [TYPE Module] [SEMANTICS api,routes,translate,metrics]
# @COMPLEXITY: 2 # @BRIEF Translation Metrics endpoints providing aggregated stats on runs and corrections.
# @PURPOSE: Translation Metrics endpoints. # @LAYER UI
# @LAYER: API # @RELATION DEPENDS_ON -> [TranslationMetrics]
# @SEMANTICS: api, routes, translate, metrics # @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
from fastapi import APIRouter, Depends, HTTPException, status, Query from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
@@ -21,10 +22,9 @@ from ._router import router
# Metrics # Metrics
# ============================================================ # ============================================================
# [DEF:get_metrics:Function] # #region get_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics]
# @PURPOSE: Get translation metrics, optionally filtered by job. # @BRIEF Get translation metrics across all jobs, optionally filtered by job.
# @PRE: User has translate.metrics.view permission. # @RELATION DEPENDS_ON -> [TranslationMetrics]
# @POST: Returns metrics data.
@router.get("/metrics") @router.get("/metrics")
async def get_metrics( async def get_metrics(
job_id: Optional[str] = Query(None), job_id: Optional[str] = Query(None),
@@ -41,13 +41,12 @@ async def get_metrics(
return metrics_svc.get_all_metrics() return metrics_svc.get_all_metrics()
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_metrics:Function] # #endregion get_metrics
# [DEF:get_job_metrics:Function] # #region get_job_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics]
# @PURPOSE: Get aggregated metrics for a specific job. # @BRIEF Get aggregated metrics for a specific translation job.
# @PRE: User has translate.metrics.view permission. # @RELATION DEPENDS_ON -> [TranslationMetrics]
# @POST: Returns metrics dict.
@router.get("/jobs/{job_id}/metrics") @router.get("/jobs/{job_id}/metrics")
async def get_job_metrics( async def get_job_metrics(
job_id: str, job_id: str,
@@ -62,6 +61,6 @@ async def get_job_metrics(
return metrics_svc.get_job_metrics(job_id) return metrics_svc.get_job_metrics(job_id)
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_job_metrics:Function] # #endregion get_job_metrics
# [/DEF:TranslateMetricsRoutesModule:Module] # #endregion TranslateMetricsRoutesModule

81
backend/src/api/routes/translate/_preview_routes.py
View File

@@ -1,8 +1,13 @@
# [DEF:TranslatePreviewRoutesModule:Module] # #region TranslatePreviewRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,preview]
# @COMPLEXITY: 3 # @BRIEF Translation Preview session management routes.
# @PURPOSE: Translation Preview session management routes. # @LAYER UI
# @LAYER: API # @RELATION DEPENDS_ON -> [TranslationPreview]
# @SEMANTICS: api, routes, translate, preview # @RELATION DEPENDS_ON -> [ConfigManager]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
# @PRE ConfigManager and DB session initialized. User authenticated with translate permissions.
# @POST Preview sessions created, rows reviewed/accepted, records queried.
# @SIDE_EFFECT Creates/updates preview sessions and rows in DB; fetches sample data from Superset; calls LLM provider.
from fastapi import APIRouter, Depends, HTTPException, status, Query from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
@@ -28,11 +33,13 @@ from ._router import router
# Preview # Preview
# ============================================================ # ============================================================
# [DEF:preview_translation:Function] # #region preview_translation [C:4] [TYPE Function] [SEMANTICS api,translate,preview]
# @PURPOSE: Preview a translation before applying it. # @BRIEF Preview a translation before applying it.
# @PRE: User has translate.job.execute permission. # @PRE User has translate.job.execute permission. Job exists.
# @POST: Returns a preview session with records and cost estimation. # @POST Preview session created with sample rows and cost estimation.
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows. # @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates PreviewSession and PreviewRecord rows in DB.
# @RELATION DEPENDS_ON -> [TranslationPreview]
# @RELATION DEPENDS_ON -> [SupersetClient]
@router.post("/jobs/{job_id}/preview", status_code=status.HTTP_201_CREATED) @router.post("/jobs/{job_id}/preview", status_code=status.HTTP_201_CREATED)
async def preview_translation( async def preview_translation(
job_id: str, job_id: str,
@@ -60,13 +67,15 @@ async def preview_translation(
except Exception as e: except Exception as e:
logger.error(f"[translate_routes][preview_translation] Error: {e}") logger.error(f"[translate_routes][preview_translation] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Preview failed: {e}") raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Preview failed: {e}")
# [/DEF:preview_translation:Function] # #endregion preview_translation
# [DEF:update_preview_row:Function] # #region update_preview_row [C:4] [TYPE Function] [SEMANTICS api,translate,preview]
# @PURPOSE: Approve, edit, or reject a preview row. # @BRIEF Approve, edit, or reject a preview row.
# @PRE: User has translate.job.execute permission. # @PRE User has translate.job.execute permission. Preview row exists.
# @POST: Preview row status is updated. # @POST Preview row status updated to APPROVED/REJECTED/EDITED.
# @SIDE_EFFECT Updates PreviewRecord status in DB.
# @RELATION DEPENDS_ON -> [TranslationPreview]
@router.put("/jobs/{job_id}/preview/rows/{row_key}") @router.put("/jobs/{job_id}/preview/rows/{row_key}")
async def update_preview_row( async def update_preview_row(
job_id: str, job_id: str,
@@ -91,13 +100,15 @@ async def update_preview_row(
return result return result
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:update_preview_row:Function] # #endregion update_preview_row
# [DEF:accept_preview_session:Function] # #region accept_preview_session [C:4] [TYPE Function] [SEMANTICS api,translate,preview]
# @PURPOSE: Accept a preview session, marking it as the quality gate for full execution. # @BRIEF Accept a preview session, marking it as the quality gate for full execution.
# @PRE: User has translate.job.execute permission. Job has an ACTIVE preview session. # @PRE User has translate.job.execute permission. Job has an ACTIVE preview session.
# @POST: Preview session is marked as APPLIED; full execution can proceed. # @POST Preview session status set to APPLIED. Full execution can proceed.
# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB.
# @RELATION DEPENDS_ON -> [TranslationPreview]
@router.post("/jobs/{job_id}/preview/accept") @router.post("/jobs/{job_id}/preview/accept")
async def accept_preview_session( async def accept_preview_session(
job_id: str, job_id: str,
@@ -114,13 +125,15 @@ async def accept_preview_session(
return result return result
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:accept_preview_session:Function] # #endregion accept_preview_session
# [DEF:apply_preview:Function] # #region apply_preview [C:4] [TYPE Function] [SEMANTICS api,translate,preview]
# @PURPOSE: Apply a preview session (alias for accept when accepting at session level). # @BRIEF Apply a preview session (alias for accept when accepting at session level).
# @PRE: User has translate.job.execute permission. # @PRE User has translate.job.execute permission. Session exists.
# @POST: Preview is applied. # @POST Preview session applied and marked as quality gate.
# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB.
# @RELATION DEPENDS_ON -> [TranslationPreview]
@router.post("/preview/{session_id}/apply") @router.post("/preview/{session_id}/apply")
async def apply_preview( async def apply_preview(
session_id: str, session_id: str,
@@ -132,7 +145,7 @@ async def apply_preview(
"""Apply a preview session by session ID.""" """Apply a preview session by session ID."""
logger.info(f"[translate_routes][apply_preview] Session: {session_id}, User: {current_user.username}") logger.info(f"[translate_routes][apply_preview] Session: {session_id}, User: {current_user.username}")
# Find job_id from session # Find job_id from session
from ...models.translate import TranslationPreviewSession from ....models.translate import TranslationPreviewSession
session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first() session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first()
if not session: if not session:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found") raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found")
@@ -142,17 +155,17 @@ async def apply_preview(
return result return result
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:apply_preview:Function] # #endregion apply_preview
# ============================================================ # ============================================================
# Preview Records # Preview Records
# ============================================================ # ============================================================
# [DEF:get_preview_records:Function] # #region get_preview_records [C:3] [TYPE Function] [SEMANTICS api,translate,preview]
# @PURPOSE: Get records for a preview session. # @BRIEF Get records for a preview session.
# @PRE: User has translate.job.view permission. # @RELATION DEPENDS_ON -> [TranslationPreviewSession]
# @POST: Returns list of preview records. # @RELATION DEPENDS_ON -> [TranslationPreviewRecord]
@router.get("/preview/{session_id}/records", response_model=List[PreviewRow]) @router.get("/preview/{session_id}/records", response_model=List[PreviewRow])
async def get_preview_records( async def get_preview_records(
session_id: str, session_id: str,
@@ -163,7 +176,7 @@ async def get_preview_records(
"""Get records for a preview session.""" """Get records for a preview session."""
logger.info(f"[translate_routes][get_preview_records] Session: {session_id}, User: {current_user.username}") logger.info(f"[translate_routes][get_preview_records] Session: {session_id}, User: {current_user.username}")
try: try:
from ...models.translate import TranslationPreviewSession, TranslationPreviewRecord from ....models.translate import TranslationPreviewSession, TranslationPreviewRecord
session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first() session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first()
if not session: if not session:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found") raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found")
@@ -190,6 +203,6 @@ async def get_preview_records(
except Exception as e: except Exception as e:
logger.error(f"[translate_routes][get_preview_records] Error: {e}") logger.error(f"[translate_routes][get_preview_records] Error: {e}")
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:get_preview_records:Function] # #endregion get_preview_records
# [/DEF:TranslatePreviewRoutesModule:Module] # #endregion TranslatePreviewRoutesModule

14
backend/src/api/routes/translate/_router.py
View File

@@ -1,14 +1,10 @@
# [DEF:TranslateRouterModule:Module] # #region TranslateRouterModule [C:1] [TYPE Module] [SEMANTICS api,routes,translate,router]
# @COMPLEXITY: 1 # @LAYER UI
# @PURPOSE: APIRouter instance for translate routes.
# @LAYER: API
# @SEMANTICS: api, routes, translate, router
from fastapi import APIRouter from fastapi import APIRouter
# [DEF:translate_router:Global] # #region translate_router [C:1] [TYPE Global]
# @PURPOSE: APIRouter instance for all translate sub-routes.
router = APIRouter(prefix="/api/translate", tags=["translate"]) router = APIRouter(prefix="/api/translate", tags=["translate"])
# [/DEF:translate_router:Global] # #endregion translate_router
# [/DEF:TranslateRouterModule:Module] # #endregion TranslateRouterModule

51
backend/src/api/routes/translate/_run_list_routes.py
View File

@@ -1,8 +1,10 @@
# [DEF:TranslateRunListRoutesModule:Module] # #region TranslateRunListRoutesModule [C:3] [TYPE Module] [SEMANTICS api,routes,translate,runs,list,detail]
# @COMPLEXITY: 3 # @BRIEF Translation Run listing, detail, and CSV download routes (cross-job).
# @PURPOSE: Translation Run listing, detail, and CSV download routes (cross-job). # @LAYER UI
# @LAYER: API # @RELATION DEPENDS_ON -> [TranslationOrchestrator]
# @SEMANTICS: api, routes, translate, runs, list, detail # @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
from fastapi import APIRouter, Depends, HTTPException, status, Query from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
@@ -23,14 +25,13 @@ from ._router import router
# List Runs (cross-job) # List Runs (cross-job)
# ============================================================ # ============================================================
# [DEF:list_runs:Function] # #region list_runs [C:3] [TYPE Function] [SEMANTICS api,translate,runs,list]
# @PURPOSE: List all runs with cross-job filtering and pagination. # @BRIEF List all runs with cross-job filtering and pagination.
# @PRE: User has translate.history.view permission. # @RELATION DEPENDS_ON -> [TranslationRun]
# @POST: Returns paginated list of runs.
@router.get("/runs") @router.get("/runs")
async def list_runs( async def list_runs(
job_id: Optional[str] = Query(None), job_id: Optional[str] = Query(None),
run_status: Optional[str] = Query(None), run_status: Optional[str] = Query(None, alias="status"),
trigger_type: Optional[str] = Query(None), trigger_type: Optional[str] = Query(None),
created_by: Optional[str] = Query(None), created_by: Optional[str] = Query(None),
date_from: Optional[str] = Query(None), date_from: Optional[str] = Query(None),
@@ -44,7 +45,7 @@ async def list_runs(
"""List all runs with cross-job filtering and pagination.""" """List all runs with cross-job filtering and pagination."""
logger.info(f"[translate_routes][list_runs] User: {current_user.username}") logger.info(f"[translate_routes][list_runs] User: {current_user.username}")
try: try:
from ...models.translate import TranslationRun from ....models.translate import TranslationRun
query = db.query(TranslationRun) query = db.query(TranslationRun)
if job_id: if job_id:
@@ -104,17 +105,18 @@ async def list_runs(
except Exception as e: except Exception as e:
logger.error(f"[translate_routes][list_runs] Error: {e}") logger.error(f"[translate_routes][list_runs] Error: {e}")
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:list_runs:Function] # #endregion list_runs
# ============================================================ # ============================================================
# Run Detail # Run Detail
# ============================================================ # ============================================================
# [DEF:get_run_detail:Function] # #region get_run_detail [C:3] [TYPE Function] [SEMANTICS api,translate,runs,detail]
# @PURPOSE: Get detailed run info with config_snapshot, records, events. # @BRIEF Get detailed run info with config_snapshot, records, events.
# @PRE: User has translate.history.view permission. # @RELATION DEPENDS_ON -> [TranslationOrchestrator]
# @POST: Returns run detail with records and events. # @RELATION DEPENDS_ON -> [TranslationEventLog]
# @RELATION DEPENDS_ON -> [TranslationRun]
@router.get("/runs/{run_id}/detail") @router.get("/runs/{run_id}/detail")
async def get_run_detail( async def get_run_detail(
run_id: str, run_id: str,
@@ -133,7 +135,7 @@ async def get_run_detail(
event_summary = TranslationEventLog(db).get_run_event_summary(run_id) event_summary = TranslationEventLog(db).get_run_event_summary(run_id)
# Get config snapshot from run # Get config snapshot from run
from ...models.translate import TranslationRun from ....models.translate import TranslationRun
run = db.query(TranslationRun).filter(TranslationRun.id == run_id).first() run = db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
config_snapshot = run.config_snapshot if run else None config_snapshot = run.config_snapshot if run else None
@@ -147,17 +149,16 @@ async def get_run_detail(
} }
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_run_detail:Function] # #endregion get_run_detail
# ============================================================ # ============================================================
# Download Skipped CSV # Download Skipped CSV
# ============================================================ # ============================================================
# [DEF:download_skipped_csv:Function] # #region download_skipped_csv [C:3] [TYPE Function] [SEMANTICS api,translate,runs,csv]
# @PURPOSE: Download a CSV of skipped translation records from a run. # @BRIEF Download a CSV of skipped translation records from a run.
# @PRE: User has translate.job.view permission. # @RELATION DEPENDS_ON -> [TranslationRecord]
# @POST: Returns CSV file of skipped records.
@router.get("/runs/{run_id}/skipped.csv") @router.get("/runs/{run_id}/skipped.csv")
async def download_skipped_csv( async def download_skipped_csv(
run_id: str, run_id: str,
@@ -168,7 +169,7 @@ async def download_skipped_csv(
"""Download skipped translation records as CSV.""" """Download skipped translation records as CSV."""
logger.info(f"[translate_routes][download_skipped_csv] Run: {run_id}, User: {current_user.username}") logger.info(f"[translate_routes][download_skipped_csv] Run: {run_id}, User: {current_user.username}")
try: try:
from ...models.translate import TranslationRecord from ....models.translate import TranslationRecord
from fastapi.responses import StreamingResponse from fastapi.responses import StreamingResponse
import csv import csv
import io import io
@@ -201,6 +202,6 @@ async def download_skipped_csv(
except Exception as e: except Exception as e:
logger.error(f"[translate_routes][download_skipped_csv] Error: {e}") logger.error(f"[translate_routes][download_skipped_csv] Error: {e}")
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:download_skipped_csv:Function] # #endregion download_skipped_csv
# [/DEF:TranslateRunListRoutesModule:Module] # #endregion TranslateRunListRoutesModule

139
backend/src/api/routes/translate/_run_routes.py
View File

@@ -1,20 +1,26 @@
# [DEF:TranslateRunRoutesModule:Module] # #region TranslateRunRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,runs]
# @COMPLEXITY: 3 # @BRIEF Translation Run execution, history, status, records and batches routes.
# @PURPOSE: Translation Run execution, history, status, records and batches routes. # @LAYER UI
# @LAYER: API # @RELATION DEPENDS_ON -> [TranslationOrchestrator]
# @SEMANTICS: api, routes, translate, runs # @RELATION DEPENDS_ON -> [ConfigManager]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
# @PRE ConfigManager and DB session initialized. User authenticated with translate permissions.
# @POST Translation run executed, manipulated, or queried. Results returned to caller.
# @SIDE_EFFECT Creates/updates TranslationRun records in DB; spawns background threads for execution; queries Superset API.
from fastapi import APIRouter, Depends, HTTPException, status, Query from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from ....core.database import get_db from ....core.database import get_db, SessionLocal
from ....core.logger import logger, belief_scope from ....core.logger import logger, belief_scope
from ....schemas.auth import User from ....schemas.auth import User
from ....dependencies import get_current_user, has_permission, get_config_manager from ....dependencies import get_current_user, has_permission, get_config_manager
from ....core.config_manager import ConfigManager from ....core.config_manager import ConfigManager
from ....plugins.translate.orchestrator import TranslationOrchestrator from ....plugins.translate.orchestrator import TranslationOrchestrator
from ....plugins.translate.events import TranslationEventLog from ....plugins.translate.events import TranslationEventLog
from ....models.translate import TranslationRun
from ._router import router from ._router import router
from ._helpers import _run_to_response from ._helpers import _run_to_response
@@ -24,43 +30,64 @@ from ._helpers import _run_to_response
# Translation Run / Execute # Translation Run / Execute
# ============================================================ # ============================================================
# [DEF:run_translation:Function] # #region run_translation [C:4] [TYPE Function] [SEMANTICS api,translate,runs]
# @PURPOSE: Execute a translation job (trigger a run). # @BRIEF Execute a translation job (trigger a run).
# @PRE: User has translate.job.execute permission. # @PRE User has translate.job.execute permission. Job exists and preview is accepted.
# @POST: Returns the created translation run. # @POST Translation run created and started in background. Run object returned.
# @SIDE_EFFECT Creates TranslationRun record in DB; spawns background thread for translation execution.
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
@router.post("/jobs/{job_id}/run", status_code=status.HTTP_201_CREATED) @router.post("/jobs/{job_id}/run", status_code=status.HTTP_201_CREATED)
async def run_translation( async def run_translation(
job_id: str, job_id: str,
full_translation: bool = Query(False, description="If True, fetch ALL rows from Superset dataset instead of preview-only rows"),
current_user: User = Depends(get_current_user), current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")), _ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db), db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager), config_manager: ConfigManager = Depends(get_config_manager),
): ):
"""Execute a translation job (trigger a run).""" """Execute a translation job (trigger a run).
logger.info(f"[translate_routes][run_translation] Job: {job_id}, User: {current_user.username}")
By default runs translation on preview-approved rows only.
Set full_translation=true to fetch ALL rows from the Superset source dataset.
"""
logger.info(f"[translate_routes][run_translation] Job: {job_id}, User: {current_user.username}, full={full_translation}")
try: try:
orch = TranslationOrchestrator(db, config_manager, current_user.username) orch = TranslationOrchestrator(db, config_manager, current_user.username)
run = orch.start_run(job_id=job_id, is_scheduled=False) run = orch.start_run(job_id=job_id, is_scheduled=False)
# Execute asynchronously in background # Execute asynchronously in background with a FRESH session.
# The request-scoped session from Depends(get_db) is closed after the handler returns,
# so the background thread must create its own session to avoid "This transaction is closed".
import threading import threading
threading.Thread(
target=orch.execute_run, def _execute_background():
args=(run,), thread_db = SessionLocal()
daemon=True, try:
).start() thread_orch = TranslationOrchestrator(thread_db, config_manager, current_user.username)
# Reload run from DB with the new session (the passed run object is detached)
thread_run = thread_db.query(TranslationRun).filter(TranslationRun.id == run.id).first()
if thread_run:
thread_orch.execute_run(thread_run, full_translation=full_translation)
except Exception as e:
logger.error(f"[translate_routes][run_translation] Background execution error: {e}")
finally:
thread_db.close()
threading.Thread(target=_execute_background, daemon=True).start()
return _run_to_response(run) return _run_to_response(run)
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
except Exception as e: except Exception as e:
logger.error(f"[translate_routes][run_translation] Error: {e}") logger.error(f"[translate_routes][run_translation] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Run failed: {e}") raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Run failed: {e}")
# [/DEF:run_translation:Function] # #endregion run_translation
# [DEF:retry_run:Function] # #region retry_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs]
# @PURPOSE: Retry failed batches in a translation run. # @BRIEF Retry failed batches in a translation run.
# @PRE: User has translate.job.execute permission. # @PRE User has translate.job.execute permission. Run exists and has failed batches.
# @POST: Returns the updated translation run. # @POST Failed batches re-executed. Updated run returned.
# @SIDE_EFFECT Updates TranslationRun and TranslationBatch records in DB.
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
@router.post("/runs/{run_id}/retry") @router.post("/runs/{run_id}/retry")
async def retry_run( async def retry_run(
run_id: str, run_id: str,
@@ -80,13 +107,15 @@ async def retry_run(
except Exception as e: except Exception as e:
logger.error(f"[translate_routes][retry_run] Error: {e}") logger.error(f"[translate_routes][retry_run] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry failed: {e}") raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry failed: {e}")
# [/DEF:retry_run:Function] # #endregion retry_run
# [DEF:retry_insert:Function] # #region retry_insert [C:4] [TYPE Function] [SEMANTICS api,translate,runs]
# @PURPOSE: Retry the SQL insert phase for a completed run. # @BRIEF Retry the SQL insert phase for a completed run.
# @PRE: User has translate.job.execute permission. # @PRE User has translate.job.execute permission. Run completed with insert_status FAILED.
# @POST: Returns the updated run. # @POST SQL insert phase re-executed. Updated run returned.
# @SIDE_EFFECT Re-executes SQL insert into Superset; updates TranslationRun insert_status.
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
@router.post("/runs/{run_id}/retry-insert") @router.post("/runs/{run_id}/retry-insert")
async def retry_insert( async def retry_insert(
run_id: str, run_id: str,
@@ -106,13 +135,15 @@ async def retry_insert(
except Exception as e: except Exception as e:
logger.error(f"[translate_routes][retry_insert] Error: {e}") logger.error(f"[translate_routes][retry_insert] Error: {e}")
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry insert failed: {e}") raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry insert failed: {e}")
# [/DEF:retry_insert:Function] # #endregion retry_insert
# [DEF:cancel_run:Function] # #region cancel_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs]
# @PURPOSE: Cancel a running translation. # @BRIEF Cancel a running translation.
# @PRE: User has translate.job.execute permission. # @PRE User has translate.job.execute permission. Run is in RUNNING state.
# @POST: Run is cancelled. # @POST Run is cancelled. Updated run returned.
# @SIDE_EFFECT Updates TranslationRun status to CANCELLED in DB.
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
@router.post("/runs/{run_id}/cancel") @router.post("/runs/{run_id}/cancel")
async def cancel_run( async def cancel_run(
run_id: str, run_id: str,
@@ -129,13 +160,12 @@ async def cancel_run(
return _run_to_response(run) return _run_to_response(run)
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:cancel_run:Function] # #endregion cancel_run
# [DEF:get_run_history:Function] # #region get_run_history [C:3] [TYPE Function] [SEMANTICS api,translate,runs]
# @PURPOSE: Get run history for a translation job. # @BRIEF Get run history for a translation job.
# @PRE: User has translate.history.view permission. # @RELATION DEPENDS_ON -> [TranslationOrchestrator]
# @POST: Returns list of runs.
@router.get("/jobs/{job_id}/runs") @router.get("/jobs/{job_id}/runs")
async def get_run_history( async def get_run_history(
job_id: str, job_id: str,
@@ -154,13 +184,12 @@ async def get_run_history(
return {"items": runs, "total": total, "page": page, "page_size": page_size} return {"items": runs, "total": total, "page": page, "page_size": page_size}
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_run_history:Function] # #endregion get_run_history
# [DEF:get_run_status:Function] # #region get_run_status [C:3] [TYPE Function] [SEMANTICS api,translate,runs]
# @PURPOSE: Get status and statistics for a translation run. # @BRIEF Get status and statistics for a translation run.
# @PRE: User has translate.history.view permission. # @RELATION DEPENDS_ON -> [TranslationOrchestrator]
# @POST: Returns run details with statistics.
@router.get("/runs/{run_id}") @router.get("/runs/{run_id}")
async def get_run_status( async def get_run_status(
run_id: str, run_id: str,
@@ -176,13 +205,12 @@ async def get_run_status(
return orch.get_run_status(run_id) return orch.get_run_status(run_id)
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_run_status:Function] # #endregion get_run_status
# [DEF:get_run_records:Function] # #region get_run_records [C:3] [TYPE Function] [SEMANTICS api,translate,runs]
# @PURPOSE: Get paginated records for a translation run. # @BRIEF Get paginated records for a translation run.
# @PRE: User has translate.history.view permission. # @RELATION DEPENDS_ON -> [TranslationOrchestrator]
# @POST: Returns paginated records.
@router.get("/runs/{run_id}/records") @router.get("/runs/{run_id}/records")
async def get_run_records( async def get_run_records(
run_id: str, run_id: str,
@@ -201,17 +229,16 @@ async def get_run_records(
return orch.get_run_records(run_id, page=page, page_size=page_size, status_filter=status) return orch.get_run_records(run_id, page=page, page_size=page_size, status_filter=status)
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_run_records:Function] # #endregion get_run_records
# ============================================================ # ============================================================
# Batches # Batches
# ============================================================ # ============================================================
# [DEF:get_batches:Function] # #region get_batches [C:3] [TYPE Function] [SEMANTICS api,translate,runs]
# @PURPOSE: Get batches for a translation run. # @BRIEF Get batches for a translation run.
# @PRE: User has translate.job.view permission. # @RELATION DEPENDS_ON -> [TranslationBatch]
# @POST: Returns list of batches.
@router.get("/runs/{run_id}/batches") @router.get("/runs/{run_id}/batches")
async def get_batches( async def get_batches(
run_id: str, run_id: str,
@@ -222,7 +249,7 @@ async def get_batches(
"""Get batches for a translation run.""" """Get batches for a translation run."""
logger.info(f"[translate_routes][get_batches] Run: {run_id}, User: {current_user.username}") logger.info(f"[translate_routes][get_batches] Run: {run_id}, User: {current_user.username}")
try: try:
from ...models.translate import TranslationBatch from ....models.translate import TranslationBatch
batches = ( batches = (
db.query(TranslationBatch) db.query(TranslationBatch)
.filter(TranslationBatch.run_id == run_id) .filter(TranslationBatch.run_id == run_id)
@@ -247,6 +274,6 @@ async def get_batches(
except Exception as e: except Exception as e:
logger.error(f"[translate_routes][get_batches] Error: {e}") logger.error(f"[translate_routes][get_batches] Error: {e}")
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:get_batches:Function] # #endregion get_batches
# [/DEF:TranslateRunRoutesModule:Module] # #endregion TranslateRunRoutesModule

93
backend/src/api/routes/translate/_schedule_routes.py
View File

@@ -1,8 +1,13 @@
# [DEF:TranslateScheduleRoutesModule:Module] # #region TranslateScheduleRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,schedule]
# @COMPLEXITY: 3 # @BRIEF Translation Schedule management routes.
# @PURPOSE: Translation Schedule management routes. # @LAYER UI
# @LAYER: API # @RELATION DEPENDS_ON -> [TranslationScheduler]
# @SEMANTICS: api, routes, translate, schedule # @RELATION DEPENDS_ON -> [ConfigManager]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
# @PRE ConfigManager and DB session initialized. User authenticated with translate.schedule permissions.
# @POST Schedule CRUD operations completed. APScheduler jobs registered/unregistered.
# @SIDE_EFFECT Creates/updates/deletes TranslationSchedule records in DB; registers/removes jobs in APScheduler.
from fastapi import APIRouter, Depends, HTTPException, status, Query from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
@@ -23,10 +28,9 @@ from ._router import router
# Schedule # Schedule
# ============================================================ # ============================================================
# [DEF:get_schedule:Function] # #region get_schedule [C:3] [TYPE Function] [SEMANTICS api,translate,schedule]
# @PURPOSE: Get the schedule for a translation job. # @BRIEF Get the schedule configuration for a translation job.
# @PRE: User has translate.schedule.view permission. # @RELATION DEPENDS_ON -> [TranslationScheduler]
# @POST: Returns the schedule configuration.
@router.get("/jobs/{job_id}/schedule") @router.get("/jobs/{job_id}/schedule")
async def get_schedule( async def get_schedule(
job_id: str, job_id: str,
@@ -54,13 +58,15 @@ async def get_schedule(
} }
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_schedule:Function] # #endregion get_schedule
# [DEF:set_schedule:Function] # #region set_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]
# @PURPOSE: Set or update the schedule for a translation job. # @BRIEF Set or update the schedule for a translation job.
# @PRE: User has translate.schedule.manage permission. # @PRE User has translate.schedule.manage permission. Job exists.
# @POST: Schedule is created or updated. # @POST Schedule created or updated. APScheduler job registered.
# @SIDE_EFFECT Creates/updates TranslationSchedule record in DB; registers job in APScheduler.
# @RELATION DEPENDS_ON -> [TranslationScheduler]
@router.put("/jobs/{job_id}/schedule") @router.put("/jobs/{job_id}/schedule")
async def set_schedule( async def set_schedule(
job_id: str, job_id: str,
@@ -75,7 +81,7 @@ async def set_schedule(
try: try:
scheduler = TranslationScheduler(db, config_manager, current_user.username) scheduler = TranslationScheduler(db, config_manager, current_user.username)
# Check if schedule already exists # Check if schedule already exists
from ...models.translate import TranslationSchedule from ....models.translate import TranslationSchedule
existing = db.query(TranslationSchedule).filter( existing = db.query(TranslationSchedule).filter(
TranslationSchedule.job_id == job_id TranslationSchedule.job_id == job_id
).first() ).first()
@@ -94,7 +100,7 @@ async def set_schedule(
is_active=payload.is_active, is_active=payload.is_active,
) )
# Register with APScheduler via SchedulerService # Register with APScheduler via SchedulerService
from ...dependencies import get_scheduler_service from ....dependencies import get_scheduler_service
sched_svc = get_scheduler_service() sched_svc = get_scheduler_service()
sched_svc.add_translation_job( sched_svc.add_translation_job(
schedule_id=schedule.id, schedule_id=schedule.id,
@@ -115,13 +121,15 @@ async def set_schedule(
} }
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:set_schedule:Function] # #endregion set_schedule
# [DEF:enable_schedule:Function] # #region enable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]
# @PURPOSE: Enable a schedule for a translation job. # @BRIEF Enable a schedule for a translation job.
# @PRE: User has translate.schedule.manage permission. # @PRE User has translate.schedule.manage permission. Schedule exists.
# @POST: Schedule is enabled. # @POST Schedule is enabled. APScheduler job registered.
# @SIDE_EFFECT Updates TranslationSchedule is_active to True in DB; registers job in APScheduler.
# @RELATION DEPENDS_ON -> [TranslationScheduler]
@router.post("/jobs/{job_id}/schedule/enable") @router.post("/jobs/{job_id}/schedule/enable")
async def enable_schedule( async def enable_schedule(
job_id: str, job_id: str,
@@ -135,7 +143,7 @@ async def enable_schedule(
try: try:
scheduler = TranslationScheduler(db, config_manager, current_user.username) scheduler = TranslationScheduler(db, config_manager, current_user.username)
schedule = scheduler.set_schedule_active(job_id, is_active=True) schedule = scheduler.set_schedule_active(job_id, is_active=True)
from ...dependencies import get_scheduler_service from ....dependencies import get_scheduler_service
sched_svc = get_scheduler_service() sched_svc = get_scheduler_service()
sched_svc.add_translation_job( sched_svc.add_translation_job(
schedule_id=schedule.id, schedule_id=schedule.id,
@@ -146,13 +154,15 @@ async def enable_schedule(
return {"status": "enabled", "job_id": job_id} return {"status": "enabled", "job_id": job_id}
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:enable_schedule:Function] # #endregion enable_schedule
# [DEF:disable_schedule:Function] # #region disable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]
# @PURPOSE: Disable a schedule for a translation job. # @BRIEF Disable a schedule for a translation job.
# @PRE: User has translate.schedule.manage permission. # @PRE User has translate.schedule.manage permission. Schedule exists.
# @POST: Schedule is disabled. # @POST Schedule is disabled. APScheduler job removed.
# @SIDE_EFFECT Updates TranslationSchedule is_active to False in DB; removes job from APScheduler.
# @RELATION DEPENDS_ON -> [TranslationScheduler]
@router.post("/jobs/{job_id}/schedule/disable") @router.post("/jobs/{job_id}/schedule/disable")
async def disable_schedule( async def disable_schedule(
job_id: str, job_id: str,
@@ -166,19 +176,21 @@ async def disable_schedule(
try: try:
scheduler = TranslationScheduler(db, config_manager, current_user.username) scheduler = TranslationScheduler(db, config_manager, current_user.username)
scheduler.set_schedule_active(job_id, is_active=False) scheduler.set_schedule_active(job_id, is_active=False)
from ...dependencies import get_scheduler_service from ....dependencies import get_scheduler_service
sched_svc = get_scheduler_service() sched_svc = get_scheduler_service()
sched_svc.remove_translation_job(schedule_id=job_id) sched_svc.remove_translation_job(schedule_id=job_id)
return {"status": "disabled", "job_id": job_id} return {"status": "disabled", "job_id": job_id}
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:disable_schedule:Function] # #endregion disable_schedule
# [DEF:delete_schedule:Function] # #region delete_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]
# @PURPOSE: Delete the schedule for a translation job. # @BRIEF Delete the schedule for a translation job.
# @PRE: User has translate.schedule.manage permission. # @PRE User has translate.schedule.manage permission. Schedule exists.
# @POST: Schedule is removed. # @POST Schedule is removed. APScheduler job removed.
# @SIDE_EFFECT Deletes TranslationSchedule record from DB; removes job from APScheduler.
# @RELATION DEPENDS_ON -> [TranslationScheduler]
@router.delete("/jobs/{job_id}/schedule", status_code=status.HTTP_204_NO_CONTENT) @router.delete("/jobs/{job_id}/schedule", status_code=status.HTTP_204_NO_CONTENT)
async def delete_schedule( async def delete_schedule(
job_id: str, job_id: str,
@@ -192,19 +204,18 @@ async def delete_schedule(
try: try:
scheduler = TranslationScheduler(db, config_manager, current_user.username) scheduler = TranslationScheduler(db, config_manager, current_user.username)
scheduler.delete_schedule(job_id) scheduler.delete_schedule(job_id)
from ...dependencies import get_scheduler_service from ....dependencies import get_scheduler_service
sched_svc = get_scheduler_service() sched_svc = get_scheduler_service()
sched_svc.remove_translation_job(schedule_id=job_id) sched_svc.remove_translation_job(schedule_id=job_id)
return None return None
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:delete_schedule:Function] # #endregion delete_schedule
# [DEF:get_next_executions:Function] # #region get_next_executions [C:3] [TYPE Function] [SEMANTICS api,translate,schedule]
# @PURPOSE: Preview next N executions for a job's schedule. # @BRIEF Preview next N execution times for a job's schedule.
# @PRE: User has translate.schedule.view permission. # @RELATION DEPENDS_ON -> [TranslationScheduler]
# @POST: Returns next execution times.
@router.get("/jobs/{job_id}/schedule/next-executions") @router.get("/jobs/{job_id}/schedule/next-executions")
async def get_next_executions( async def get_next_executions(
job_id: str, job_id: str,
@@ -230,6 +241,6 @@ async def get_next_executions(
} }
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:get_next_executions:Function] # #endregion get_next_executions
# [/DEF:TranslateScheduleRoutesModule:Module] # #endregion TranslateScheduleRoutesModule

5
backend/src/models/__init__.py
View File

@@ -1,3 +1,2 @@
# [DEF:ModelsPackage:Package] # #region ModelsPackage [C:1] [TYPE Package]
# @PURPOSE: Domain model package root. # #endregion ModelsPackage
# [/DEF:ModelsPackage:Package]

78
backend/src/models/translate.py
View File

@@ -1,9 +1,6 @@
# [DEF:TranslateModels:Module] # #region TranslateModels [C:2] [TYPE Module]
# @COMPLEXITY: 3 # @BRIEF SQLAlchemy ORM models for LLM-based SQL/dashboard translation across dialects.
# @SEMANTICS: translate, models, sqlalchemy, persistence # @LAYER Domain
# @PURPOSE: SQLAlchemy ORM models for LLM-based SQL/dashboard translation across dialects.
# @LAYER: Domain
# @RELATION: INHERITS_FROM -> MappingModels:Base
from sqlalchemy import ( from sqlalchemy import (
Column, String, Boolean, DateTime, JSON, Text, Column, String, Boolean, DateTime, JSON, Text,
@@ -16,14 +13,13 @@ import hashlib
from .mapping import Base from .mapping import Base
# #region generate_uuid [C:1] [TYPE Function]
def generate_uuid(): def generate_uuid():
return str(uuid.uuid4()) return str(uuid.uuid4())
# #endregion generate_uuid
# [DEF:TranslationJob:Class] # #region TranslationJob [C:1] [TYPE Class]
# @PURPOSE: A translation job representing a multi-dialect conversion task with column mappings, LLM config, and dictionary attachments.
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity.
class TranslationJob(Base): class TranslationJob(Base):
__tablename__ = "translation_jobs" __tablename__ = "translation_jobs"
@@ -55,15 +51,15 @@ class TranslationJob(Base):
# Environment association # Environment association
environment_id = Column(String, nullable=True, comment="Superset environment ID for datasource access") environment_id = Column(String, nullable=True, comment="Superset environment ID for datasource access")
target_database_id = Column(String, nullable=True, comment="Superset database ID for INSERT via SQL Lab")
created_by = Column(String, nullable=True) created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
# [/DEF:TranslationJob:Class] # #endregion TranslationJob
# [DEF:TranslationRun:Class] # #region TranslationRun [C:1] [TYPE Class]
# @PURPOSE: Represents a single execution of a translation job, capturing status, timing, and Superset execution metadata.
class TranslationRun(Base): class TranslationRun(Base):
__tablename__ = "translation_runs" __tablename__ = "translation_runs"
@@ -87,11 +83,10 @@ class TranslationRun(Base):
dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at run time") dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at run time")
created_by = Column(String, nullable=True) created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
# [/DEF:TranslationRun:Class] # #endregion TranslationRun
# [DEF:TranslationBatch:Class] # #region TranslationBatch [C:1] [TYPE Class]
# @PURPOSE: Groups translation records within a run into manageable batches with timing and record counts.
class TranslationBatch(Base): class TranslationBatch(Base):
__tablename__ = "translation_batches" __tablename__ = "translation_batches"
@@ -105,11 +100,10 @@ class TranslationBatch(Base):
started_at = Column(DateTime, nullable=True) started_at = Column(DateTime, nullable=True)
completed_at = Column(DateTime, nullable=True) completed_at = Column(DateTime, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
# [/DEF:TranslationBatch:Class] # #endregion TranslationBatch
# [DEF:TranslationRecord:Class] # #region TranslationRecord [C:1] [TYPE Class]
# @PURPOSE: Individual translation result for a single SQL statement or dashboard element, tracking source, target, and error state.
class TranslationRecord(Base): class TranslationRecord(Base):
__tablename__ = "translation_records" __tablename__ = "translation_records"
@@ -126,16 +120,16 @@ class TranslationRecord(Base):
token_count_input = Column(Integer, nullable=True) token_count_input = Column(Integer, nullable=True)
token_count_output = Column(Integer, nullable=True) token_count_output = Column(Integer, nullable=True)
translation_duration_ms = Column(Integer, nullable=True) translation_duration_ms = Column(Integer, nullable=True)
source_data = Column(JSON, nullable=True, comment="Snapshot of source row key/context column values for INSERT phase")
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
__table_args__ = ( __table_args__ = (
Index("ix_translation_records_run_status", "run_id", "status"), Index("ix_translation_records_run_status", "run_id", "status"),
) )
# [/DEF:TranslationRecord:Class] # #endregion TranslationRecord
# [DEF:TranslationEvent:Class] # #region TranslationEvent [C:1] [TYPE Class]
# @PURPOSE: Audit/event log for translation operations, with optional run_id for context.
class TranslationEvent(Base): class TranslationEvent(Base):
__tablename__ = "translation_events" __tablename__ = "translation_events"
@@ -146,11 +140,10 @@ class TranslationEvent(Base):
event_data = Column(JSON, nullable=True) event_data = Column(JSON, nullable=True)
created_by = Column(String, nullable=True) created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
# [/DEF:TranslationEvent:Class] # #endregion TranslationEvent
# [DEF:TranslationPreviewSession:Class] # #region TranslationPreviewSession [C:1] [TYPE Class]
# @PURPOSE: A preview session allowing users to review proposed translations before applying them.
class TranslationPreviewSession(Base): class TranslationPreviewSession(Base):
__tablename__ = "translation_preview_sessions" __tablename__ = "translation_preview_sessions"
@@ -161,11 +154,10 @@ class TranslationPreviewSession(Base):
created_by = Column(String, nullable=True) created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
expires_at = Column(DateTime, nullable=True) expires_at = Column(DateTime, nullable=True)
# [/DEF:TranslationPreviewSession:Class] # #endregion TranslationPreviewSession
# [DEF:TranslationPreviewRecord:Class] # #region TranslationPreviewRecord [C:1] [TYPE Class]
# @PURPOSE: Individual preview entry within a preview session, showing original and translated content side by side.
class TranslationPreviewRecord(Base): class TranslationPreviewRecord(Base):
__tablename__ = "translation_preview_records" __tablename__ = "translation_preview_records"
@@ -178,12 +170,12 @@ class TranslationPreviewRecord(Base):
source_object_name = Column(String, nullable=True) source_object_name = Column(String, nullable=True)
status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED
feedback = Column(Text, nullable=True) feedback = Column(Text, nullable=True)
source_data = Column(JSON, nullable=True, comment="Snapshot of source row values for key/context columns")
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
# [/DEF:TranslationPreviewRecord:Class] # #endregion TranslationPreviewRecord
# [DEF:TerminologyDictionary:Class] # #region TerminologyDictionary [C:1] [TYPE Class]
# @PURPOSE: A named collection of terminology mappings used during translation to ensure consistent term translation.
class TerminologyDictionary(Base): class TerminologyDictionary(Base):
__tablename__ = "terminology_dictionaries" __tablename__ = "terminology_dictionaries"
@@ -196,11 +188,10 @@ class TerminologyDictionary(Base):
created_by = Column(String, nullable=True) created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
# [/DEF:TerminologyDictionary:Class] # #endregion TerminologyDictionary
# [DEF:DictionaryEntry:Class] # #region DictionaryEntry [C:1] [TYPE Class]
# @PURPOSE: A single term mapping entry with case-insensitive unique constraint on source_term_normalized and origin tracking.
class DictionaryEntry(Base): class DictionaryEntry(Base):
__tablename__ = "dictionary_entries" __tablename__ = "dictionary_entries"
@@ -222,11 +213,10 @@ class DictionaryEntry(Base):
name="uq_dictionary_entry_term" name="uq_dictionary_entry_term"
), ),
) )
# [/DEF:DictionaryEntry:Class] # #endregion DictionaryEntry
# [DEF:TranslationSchedule:Class] # #region TranslationSchedule [C:1] [TYPE Class]
# @PURPOSE: Defines a cron-based schedule for recurring translation jobs.
class TranslationSchedule(Base): class TranslationSchedule(Base):
__tablename__ = "translation_schedules" __tablename__ = "translation_schedules"
@@ -240,11 +230,10 @@ class TranslationSchedule(Base):
created_by = Column(String, nullable=True) created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
# [/DEF:TranslationSchedule:Class] # #endregion TranslationSchedule
# [DEF:TranslationJobDictionary:Class] # #region TranslationJobDictionary [C:1] [TYPE Class]
# @PURPOSE: Many-to-many association between translation jobs and terminology dictionaries.
class TranslationJobDictionary(Base): class TranslationJobDictionary(Base):
__tablename__ = "translation_job_dictionaries" __tablename__ = "translation_job_dictionaries"
@@ -259,11 +248,10 @@ class TranslationJobDictionary(Base):
name="uq_job_dictionary" name="uq_job_dictionary"
), ),
) )
# [/DEF:TranslationJobDictionary:Class] # #endregion TranslationJobDictionary
# [DEF:MetricSnapshot:Class] # #region MetricSnapshot [C:1] [TYPE Class]
# @PURPOSE: Captures translation performance metrics at a point in time with config/content/dictionary hash keys for aggregation.
class MetricSnapshot(Base): class MetricSnapshot(Base):
__tablename__ = "translation_metric_snapshots" __tablename__ = "translation_metric_snapshots"
@@ -290,6 +278,6 @@ class MetricSnapshot(Base):
__table_args__ = ( __table_args__ = (
Index("ix_metric_snapshots_job_date", "job_id", "snapshot_date"), Index("ix_metric_snapshots_job_date", "job_id", "snapshot_date"),
) )
# [/DEF:MetricSnapshot:Class] # #endregion MetricSnapshot
# [/DEF:TranslateModels:Module] # #endregion TranslateModels

5
backend/src/plugins/translate/__init__.py
View File

@@ -1,3 +1,2 @@
# [DEF:TranslatePluginPackage:Package] # #region TranslatePluginPackage [C:1] [TYPE Package]
# @PURPOSE: Translation plugin package for LLM-based cross-Superset SQL/dashboard translation. # #endregion TranslatePluginPackage
# [/DEF:TranslatePluginPackage:Package]

2
backend/src/plugins/translate/__tests__/test_orchestrator.py
View File

@@ -41,6 +41,7 @@ def mock_job() -> MagicMock:
job.source_datasource_id = "42" job.source_datasource_id = "42"
job.translation_column = "name" job.translation_column = "name"
job.context_columns = ["category"] job.context_columns = ["category"]
job.source_table = "source_table_name"
job.target_schema = "public" job.target_schema = "public"
job.target_table = "translated_data" job.target_table = "translated_data"
job.target_key_cols = ["id"] job.target_key_cols = ["id"]
@@ -152,7 +153,6 @@ class TestTranslationOrchestrator:
# @PURPOSE: start_run creates a PENDING run with event. # @PURPOSE: start_run creates a PENDING run with event.
def test_start_run_success( def test_start_run_success(
self, self,
mock_service,
mock_job: MagicMock, mock_job: MagicMock,
mock_preview_session: MagicMock, mock_preview_session: MagicMock,
) -> None: ) -> None:

22
backend/src/plugins/translate/_utils.py
View File

@@ -1,30 +1,22 @@
# [DEF:DictionaryUtils:Module] # #region DictionaryUtils [C:1] [TYPE Module]
# @COMPLEXITY: 1 # #endregion DictionaryUtils
# @SEMANTICS: dictionary, utils, normalize, delimiter
# @PURPOSE: Utility helpers for dictionary management (term normalization and delimiter detection).
# @LAYER: Domain
import re import re
import unicodedata import unicodedata
# [DEF:_normalize_term:Function] # #region _normalize_term [C:2] [TYPE Function] [SEMANTICS dictionary,normalize]
# @PURPOSE: Normalize a term for case-insensitive unique constraint lookup. # @BRIEF Normalize a term for case-insensitive unique constraint lookup using NFC normalization.
# @RATIONALE: NFC normalization is applied before lowercasing to ensure consistent
# comparison of Unicode characters (e.g. precomposed vs decomposed forms).
# @REJECTED: Lowercasing without NFC normalization — would cause duplicate entries
# for semantically identical Unicode strings in different normalization forms.
def _normalize_term(term: str) -> str: def _normalize_term(term: str) -> str:
"""Normalize a term by NFC, lowercasing, and removing extra whitespace.""" """Normalize a term by NFC, lowercasing, and removing extra whitespace."""
if not term: if not term:
return "" return ""
term = unicodedata.normalize('NFC', term) term = unicodedata.normalize('NFC', term)
return " ".join(term.lower().split()) return " ".join(term.lower().split())
# [/DEF:_normalize_term:Function] # #endregion _normalize_term
# [DEF:_detect_delimiter:Function] # #region _detect_delimiter [C:1] [TYPE Function] [SEMANTICS dictionary,delimiter]
# @PURPOSE: Detect the delimiter used in a CSV/TSV header line.
def _detect_delimiter(header_line: str) -> str: def _detect_delimiter(header_line: str) -> str:
"""Detect delimiter by counting tabs vs commas in the first line.""" """Detect delimiter by counting tabs vs commas in the first line."""
if not header_line: if not header_line:
@@ -32,4 +24,4 @@ def _detect_delimiter(header_line: str) -> str:
tab_count = header_line.count("\t") tab_count = header_line.count("\t")
comma_count = header_line.count(",") comma_count = header_line.count(",")
return "\t" if tab_count > comma_count else "," return "\t" if tab_count > comma_count else ","
# [/DEF:_detect_delimiter:Function] # #endregion _detect_delimiter

191
backend/src/plugins/translate/dictionary.py
View File

@@ -1,15 +1,15 @@
# [DEF:DictionaryManagerModule:Module] # #region DictionaryManagerModule [C:5] [TYPE Module] [SEMANTICS dictionary,manager,terminology,crud,import,filter]
# @COMPLEXITY: 4 # @BRIEF Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering.
# @SEMANTICS: dictionary, manager, terminology, crud, import, filter # @LAYER Domain
# @PURPOSE: Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering. # @RELATION DEPENDS_ON -> [TerminologyDictionary]
# @LAYER: Domain # @RELATION DEPENDS_ON -> [DictionaryEntry]
# @RELATION: DEPENDS_ON -> [TerminologyDictionary:Class] # @RELATION DEPENDS_ON -> [TranslationJobDictionary]
# @RELATION: DEPENDS_ON -> [DictionaryEntry:Class] # @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION: DEPENDS_ON -> [TranslationJobDictionary:Class] # @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary]
# @RELATION: DEPENDS_ON -> [TranslationJob:Class] # @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) prohibits duplicate entries within a dictionary.
# @RATIONALE: C4 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term. # @RATIONALE C5 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term.
# @REJECTED: Pure C3 CRUD without state guards would allow orphaned job-dictionary links. # @REJECTED Pure C3 CRUD without state guards would allow orphaned job-dictionary links.
# @REJECTED: "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing. # @REJECTED "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing.
import csv import csv
import io import io
@@ -27,18 +27,17 @@ from ...models.translate import (
from ._utils import _normalize_term, _detect_delimiter from ._utils import _normalize_term, _detect_delimiter
# [DEF:DictionaryManager:Class] # #region DictionaryManager [C:5] [TYPE Class] [SEMANTICS dictionary,manager,crud,import]
# @COMPLEXITY: 4 # @BRIEF Manages terminology dictionaries and their entries with referential integrity, import, and batch filtering.
# @PURPOSE: Manages terminology dictionaries and their entries with referential integrity. # @PRE Database session is open and valid.
# @PRE: Database session is open and valid. # @POST Dictionary and entry mutations are persisted with conflict detection.
# @POST: Dictionary and entry mutations are persisted with conflict detection. # @SIDE_EFFECT Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
# @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards. # @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary]
# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) enforces no duplicates within a dictionary.
class DictionaryManager: class DictionaryManager:
# [DEF:DictionaryManager.create_dictionary:Function] # #region DictionaryManager.create_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,create]
# @COMPLEXITY: 3 # @BRIEF Create a new terminology dictionary with name, source_dialect, and target_dialect.
# @PURPOSE: Create a new terminology dictionary. # @RELATION DEPENDS_ON -> [TerminologyDictionary]
# @PRE: payload contains name, source_dialect, target_dialect.
# @POST: New TerminologyDictionary row is created and returned.
@staticmethod @staticmethod
def create_dictionary( def create_dictionary(
db: Session, name: str, source_dialect: str, target_dialect: str, db: Session, name: str, source_dialect: str, target_dialect: str,
@@ -60,13 +59,11 @@ class DictionaryManager:
db.refresh(dictionary) db.refresh(dictionary)
logger.reflect("Dictionary created", {"id": dictionary.id}) logger.reflect("Dictionary created", {"id": dictionary.id})
return dictionary return dictionary
# [/DEF:DictionaryManager.create_dictionary:Function] # #endregion DictionaryManager.create_dictionary
# [DEF:DictionaryManager.update_dictionary:Function] # #region DictionaryManager.update_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,update]
# @COMPLEXITY: 3 # @BRIEF Update an existing terminology dictionary's metadata fields.
# @PURPOSE: Update an existing terminology dictionary. # @RELATION DEPENDS_ON -> [TerminologyDictionary]
# @PRE: dict_id exists in terminology_dictionaries table.
# @POST: Dictionary metadata is updated and returned.
@staticmethod @staticmethod
def update_dictionary( def update_dictionary(
db: Session, dict_id: str, name: Optional[str] = None, db: Session, dict_id: str, name: Optional[str] = None,
@@ -92,14 +89,13 @@ class DictionaryManager:
db.refresh(dictionary) db.refresh(dictionary)
logger.reflect("Dictionary updated", {"id": dictionary.id}) logger.reflect("Dictionary updated", {"id": dictionary.id})
return dictionary return dictionary
# [/DEF:DictionaryManager.update_dictionary:Function] # #endregion DictionaryManager.update_dictionary
# [DEF:DictionaryManager.delete_dictionary:Function] # #region DictionaryManager.delete_dictionary [C:4] [TYPE Function] [SEMANTICS dictionary,delete]
# @COMPLEXITY: 4 # @BRIEF Delete a dictionary, blocked if attached to active/scheduled jobs (referential integrity guard).
# @PURPOSE: Delete a dictionary, blocked if attached to active/scheduled jobs. # @PRE dict_id exists.
# @PRE: dict_id exists. # @POST Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.
# @POST: Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs. # @SIDE_EFFECT Deletes TerminologyDictionary row and all DictionaryEntry rows; checks job attachments first.
# @SIDE_EFFECT: Deletes TerminologyDictionary row and all DictionaryEntry rows.
@staticmethod @staticmethod
def delete_dictionary(db: Session, dict_id: str) -> None: def delete_dictionary(db: Session, dict_id: str) -> None:
with belief_scope("DictionaryManager.delete_dictionary"): with belief_scope("DictionaryManager.delete_dictionary"):
@@ -136,26 +132,20 @@ class DictionaryManager:
db.delete(dictionary) db.delete(dictionary)
db.commit() db.commit()
logger.reflect("Dictionary deleted", {"id": dict_id}) logger.reflect("Dictionary deleted", {"id": dict_id})
# [/DEF:DictionaryManager.delete_dictionary:Function] # #endregion DictionaryManager.delete_dictionary
# [DEF:DictionaryManager.get_dictionary:Function] # #region DictionaryManager.get_dictionary [C:2] [TYPE Function] [SEMANTICS dictionary,get]
# @COMPLEXITY: 2 # @BRIEF Get a single dictionary by ID with entry count.
# @PURPOSE: Get a single dictionary by ID with entry count.
# @PRE: dict_id exists.
# @POST: Returns dict with dictionary + entry_count or raises ValueError.
@staticmethod @staticmethod
def get_dictionary(db: Session, dict_id: str) -> TerminologyDictionary: def get_dictionary(db: Session, dict_id: str) -> TerminologyDictionary:
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
if not dictionary: if not dictionary:
raise ValueError(f"Dictionary not found: {dict_id}") raise ValueError(f"Dictionary not found: {dict_id}")
return dictionary return dictionary
# [/DEF:DictionaryManager.get_dictionary:Function] # #endregion DictionaryManager.get_dictionary
# [DEF:DictionaryManager.list_dictionaries:Function] # #region DictionaryManager.list_dictionaries [C:2] [TYPE Function] [SEMANTICS dictionary,list]
# @COMPLEXITY: 2 # @BRIEF List dictionaries with pagination and total count.
# @PURPOSE: List dictionaries with pagination and entry counts.
# @PRE: page >= 1, page_size between 1 and 100.
# @POST: Returns (list of dicts, total_count).
@staticmethod @staticmethod
def list_dictionaries( def list_dictionaries(
db: Session, page: int = 1, page_size: int = 20, db: Session, page: int = 1, page_size: int = 20,
@@ -169,13 +159,11 @@ class DictionaryManager:
.all() .all()
) )
return dictionaries, total return dictionaries, total
# [/DEF:DictionaryManager.list_dictionaries:Function] # #endregion DictionaryManager.list_dictionaries
# [DEF:DictionaryManager.add_entry:Function] # #region DictionaryManager.add_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,add]
# @COMPLEXITY: 3 # @BRIEF Add an entry to a dictionary, enforcing unique source_term_normalized per dictionary.
# @PURPOSE: Add an entry to a dictionary, enforcing unique source_term_normalized. # @RELATION DEPENDS_ON -> [DictionaryEntry]
# @PRE: dict_id exists. source_term and target_term are non-empty.
# @POST: New DictionaryEntry row is created or raises on duplicate.
@staticmethod @staticmethod
def add_entry( def add_entry(
db: Session, dict_id: str, source_term: str, target_term: str, db: Session, dict_id: str, source_term: str, target_term: str,
@@ -210,13 +198,11 @@ class DictionaryManager:
db.refresh(entry) db.refresh(entry)
logger.reflect("Entry added", {"entry_id": entry.id}) logger.reflect("Entry added", {"entry_id": entry.id})
return entry return entry
# [/DEF:DictionaryManager.add_entry:Function] # #endregion DictionaryManager.add_entry
# [DEF:DictionaryManager.edit_entry:Function] # #region DictionaryManager.edit_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,edit]
# @COMPLEXITY: 3 # @BRIEF Edit an existing dictionary entry with duplicate-aware normalization check.
# @PURPOSE: Edit an existing dictionary entry with duplicate-aware normalization. # @RELATION DEPENDS_ON -> [DictionaryEntry]
# @PRE: entry_id exists.
# @POST: Entry fields are updated.
@staticmethod @staticmethod
def edit_entry( def edit_entry(
db: Session, entry_id: str, source_term: Optional[str] = None, db: Session, entry_id: str, source_term: Optional[str] = None,
@@ -255,13 +241,10 @@ class DictionaryManager:
db.refresh(entry) db.refresh(entry)
logger.reflect("Entry updated", {"entry_id": entry.id}) logger.reflect("Entry updated", {"entry_id": entry.id})
return entry return entry
# [/DEF:DictionaryManager.edit_entry:Function] # #endregion DictionaryManager.edit_entry
# [DEF:DictionaryManager.delete_entry:Function] # #region DictionaryManager.delete_entry [C:2] [TYPE Function] [SEMANTICS dictionary,entry,delete]
# @COMPLEXITY: 2 # @BRIEF Delete a single dictionary entry.
# @PURPOSE: Delete a single dictionary entry.
# @PRE: entry_id exists.
# @POST: Entry is deleted.
@staticmethod @staticmethod
def delete_entry(db: Session, entry_id: str) -> None: def delete_entry(db: Session, entry_id: str) -> None:
with belief_scope("DictionaryManager.delete_entry"): with belief_scope("DictionaryManager.delete_entry"):
@@ -272,13 +255,10 @@ class DictionaryManager:
db.delete(entry) db.delete(entry)
db.commit() db.commit()
logger.reflect("Entry deleted", {"entry_id": entry_id}) logger.reflect("Entry deleted", {"entry_id": entry_id})
# [/DEF:DictionaryManager.delete_entry:Function] # #endregion DictionaryManager.delete_entry
# [DEF:DictionaryManager.clear_entries:Function] # #region DictionaryManager.clear_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,clear]
# @COMPLEXITY: 2 # @BRIEF Delete all entries for a dictionary.
# @PURPOSE: Delete all entries for a dictionary.
# @PRE: dict_id exists.
# @POST: All entries for the dictionary are deleted.
@staticmethod @staticmethod
def clear_entries(db: Session, dict_id: str) -> int: def clear_entries(db: Session, dict_id: str) -> int:
with belief_scope("DictionaryManager.clear_entries"): with belief_scope("DictionaryManager.clear_entries"):
@@ -290,13 +270,10 @@ class DictionaryManager:
db.commit() db.commit()
logger.reflect("Entries cleared", {"dict_id": dict_id, "count": deleted}) logger.reflect("Entries cleared", {"dict_id": dict_id, "count": deleted})
return deleted return deleted
# [/DEF:DictionaryManager.clear_entries:Function] # #endregion DictionaryManager.clear_entries
# [DEF:DictionaryManager.list_entries:Function] # #region DictionaryManager.list_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,list]
# @COMPLEXITY: 2 # @BRIEF List entries for a dictionary with pagination.
# @PURPOSE: List entries for a dictionary with pagination.
# @PRE: dict_id exists.
# @POST: Returns (list of entries, total_count).
@staticmethod @staticmethod
def list_entries( def list_entries(
db: Session, dict_id: str, page: int = 1, page_size: int = 50, db: Session, dict_id: str, page: int = 1, page_size: int = 50,
@@ -316,14 +293,13 @@ class DictionaryManager:
.all() .all()
) )
return entries, total return entries, total
# [/DEF:DictionaryManager.list_entries:Function] # #endregion DictionaryManager.list_entries
# [DEF:DictionaryManager.import_entries:Function] # #region DictionaryManager.import_entries [C:4] [TYPE Function] [SEMANTICS dictionary,import,csv]
# @COMPLEXITY: 4 # @BRIEF Import entries from CSV/TSV content with duplicate detection and conflict resolution (overwrite/keep_existing/cancel).
# @PURPOSE: Import entries from CSV/TSV content with duplicate detection and conflict resolution. # @PRE content is valid CSV or TSV. dict_id exists.
# @PRE: content is valid CSV or TSV. dict_id exists. # @POST Entries are created/updated/skipped per conflict mode. Returns result summary with preview support.
# @POST: Entries are created/updated/skipped per conflict mode. Returns result summary. # @SIDE_EFFECT Batch-inserts or updates DictionaryEntry rows.
# @SIDE_EFFECT: Batch-inserts or updates DictionaryEntry rows.
@staticmethod @staticmethod
def import_entries( def import_entries(
db: Session, dict_id: str, content: str, db: Session, dict_id: str, content: str,
@@ -456,14 +432,13 @@ class DictionaryManager:
"errors": len(result["errors"]), "errors": len(result["errors"]),
}) })
return result return result
# [/DEF:DictionaryManager.import_entries:Function] # #endregion DictionaryManager.import_entries
# [DEF:DictionaryManager.filter_for_batch:Function] # #region DictionaryManager.filter_for_batch [C:4] [TYPE Function] [SEMANTICS dictionary,filter,batch]
# @COMPLEXITY: 4 # @BRIEF Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.
# @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job. # @PRE job_id exists and source_texts is a list of strings.
# @PRE: job_id exists and source_texts is a list of strings. # @POST Returns list of matched entries with match info, sorted by dictionary link priority.
# @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries. # @SIDE_EFFECT Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.
# @SIDE_EFFECT: Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.
@staticmethod @staticmethod
def filter_for_batch( def filter_for_batch(
db: Session, source_texts: List[str], job_id: str, db: Session, source_texts: List[str], job_id: str,
@@ -554,15 +529,14 @@ class DictionaryManager:
"dictionaries_used": len(dictionaries), "dictionaries_used": len(dictionaries),
}) })
return matched return matched
# [/DEF:DictionaryManager.filter_for_batch:Function] # #endregion DictionaryManager.filter_for_batch
# [DEF:DictionaryManager.submit_correction:Function] # #region DictionaryManager.submit_correction [C:4] [TYPE Function] [SEMANTICS dictionary,correction,submit]
# @COMPLEXITY: 4 # @BRIEF Submit a term correction from a run result with conflict detection and origin tracking.
# @PURPOSE: Submit a term correction from a run result. Validates language match, detects conflicts. # @PRE source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.
# @PRE: source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists. # @POST Entry created or updated; origin tracking populated. Returns action + conflict info.
# @POST: Entry created or updated; origin tracking populated. Returns action + conflict info. # @SIDE_EFFECT Creates/updates DictionaryEntry row.
# @SIDE_EFFECT: Creates/updates DictionaryEntry row.
@staticmethod @staticmethod
def submit_correction( def submit_correction(
db: Session, db: Session,
@@ -653,14 +627,13 @@ class DictionaryManager:
db.commit() db.commit()
logger.reflect("Correction processed", result) logger.reflect("Correction processed", result)
return result return result
# [/DEF:DictionaryManager.submit_correction:Function] # #endregion DictionaryManager.submit_correction
# [DEF:DictionaryManager.submit_bulk_corrections:Function] # #region DictionaryManager.submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS dictionary,correction,bulk]
# @COMPLEXITY: 4 # @BRIEF Submit multiple term corrections atomically — all succeed or all fail on conflict.
# @PURPOSE: Submit multiple term corrections atomically — all succeed or all fail. # @PRE corrections list is non-empty. dict_id exists.
# @PRE: corrections list is non-empty. dict_id exists. # @POST All corrections applied or none applied with conflict list (atomic).
# @POST: All corrections applied or none applied with conflict list. # @SIDE_EFFECT Creates/updates DictionaryEntry rows; commits once.
# @SIDE_EFFECT: Creates/updates DictionaryEntry rows; commits once.
@staticmethod @staticmethod
def submit_bulk_corrections( def submit_bulk_corrections(
db: Session, db: Session,
@@ -774,8 +747,8 @@ class DictionaryManager:
"total": len(corrections), "total": len(corrections),
"applied": sum(1 for r in results if r["action"] in ("created", "updated")), "applied": sum(1 for r in results if r["action"] in ("created", "updated")),
} }
# [/DEF:DictionaryManager.submit_bulk_corrections:Function] # #endregion DictionaryManager.submit_bulk_corrections
# [/DEF:DictionaryManager:Class] # #endregion DictionaryManager
# [/DEF:DictionaryManagerModule:Module] # #endregion DictionaryManagerModule

154
backend/src/plugins/translate/events.py
View File

@@ -1,18 +1,16 @@
# [DEF:TranslationEventLog:Module] # #region TranslationEventLog [C:5] [TYPE Module] [SEMANTICS translate,events,audit,logging]
# @COMPLEXITY: 5 # @BRIEF Structured event logging for translation operations with terminal event invariant enforcement.
# @SEMANTICS: translate, events, audit, logging # @LAYER Domain
# @PURPOSE: Structured event logging for translation operations with terminal event invariant enforcement. # @RELATION DEPENDS_ON -> [TranslationEvent]
# @LAYER: Domain # @RELATION DEPENDS_ON -> [MetricSnapshot]
# @RELATION: DEPENDS_ON -> [TranslationEvent] # @PRE Database session is open and valid.
# @RELATION: DEPENDS_ON -> [MetricSnapshot] # @POST Events are persisted immutably; terminal events enforce exactly-one invariant per run.
# @PRE: Database session is open and valid. # @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion.
# @POST: Events are persisted immutably; terminal events enforce exactly-one invariant per run. # @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]
# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion. # @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id.
# @DATA_CONTRACT: Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent] # @RATIONALE Immutable event log with nullable run_id allows job-level events (no run context) and run-level events.
# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id. # @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# @RATIONALE: Immutable event log with nullable run_id allows job-level events (no run context) and run-level events. # @REJECTED Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit.
# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# @REJECTED: Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit.
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from datetime import datetime, timezone, timedelta from datetime import datetime, timezone, timedelta
@@ -38,24 +36,48 @@ VALID_EVENT_TYPES = TERMINAL_EVENT_TYPES | {
DEFAULT_RETENTION_DAYS = 90 DEFAULT_RETENTION_DAYS = 90
# [DEF:TranslationEventLog:Class] # #region TranslationEventLog [C:5] [TYPE Class] [SEMANTICS translate,events,log,invariants]
# @COMPLEXITY: 5 # @BRIEF Structured event logging for translation operations with terminal event invariant enforcement and pruning.
# @PURPOSE: Structured event logging for translation operations with terminal event invariant enforcement. # @PRE Database session is available.
# @PRE: Database session is available. # @POST Events are written immutably; terminal events enforced per run; expired events pruned with MetricSnapshot.
# @POST: Events are written immutably; terminal events enforced per run. # @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events.
# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events. # @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]
# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id. # @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id.
class TranslationEventLog: class TranslationEventLog:
def __init__(self, db: Session): def __init__(self, db: Session):
self.db = db self.db = db
# [DEF:log_event:Function] # #region _create_event_raw [C:3] [TYPE Function] [SEMANTICS translate,events,create]
# @COMPLEXITY: 4 # @BRIEF Create a TranslationEvent row without invariant checks. Internal helper for callers that manage invariants themselves.
# @PURPOSE: Write an immutable event. Enforces terminal event invariant for non-null run_id. # @RELATION DEPENDS_ON -> [TranslationEvent]
# @PRE: event_type must be a known type. If run_id is not None, enforce terminal invariant. def _create_event_raw(
# @POST: TranslationEvent row is created. self,
# @SIDE_EFFECT: DB write. job_id: str,
event_type: str,
payload: Optional[Dict[str, Any]] = None,
run_id: Optional[str] = None,
created_by: Optional[str] = None,
) -> TranslationEvent:
event = TranslationEvent(
id=str(uuid.uuid4()),
job_id=job_id,
run_id=run_id,
event_type=event_type,
event_data=payload or {},
created_by=created_by,
created_at=datetime.now(timezone.utc),
)
self.db.add(event)
self.db.flush()
return event
# #endregion _create_event_raw
# #region log_event [C:4] [TYPE Function] [SEMANTICS translate,events,log]
# @BRIEF Write an immutable event. Enforces terminal event invariant and run_started ordering for non-null run_id.
# @PRE event_type must be a known type. If run_id is not None, enforce terminal + start invariants.
# @POST TranslationEvent row is created; invariants checked before write.
# @SIDE_EFFECT DB write.
def log_event( def log_event(
self, self,
job_id: str, job_id: str,
@@ -82,9 +104,10 @@ class TranslationEventLog:
.first() .first()
) )
if existing_terminal: if existing_terminal:
existing_id = existing_terminal[0] if isinstance(existing_terminal, (tuple, list)) else str(existing_terminal.id)
raise ValueError( raise ValueError(
f"Run '{run_id}' already has a terminal event " f"Run '{run_id}' already has a terminal event "
f"({existing_terminal[0]}). Cannot add another terminal event." f"({existing_id}). Cannot add another terminal event."
) )
# Enforce run_started invariant — must exist before other run events # Enforce run_started invariant — must exist before other run events
@@ -103,18 +126,13 @@ class TranslationEventLog:
f"RUN_STARTED event must precede other run events." f"RUN_STARTED event must precede other run events."
) )
event = TranslationEvent( event = self._create_event_raw(
id=str(uuid.uuid4()),
job_id=job_id, job_id=job_id,
run_id=run_id,
event_type=event_type, event_type=event_type,
event_data=payload or {}, payload=payload,
run_id=run_id,
created_by=created_by, created_by=created_by,
created_at=datetime.now(timezone.utc),
) )
self.db.add(event)
self.db.flush()
logger.reason(f"Event logged: {event_type}", { logger.reason(f"Event logged: {event_type}", {
"event_id": event.id, "event_id": event.id,
"job_id": job_id, "job_id": job_id,
@@ -122,12 +140,10 @@ class TranslationEventLog:
"event_type": event_type, "event_type": event_type,
}) })
return event return event
# [/DEF:log_event:Function] # #endregion log_event
# [DEF:query_events:Function] # #region query_events [C:2] [TYPE Function] [SEMANTICS translate,events,query]
# @PURPOSE: Query events with optional filters. # @BRIEF Query events with optional filters (job_id, run_id, event_type) and pagination.
# @PRE: None.
# @POST: Returns list of TranslationEvent dicts matching filters.
def query_events( def query_events(
self, self,
job_id: Optional[str] = None, job_id: Optional[str] = None,
@@ -165,13 +181,13 @@ class TranslationEventLog:
} }
for e in events for e in events
] ]
# [/DEF:query_events:Function] # #endregion query_events
# [DEF:prune_expired:Function] # #region prune_expired [C:4] [TYPE Function] [SEMANTICS translate,events,prune]
# @PURPOSE: Delete events older than retention_days. Persists MetricSnapshot before pruning. # @BRIEF Delete events older than retention_days. Creates MetricSnapshot before pruning for audit trail.
# @PRE: None. # @PRE None.
# @POST: Expired events are deleted; MetricSnapshot is created before deletion. # @POST Expired events are deleted; MetricSnapshot is created before deletion in batch.
# @SIDE_EFFECT: Creates MetricSnapshot row; deletes TranslationEvent rows. # @SIDE_EFFECT Creates MetricSnapshot row; deletes TranslationEvent rows in batches.
def prune_expired( def prune_expired(
self, self,
retention_days: int = DEFAULT_RETENTION_DAYS, retention_days: int = DEFAULT_RETENTION_DAYS,
@@ -233,12 +249,11 @@ class TranslationEventLog:
"snapshot_id": snapshot_id, "snapshot_id": snapshot_id,
}) })
return {"pruned": pruned, "snapshot_id": snapshot_id} return {"pruned": pruned, "snapshot_id": snapshot_id}
# [/DEF:prune_expired:Function] # #endregion prune_expired
# [DEF:get_run_event_summary:Function] # #region get_run_event_summary [C:3] [TYPE Function] [SEMANTICS translate,events,summary]
# @PURPOSE: Get a summary of events for a run, including invariant check. # @BRIEF Get a summary of events for a run, including invariant validity check.
# @PRE: run_id is not None. # @RELATION DEPENDS_ON -> [TranslationEvent]
# @POST: Returns dict with event list and invariant validity.
def get_run_event_summary(self, run_id: str) -> Dict[str, Any]: def get_run_event_summary(self, run_id: str) -> Dict[str, Any]:
with belief_scope("TranslationEventLog.get_run_event_summary"): with belief_scope("TranslationEventLog.get_run_event_summary"):
events = self.query_events(run_id=run_id) events = self.query_events(run_id=run_id)
@@ -255,8 +270,35 @@ class TranslationEventLog:
"invariant_valid": has_started and len(terminal_events) <= 1, "invariant_valid": has_started and len(terminal_events) <= 1,
"events": events, "events": events,
} }
# [/DEF:get_run_event_summary:Function] # #endregion get_run_event_summary
# #region get_run_event_invariants_lightweight [C:3] [TYPE Function] [SEMANTICS translate,events,invariants]
# @BRIEF Lightweight invariant check for a run — fetches only event_type column, avoids event_data for performance.
# @RELATION DEPENDS_ON -> [TranslationEvent]
def get_run_event_invariants_lightweight(self, run_id: str) -> Dict[str, Any]:
with belief_scope("TranslationEventLog.get_run_event_invariants_lightweight"):
# Lightweight query: only fetch event_type column, not event_data
event_types = [
row[0]
for row in (
self.db.query(TranslationEvent.event_type)
.filter(TranslationEvent.run_id == run_id)
.order_by(TranslationEvent.created_at.desc())
.limit(100)
.all()
)
]
has_run_started = "RUN_STARTED" in event_types
terminal_count = sum(1 for et in event_types if et in TERMINAL_EVENT_TYPES)
return {
"has_run_started": has_run_started,
"terminal_event_count": terminal_count,
"invariant_valid": has_run_started and terminal_count <= 1,
}
# #endregion get_run_event_invariants_lightweight
# [/DEF:TranslationEventLog:Class] # #endregion TranslationEventLog
# [/DEF:TranslationEventLog:Module] # #endregion TranslationEventLog

268
backend/src/plugins/translate/executor.py
View File

@@ -1,19 +1,19 @@
# [DEF:TranslationExecutor:Module] # #region TranslationExecutor [C:5] [TYPE Module] [SEMANTICS translate,executor,batch,llm]
# @COMPLEXITY: 4 # @BRIEF Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows.
# @SEMANTICS: translate, executor, batch, llm # @LAYER Domain
# @PURPOSE: Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows. # @RELATION DEPENDS_ON -> [TranslationBatch]
# @LAYER: Domain # @RELATION DEPENDS_ON -> [TranslationRecord]
# @RELATION: DEPENDS_ON -> [TranslationBatch] # @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION: DEPENDS_ON -> [TranslationRecord] # @RELATION DEPENDS_ON -> [LLMProviderService]
# @RELATION: DEPENDS_ON -> [TranslationRun] # @RELATION DEPENDS_ON -> [DictionaryManager]
# @RELATION: DEPENDS_ON -> [LLMProviderService] # @RELATION DEPENDS_ON -> [TranslationPreview]
# @RELATION: DEPENDS_ON -> [DictionaryManager] # @PRE Valid TranslationRun with job configuration. DB session is available.
# @RELATION: DEPENDS_ON -> [TranslationPreview] # @POST TranslationBatch and TranslationRecord rows are created. Run status is updated.
# @PRE: Valid TranslationRun with job configuration. DB session is available. # @SIDE_EFFECT Calls LLM provider; creates DB rows; updates run statistics.
# @POST: TranslationBatch and TranslationRecord rows are created. Run status is updated. # @DATA_CONTRACT Input[TranslationRun, job] -> Output[TranslationRun with batches and records]
# @SIDE_EFFECT: Calls LLM provider; creates DB rows; updates run statistics. # @INVARIANT Batch processing with retry — independent batches allow partial recovery; MAX_RETRIES_PER_BATCH = 3.
# @RATIONALE: Batch processing with retry — independent batches allow partial recovery. # @RATIONALE Batch processing with retry — independent batches allow partial recovery.
# @REJECTED: Single monolithic LLM call — would lose all progress on any failure. # @REJECTED Single monolithic LLM call — would lose all progress on any failure.
import json import json
import time import time
@@ -35,21 +35,23 @@ from ...models.translate import (
) )
from ...services.llm_provider import LLMProviderService from ...services.llm_provider import LLMProviderService
from ...services.llm_prompt_templates import render_prompt from ...services.llm_prompt_templates import render_prompt
from ...core.superset_client import SupersetClient
from .dictionary import DictionaryManager from .dictionary import DictionaryManager
from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE
# [DEF:MAX_RETRIES_PER_BATCH:Constant] # #region MAX_RETRIES_PER_BATCH [C:1] [TYPE Constant] [SEMANTICS translate,executor,retry]
# @PURPOSE: Maximum number of retries for a single batch before marking it failed.
MAX_RETRIES_PER_BATCH = 3 MAX_RETRIES_PER_BATCH = 3
# #endregion MAX_RETRIES_PER_BATCH
# [DEF:TranslationExecutor:Class] # #region TranslationExecutor [C:5] [TYPE Class] [SEMANTICS translate,executor,batch]
# @COMPLEXITY: 4 # @BRIEF Process translation batches: fetch source rows, filter dict, call LLM, persist results.
# @PURPOSE: Process translation batches: fetch source rows, filter dict, call LLM, persist results. # @PRE DB session and config manager available.
# @PRE: DB session and config manager available. # @POST Batches and records created with status tracking; run statistics updated.
# @POST: Batches and records created with status tracking. # @SIDE_EFFECT LLM API calls; DB writes.
# @SIDE_EFFECT: LLM API calls; DB writes. # @DATA_CONTRACT Input[TranslationRun] -> Output[TranslationRun with batches, records, stats]
# @INVARIANT MAX_RETRIES_PER_BATCH limit enforced; partial batch success tracked via COMPLETED_WITH_ERRORS.
class TranslationExecutor: class TranslationExecutor:
def __init__( def __init__(
@@ -65,15 +67,16 @@ class TranslationExecutor:
self.on_batch_progress = on_batch_progress self.on_batch_progress = on_batch_progress
self._current_run_id: Optional[str] = None self._current_run_id: Optional[str] = None
# [DEF:execute_run:Function] # #region execute_run [C:4] [TYPE Function] [SEMANTICS translate,run,execute]
# @PURPOSE: Run full translation execution for a TranslationRun. # @BRIEF Run full translation execution for a TranslationRun: fetch rows, batch, call LLM, persist.
# @PRE: run is in PENDING or RUNNING status with valid job config. # @PRE run is in PENDING or RUNNING status with valid job config.
# @POST: Run is populated with batches and records. # @POST Run is populated with batches and records; run status updated to COMPLETED/FAILED.
# @SIDE_EFFECT: LLM API calls; DB batch writes. # @SIDE_EFFECT LLM API calls; DB batch writes.
def execute_run( def execute_run(
self, self,
run: TranslationRun, run: TranslationRun,
llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None, llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None,
full_translation: bool = False,
) -> TranslationRun: ) -> TranslationRun:
with belief_scope("TranslationExecutor.execute_run"): with belief_scope("TranslationExecutor.execute_run"):
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first() job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
@@ -84,6 +87,7 @@ class TranslationExecutor:
"run_id": run.id, "run_id": run.id,
"job_id": job.id, "job_id": job.id,
"batch_size": job.batch_size, "batch_size": job.batch_size,
"full_translation": full_translation,
}) })
# Mark run as RUNNING # Mark run as RUNNING
@@ -91,8 +95,13 @@ class TranslationExecutor:
run.started_at = datetime.now(timezone.utc) run.started_at = datetime.now(timezone.utc)
self.db.flush() self.db.flush()
# Fetch source rows from the accepted preview session # Fetch source rows
source_rows = self._fetch_source_rows(job.id, run.id) if full_translation:
# Full translation: fetch ALL rows from Superset dataset
source_rows = self._fetch_all_rows_from_superset(job)
else:
# Preview-based: fetch rows from the accepted preview session
source_rows = self._fetch_source_rows(job.id, run.id)
if not source_rows: if not source_rows:
logger.explore("No source rows to translate", {"run_id": run.id}) logger.explore("No source rows to translate", {"run_id": run.id})
run.status = "COMPLETED" run.status = "COMPLETED"
@@ -164,12 +173,13 @@ class TranslationExecutor:
}) })
return run return run
# [/DEF:execute_run:Function] # #endregion execute_run
# [DEF:_fetch_source_rows:Function] # #region _fetch_source_rows [C:4] [TYPE Function] [SEMANTICS translate,source,rows]
# @PURPOSE: Fetch source rows from the accepted preview session for this job. # @BRIEF Fetch source rows from the accepted preview session for this job.
# @PRE: job_id exists. # @PRE job_id exists.
# @POST: Returns list of dicts with source data. # @POST Returns list of dicts with source data (row_index, source_text, approved_translation, etc).
# @SIDE_EFFECT Queries preview session and records from DB.
def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]: def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]:
with belief_scope("TranslationExecutor._fetch_source_rows"): with belief_scope("TranslationExecutor._fetch_source_rows"):
# Get the latest APPLIED preview session # Get the latest APPLIED preview session
@@ -203,6 +213,7 @@ class TranslationExecutor:
"source_text": rec.source_sql or "", "source_text": rec.source_sql or "",
"approved_translation": rec.target_sql if rec.status == "APPROVED" else None, "approved_translation": rec.target_sql if rec.status == "APPROVED" else None,
"source_object_name": rec.source_object_name or "", "source_object_name": rec.source_object_name or "",
"source_data": rec.source_data or {},
}) })
logger.reason(f"Fetched {len(source_rows)} source rows from preview", { logger.reason(f"Fetched {len(source_rows)} source rows from preview", {
@@ -210,13 +221,115 @@ class TranslationExecutor:
"session_id": session.id, "session_id": session.id,
}) })
return source_rows return source_rows
# [/DEF:_fetch_source_rows:Function] # #endregion _fetch_source_rows
# [DEF:_process_batch:Function] # #region _fetch_all_rows_from_superset [C:4] [TYPE Function] [SEMANTICS translate,superset,rows]
# @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records. # @BRIEF Fetch ALL rows from the Superset dataset for full translation (paginated).
# @PRE: job and batch_rows are valid. # @PRE job has source_datasource_id and environment_id configured.
# @POST: TranslationBatch and TranslationRecord rows are created. # @POST Returns list of row dicts with row_index, source_text, source_data.
# @SIDE_EFFECT: LLM API call. # @SIDE_EFFECT Calls Superset chart data endpoint (paginated).
def _fetch_all_rows_from_superset(self, job: TranslationJob) -> List[Dict[str, Any]]:
with belief_scope("TranslationExecutor._fetch_all_rows_from_superset"):
env_id = job.environment_id or job.source_dialect or ""
environments = self.config_manager.get_environments()
env_config = next(
(e for e in environments if e.id == env_id),
None,
)
if not env_config:
raise ValueError(f"Superset environment '{env_id}' not found")
client = SupersetClient(env_config)
dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))
# Build query context (same as preview, but with large row_limit)
query_context = client.build_dataset_preview_query_context(
dataset_id=int(job.source_datasource_id),
dataset_record=dataset_detail,
template_params={},
effective_filters=[],
)
queries = query_context.get("queries", [])
if queries:
queries[0]["row_limit"] = 100000 # Superset max default
queries[0].pop("result_type", None)
queries[0].pop("columns", None)
queries[0]["metrics"] = []
queries[0]["row_offset"] = 0
query_context["result_type"] = "samples"
form_data = query_context.get("form_data", {})
form_data.pop("query_mode", None)
all_rows: List[Dict[str, Any]] = []
batch_size_fetch = 10000 # Fetch 10K rows per pagination request
while True:
# Set pagination for this batch
if queries:
queries[0]["row_limit"] = min(batch_size_fetch, 100000)
queries[0]["row_offset"] = len(all_rows)
try:
response = client.network.request(
method="POST",
endpoint="/api/v1/chart/data",
data=json.dumps(query_context),
headers={"Content-Type": "application/json"},
)
except Exception as e:
logger.explore("Chart data API failed during full fetch", {
"offset": len(all_rows),
"error": str(e),
})
if not all_rows:
raise ValueError(f"Failed to fetch data from Superset: {e}")
break # Return what we have
from .preview import TranslationPreview
rows = TranslationPreview._extract_data_rows(response)
if not rows:
break # No more data
all_rows.extend(rows)
logger.reason(f"Fetched {len(rows)} rows (total: {len(all_rows)})", {
"offset": len(all_rows) - len(rows),
})
if len(rows) < batch_size_fetch:
break # Last page
# Convert to the format expected by _process_batch
source_rows = []
for idx, row in enumerate(all_rows):
translation_value = str(row.get(job.translation_column, "") or "")
context_values = {}
if job.context_columns:
for col in job.context_columns:
context_values[col] = str(row.get(col, "") or "")
# Also include target_key_cols values in context_data
if job.target_key_cols:
for col in job.target_key_cols:
context_values[col] = str(row.get(col, "") or "")
source_rows.append({
"row_index": str(idx),
"source_text": translation_value,
"source_data": context_values,
"source_object_name": f"Row {idx + 1}",
"approved_translation": None,
})
logger.reason(f"Prepared {len(source_rows)} source rows for full translation", {
"job_id": job.id,
})
return source_rows
# #endregion _fetch_all_rows_from_superset
# #region _process_batch [C:4] [TYPE Function] [SEMANTICS translate,batch,process]
# @BRIEF Process a single batch: filter dictionary, build prompt, call LLM, persist records.
# @PRE job and batch_rows are valid.
# @POST TranslationBatch and TranslationRecord rows are created.
# @SIDE_EFFECT LLM API call; DB writes.
def _process_batch( def _process_batch(
self, self,
job: TranslationJob, job: TranslationJob,
@@ -275,6 +388,7 @@ class TranslationExecutor:
source_object_type="table_row", source_object_type="table_row",
source_object_id=row.get("row_index"), source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""), source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SUCCESS", status="SUCCESS",
) )
self.db.add(record) self.db.add(record)
@@ -309,13 +423,13 @@ class TranslationExecutor:
}) })
return result return result
# [/DEF:_process_batch:Function] # #endregion _process_batch
# [DEF:_call_llm_for_batch:Function] # #region _call_llm_for_batch [C:4] [TYPE Function] [SEMANTICS translate,llm,batch]
# @PURPOSE: Call LLM for a batch of rows requiring translation. Parse structured JSON response. # @BRIEF Call LLM for a batch of rows requiring translation. Parse structured JSON response.
# @PRE: job has valid provider_id. batch_rows is non-empty. # @PRE job has valid provider_id. batch_rows is non-empty.
# @POST: Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows. # @POST Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.
# @SIDE_EFFECT: HTTP call to LLM provider. # @SIDE_EFFECT HTTP call to LLM provider.
def _call_llm_for_batch( def _call_llm_for_batch(
self, self,
job: TranslationJob, job: TranslationJob,
@@ -401,6 +515,7 @@ class TranslationExecutor:
source_object_type="table_row", source_object_type="table_row",
source_object_id=row.get("row_index"), source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""), source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="FAILED", status="FAILED",
error_message=f"LLM call failed after {retries} retries: {last_error}", error_message=f"LLM call failed after {retries} retries: {last_error}",
) )
@@ -428,6 +543,7 @@ class TranslationExecutor:
source_object_type="table_row", source_object_type="table_row",
source_object_id=row.get("row_index"), source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""), source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED", status="SKIPPED",
error_message=f"LLM parse failure: {e}", error_message=f"LLM parse failure: {e}",
) )
@@ -459,6 +575,7 @@ class TranslationExecutor:
source_object_type="table_row", source_object_type="table_row",
source_object_id=row.get("row_index"), source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""), source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED", status="SKIPPED",
error_message="NULL translation returned by LLM", error_message="NULL translation returned by LLM",
) )
@@ -477,6 +594,7 @@ class TranslationExecutor:
source_object_type="table_row", source_object_type="table_row",
source_object_id=row.get("row_index"), source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""), source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED", status="SKIPPED",
error_message="Empty translation returned by LLM", error_message="Empty translation returned by LLM",
) )
@@ -493,6 +611,7 @@ class TranslationExecutor:
source_object_type="table_row", source_object_type="table_row",
source_object_id=row.get("row_index"), source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""), source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SUCCESS", status="SUCCESS",
) )
self.db.add(record) self.db.add(record)
@@ -503,13 +622,13 @@ class TranslationExecutor:
"skipped": skipped, "skipped": skipped,
"retries": retries, "retries": retries,
} }
# [/DEF:_call_llm_for_batch:Function] # #endregion _call_llm_for_batch
# [DEF:_call_llm:Function] # #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call]
# @PURPOSE: Call the configured LLM provider with the batch prompt. # @BRIEF Call the configured LLM provider with the batch prompt, routing by provider type.
# @PRE: job has valid provider_id. # @PRE job has valid provider_id.
# @POST: Returns raw LLM response string. # @POST Returns raw LLM response string.
# @SIDE_EFFECT: HTTP call to LLM provider. # @SIDE_EFFECT HTTP call to LLM provider.
def _call_llm(self, job: TranslationJob, prompt: str) -> str: def _call_llm(self, job: TranslationJob, prompt: str) -> str:
with belief_scope("TranslationExecutor._call_llm"): with belief_scope("TranslationExecutor._call_llm"):
if not job.provider_id: if not job.provider_id:
@@ -537,13 +656,13 @@ class TranslationExecutor:
) )
else: else:
raise ValueError(f"Unsupported provider type '{provider_type}'") raise ValueError(f"Unsupported provider type '{provider_type}'")
# [/DEF:_call_llm:Function] # #endregion _call_llm
# [DEF:_call_openai_compatible:Function] # #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai]
# @PURPOSE: Call OpenAI-compatible API for batch translation. # @BRIEF Call OpenAI-compatible API for batch translation with retry and structured output support.
# @PRE: Valid API endpoint, key, model, and prompt. # @PRE Valid API endpoint, key, model, and prompt.
# @POST: Returns response text. # @POST Returns response text.
# @SIDE_EFFECT: HTTP POST to LLM API. # @SIDE_EFFECT HTTP POST to LLM API.
@staticmethod @staticmethod
def _call_openai_compatible( def _call_openai_compatible(
base_url: str, base_url: str,
@@ -596,15 +715,26 @@ class TranslationExecutor:
content = choices[0].get("message", {}).get("content", "") content = choices[0].get("message", {}).get("content", "")
if not content: if not content:
raise ValueError("LLM returned empty content") # Log full response for diagnostics
finish_reason = choices[0].get("finish_reason", "unknown")
logger.explore("LLM returned empty content", {
"finish_reason": finish_reason,
"model": payload.get("model"),
"prompt_len": len(prompt),
"response_keys": list(data.keys()),
"choices_count": len(choices),
})
raise ValueError(
f"LLM returned empty content (finish_reason={finish_reason}, "
f"model={payload.get('model')}, prompt_len={len(prompt)})"
)
return content return content
# [/DEF:_call_openai_compatible:Function] # #endregion _call_openai_compatible
# [DEF:_parse_llm_response:Function] # #region _parse_llm_response [C:3] [TYPE Function] [SEMANTICS translate,llm,parse]
# @PURPOSE: Parse LLM JSON response into dict of row_id -> translation. # @BRIEF Parse LLM JSON response into dict of row_id -> translation text.
# @PRE: response_text is valid JSON with {"rows": [...]} structure. # @RELATION DEPENDS_ON -> [json]
# @POST: Returns dict mapping row_id to translation text.
@staticmethod @staticmethod
def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]: def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:
with belief_scope("TranslationExecutor._parse_llm_response"): with belief_scope("TranslationExecutor._parse_llm_response"):
@@ -637,8 +767,8 @@ class TranslationExecutor:
translations[row_id] = str(translation) translations[row_id] = str(translation)
return translations return translations
# [/DEF:_parse_llm_response:Function] # #endregion _parse_llm_response
# [/DEF:TranslationExecutor:Class] # #endregion TranslationExecutor
# [/DEF:TranslationExecutor:Module] # #endregion TranslationExecutor

47
backend/src/plugins/translate/metrics.py
View File

@@ -1,16 +1,10 @@
# [DEF:TranslationMetrics:Module] # #region TranslationMetrics [C:3] [TYPE Module] [SEMANTICS translate,metrics,aggregation]
# @COMPLEXITY: 3 # @BRIEF Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting.
# @SEMANTICS: translate, metrics, aggregation # @LAYER Domain
# @PURPOSE: Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting. # @RELATION DEPENDS_ON -> [TranslationEvent]
# @LAYER: Domain # @RELATION DEPENDS_ON -> [MetricSnapshot]
# @RELATION: DEPENDS_ON -> [TranslationEvent:Class] # @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION: DEPENDS_ON -> [MetricSnapshot:Class] # @RELATION DEPENDS_ON -> [TranslationSchedule]
# @RELATION: DEPENDS_ON -> [TranslationRun:Class]
# @RELATION: DEPENDS_ON -> [TranslationSchedule:Class]
# @PRE: Database session is open.
# @POST: Metrics are aggregated and returned; no side effects.
# @RATIONALE: Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture.
# @REJECTED: Querying all events for all time — would be slow; MetricSnapshot provides historical summary.
from datetime import datetime, timezone from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple
@@ -26,18 +20,18 @@ from ...models.translate import (
) )
# [DEF:TranslationMetrics:Class] # #region TranslationMetrics [C:3] [TYPE Class] [SEMANTICS translate,metrics]
# @COMPLEXITY: 3 # @BRIEF Aggregate translation metrics from live events and MetricSnapshot.
# @PURPOSE: Aggregate translation metrics from live events and MetricSnapshot. # @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [MetricSnapshot]
# @RELATION DEPENDS_ON -> [TranslationEvent]
class TranslationMetrics: class TranslationMetrics:
def __init__(self, db: Session): def __init__(self, db: Session):
self.db = db self.db = db
# [DEF:get_job_metrics:Function] # #region get_job_metrics [C:2] [TYPE Function] [SEMANTICS translate,metrics,job]
# @PURPOSE: Get aggregated metrics for a specific job. # @BRIEF Get aggregated metrics for a specific job including run counts, record stats, and duration.
# @PRE: job_id exists.
# @POST: Returns dict with metrics from events + latest snapshot.
def get_job_metrics(self, job_id: str) -> Dict[str, Any]: def get_job_metrics(self, job_id: str) -> Dict[str, Any]:
with belief_scope("TranslationMetrics.get_job_metrics"): with belief_scope("TranslationMetrics.get_job_metrics"):
# Run counts from TranslationRun # Run counts from TranslationRun
@@ -151,11 +145,10 @@ class TranslationMetrics:
"last_run_at": last_run.created_at.isoformat() if last_run else None, "last_run_at": last_run.created_at.isoformat() if last_run else None,
"next_scheduled_run": next_schedule.last_run_at.isoformat() if next_schedule and next_schedule.last_run_at else None, "next_scheduled_run": next_schedule.last_run_at.isoformat() if next_schedule and next_schedule.last_run_at else None,
} }
# [/DEF:get_job_metrics:Function] # #endregion get_job_metrics
# [DEF:get_all_metrics:Function] # #region get_all_metrics [C:2] [TYPE Function] [SEMANTICS translate,metrics,all]
# @PURPOSE: Get aggregated metrics for all jobs. # @BRIEF Get aggregated metrics for all jobs.
# @POST: Returns list of per-job metrics.
def get_all_metrics(self) -> List[Dict[str, Any]]: def get_all_metrics(self) -> List[Dict[str, Any]]:
with belief_scope("TranslationMetrics.get_all_metrics"): with belief_scope("TranslationMetrics.get_all_metrics"):
job_ids = ( job_ids = (
@@ -164,8 +157,8 @@ class TranslationMetrics:
.all() .all()
) )
return [self.get_job_metrics(jid[0]) for jid in job_ids if jid[0]] return [self.get_job_metrics(jid[0]) for jid in job_ids if jid[0]]
# [/DEF:get_all_metrics:Function] # #endregion get_all_metrics
# [/DEF:TranslationMetrics:Class] # #endregion TranslationMetrics
# [/DEF:TranslationMetrics:Module] # #endregion TranslationMetrics

236
backend/src/plugins/translate/orchestrator.py
View File

@@ -1,23 +1,21 @@
# [DEF:TranslationOrchestrator:Module] # #region TranslationOrchestrator [C:5] [TYPE Module] [SEMANTICS translate,orchestrator,lifecycle,run]
# @COMPLEXITY: 5 # @BRIEF Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events.
# @SEMANTICS: translate, orchestrator, lifecycle, run # @LAYER Domain
# @PURPOSE: Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events. # @RELATION DEPENDS_ON -> [TranslationRun]
# @LAYER: Domain # @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION: DEPENDS_ON -> [TranslationRun] # @RELATION DEPENDS_ON -> [TranslationExecutor]
# @RELATION: DEPENDS_ON -> [TranslationJob] # @RELATION DEPENDS_ON -> [SQLGenerator]
# @RELATION: DEPENDS_ON -> [TranslationExecutor] # @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor]
# @RELATION: DEPENDS_ON -> [SQLGenerator] # @RELATION DEPENDS_ON -> [TranslationEventLog]
# @RELATION: DEPENDS_ON -> [SupersetSqlLabExecutor] # @RELATION DEPENDS_ON -> [TranslationPreviewSession]
# @RELATION: DEPENDS_ON -> [TranslationEventLog] # @PRE Valid job and accepted preview (for manual runs). Superset and LLM are reachable.
# @RELATION: DEPENDS_ON -> [TranslationPreviewSession] # @POST Translation run is executed, SQL generated and submitted, events recorded.
# @PRE: Valid job and accepted preview (for manual runs). Superset and LLM are reachable. # @SIDE_EFFECT Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events.
# @POST: Translation run is executed, SQL generated and submitted, events recorded. # @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun]
# @SIDE_EFFECT: Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events. # @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run.
# @DATA_CONTRACT: Input[db, config_manager, current_user] -> Output[TranslationRun] # @RATIONALE C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary.
# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run. # @REJECTED Distributed actor model (Celery) — eventual-consistency challenges at current scale.
# @RATIONALE: C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary. # @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# @REJECTED: Distributed actor model (Celery) — eventual-consistency challenges at current scale.
# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
import json import json
import time import time
@@ -44,13 +42,13 @@ from .events import TranslationEventLog
from ..translate.service import TranslateJobService from ..translate.service import TranslateJobService
# [DEF:TranslationOrchestrator:Class] # #region TranslationOrchestrator [C:5] [TYPE Class] [SEMANTICS translate,orchestrator]
# @COMPLEXITY: 5 # @BRIEF Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging.
# @PURPOSE: Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging. # @PRE DB session and config manager are available.
# @PRE: DB session and config manager are available. # @POST Runs are created, executed, and finalized with event records.
# @POST: Runs are created, executed, and finalized with event records. # @SIDE_EFFECT Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows.
# @SIDE_EFFECT: Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows. # @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED.
# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. # @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun]
class TranslationOrchestrator: class TranslationOrchestrator:
def __init__( def __init__(
@@ -65,12 +63,11 @@ class TranslationOrchestrator:
self.event_log = TranslationEventLog(db) self.event_log = TranslationEventLog(db)
self._job: Optional[TranslationJob] = None self._job: Optional[TranslationJob] = None
# [DEF:start_run:Function] # #region start_run [C:5] [TYPE Function] [SEMANTICS translate,run,create]
# @COMPLEXITY: 5 # @BRIEF Start a new translation run for a job with config snapshot and hash computation.
# @PURPOSE: Start a new translation run for a job. # @PRE job_id exists. For manual runs, there must be an accepted preview session.
# @PRE: job_id exists. For manual runs, there must be an accepted preview session. # @POST TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.
# @POST: TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded. # @SIDE_EFFECT DB writes; event logging.
# @SIDE_EFFECT: DB writes.
def start_run( def start_run(
self, self,
job_id: str, job_id: str,
@@ -166,18 +163,19 @@ class TranslationOrchestrator:
"trigger_type": run.trigger_type, "trigger_type": run.trigger_type,
}) })
return run return run
# [/DEF:start_run:Function] # #endregion start_run
# [DEF:execute_run:Function] # #region execute_run [C:5] [TYPE Function] [SEMANTICS translate,run,execute]
# @PURPOSE: Execute a translation run: dispatch executor, generate SQL, submit to Superset. # @BRIEF Execute a translation run: dispatch executor, generate SQL, submit to Superset.
# @PRE: run is in PENDING status. # @PRE run is in PENDING status.
# @POST: Run is executed, SQL generated, Superset submission attempted. # @POST Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT: LLM calls, DB writes, Superset API calls. # @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
def execute_run( def execute_run(
self, self,
run: TranslationRun, run: TranslationRun,
on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None, on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None,
skip_insert: bool = False, skip_insert: bool = False,
full_translation: bool = False,
) -> TranslationRun: ) -> TranslationRun:
with belief_scope("TranslationOrchestrator.execute_run"): with belief_scope("TranslationOrchestrator.execute_run"):
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first() job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
@@ -212,7 +210,7 @@ class TranslationOrchestrator:
on_batch_progress=on_batch_progress, on_batch_progress=on_batch_progress,
) )
try: try:
run = executor.execute_run(run, llm_progress_callback=None) run = executor.execute_run(run, llm_progress_callback=None, full_translation=full_translation)
except Exception as e: except Exception as e:
logger.explore("Translation execution failed", { logger.explore("Translation execution failed", {
"run_id": run.id, "run_id": run.id,
@@ -299,13 +297,13 @@ class TranslationOrchestrator:
"insert_status": run.insert_status, "insert_status": run.insert_status,
}) })
return run return run
# [/DEF:execute_run:Function] # #endregion execute_run
# [DEF:_generate_and_insert_sql:Function] # #region _generate_and_insert_sql [C:5] [TYPE Function] [SEMANTICS translate,sql,insert]
# @PURPOSE: Generate INSERT SQL from successful records and submit to Superset SQL Lab. # @BRIEF Generate INSERT SQL from successful records and submit to Superset SQL Lab.
# @PRE: job has target table configured. run has successful records. # @PRE job has target table configured. run has successful records.
# @POST: SQL is generated and submitted. Returns execution result. # @POST SQL is generated and submitted. Returns execution result.
# @SIDE_EFFECT: Superset API call. # @SIDE_EFFECT Superset API call; event logging.
def _generate_and_insert_sql( def _generate_and_insert_sql(
self, self,
job: TranslationJob, job: TranslationJob,
@@ -333,11 +331,12 @@ class TranslationOrchestrator:
}) })
# Build rows for SQL generation # Build rows for SQL generation
columns = job.context_columns or [] # Only include the translation column — context columns are for LLM only
if job.translation_column and job.translation_column not in columns: columns = []
if job.translation_column:
columns.append(job.translation_column) columns.append(job.translation_column)
# Also include key columns if used for upsert # Include key columns if present (for UPSERT matching)
if job.target_key_cols: if job.target_key_cols:
for k in job.target_key_cols: for k in job.target_key_cols:
if k not in columns: if k not in columns:
@@ -346,14 +345,13 @@ class TranslationOrchestrator:
rows_for_sql = [] rows_for_sql = []
for rec in records: for rec in records:
row_data = {} row_data = {}
if job.context_columns:
for col in job.context_columns:
row_data[col] = ""
if job.translation_column: if job.translation_column:
row_data[job.translation_column] = rec.target_sql or "" row_data[job.translation_column] = rec.target_sql or ""
if job.target_key_cols: if job.target_key_cols:
source_data = rec.source_data or {}
for k in job.target_key_cols: for k in job.target_key_cols:
row_data[k] = "" # Use source_data value if available, fall back to empty string
row_data[k] = source_data.get(k, "")
rows_for_sql.append(row_data) rows_for_sql.append(row_data)
if not columns: if not columns:
@@ -377,6 +375,16 @@ class TranslationOrchestrator:
return {"status": "failed", "error_message": str(e), "query_id": None} return {"status": "failed", "error_message": str(e), "query_id": None}
# Log insert phase start # Log insert phase start
logger.reason("Insert SQL generated", {
"run_id": run.id,
"sql_preview": sql[:500],
"sql_length": len(sql),
"row_count": row_count,
"dialect": job.database_dialect or job.target_dialect or "postgresql",
"columns": columns,
"has_key_cols": bool(job.target_key_cols),
})
self.event_log.log_event( self.event_log.log_event(
job_id=job.id, job_id=job.id,
run_id=run.id, run_id=run.id,
@@ -388,7 +396,18 @@ class TranslationOrchestrator:
# Submit to Superset # Submit to Superset
try: try:
env_id = job.environment_id or job.source_dialect or "" env_id = job.environment_id or job.source_dialect or ""
executor = SupersetSqlLabExecutor(self.config_manager, env_id) # Resolve database_id: explicit target_database_id from job (int or UUID), or resolve from environment
target_db_id = None
if job.target_database_id:
try:
target_db_id = int(job.target_database_id)
except (ValueError, TypeError):
# Could be a UUID — pass as-is, executor will resolve it
target_db_id = job.target_database_id
logger.reason("target_database_id is not an integer, will resolve as UUID", {
"target_database_id": job.target_database_id,
})
executor = SupersetSqlLabExecutor(self.config_manager, env_id, database_id=target_db_id)
result = executor.execute_and_poll( result = executor.execute_and_poll(
sql=sql, sql=sql,
max_polls=30, max_polls=30,
@@ -411,13 +430,13 @@ class TranslationOrchestrator:
) )
return result return result
# [/DEF:_generate_and_insert_sql:Function] # #endregion _generate_and_insert_sql
# [DEF:_validate_preconditions:Function] # #region _validate_preconditions [C:5] [TYPE Function] [SEMANTICS translate,validation]
# @PURPOSE: Validate preconditions before starting a run. # @BRIEF Validate preconditions before starting a translation run.
# @PRE: None. # @PRE None.
# @POST: Raises ValueError if preconditions are not met. # @POST Raises ValueError if preconditions are not met.
# @SIDE_EFFECT: None. # @SIDE_EFFECT None.
def _validate_preconditions( def _validate_preconditions(
self, self,
job: TranslationJob, job: TranslationJob,
@@ -473,13 +492,13 @@ class TranslationOrchestrator:
"job_id": job.id, "job_id": job.id,
"is_scheduled": is_scheduled, "is_scheduled": is_scheduled,
}) })
# [/DEF:_validate_preconditions:Function] # #endregion _validate_preconditions
# [DEF:retry_failed_batches:Function] # #region retry_failed_batches [C:5] [TYPE Function] [SEMANTICS translate,retry,batch]
# @PURPOSE: Retry failed batches in a run. # @BRIEF Retry failed batches in a run by re-processing failed records.
# @PRE: run exists and has failed batches. # @PRE run exists and has failed batches.
# @POST: Failed batches are re-processed. # @POST Failed batches are re-processed; run status and stats updated.
# @SIDE_EFFECT: LLM calls, DB writes. # @SIDE_EFFECT LLM calls; DB writes; event logging.
def retry_failed_batches( def retry_failed_batches(
self, self,
run_id: str, run_id: str,
@@ -588,13 +607,13 @@ class TranslationOrchestrator:
"status": run.status, "status": run.status,
}) })
return run return run
# [/DEF:retry_failed_batches:Function] # #endregion retry_failed_batches
# [DEF:retry_insert:Function] # #region retry_insert [C:5] [TYPE Function] [SEMANTICS translate,retry,insert]
# @PURPOSE: Retry the SQL insert phase for a completed run. # @BRIEF Retry the SQL insert phase for a completed run.
# @PRE: run exists and has successful records. # @PRE run exists and has successful records.
# @POST: SQL is regenerated and re-submitted to Superset. # @POST SQL is regenerated and re-submitted to Superset.
# @SIDE_EFFECT: Superset API call. # @SIDE_EFFECT Superset API call; event logging.
def retry_insert( def retry_insert(
self, self,
run_id: str, run_id: str,
@@ -638,13 +657,13 @@ class TranslationOrchestrator:
"insert_status": run.insert_status, "insert_status": run.insert_status,
}) })
return run return run
# [/DEF:retry_insert:Function] # #endregion retry_insert
# [DEF:cancel_run:Function] # #region cancel_run [C:5] [TYPE Function] [SEMANTICS translate,run,cancel]
# @PURPOSE: Cancel a running translation. # @BRIEF Cancel a running translation run.
# @PRE: run is in PENDING or RUNNING status. # @PRE run is in PENDING or RUNNING status.
# @POST: Run status is set to CANCELLED. # @POST Run status is set to CANCELLED; event recorded.
# @SIDE_EFFECT: DB write; records event. # @SIDE_EFFECT DB write; records event.
def cancel_run(self, run_id: str) -> TranslationRun: def cancel_run(self, run_id: str) -> TranslationRun:
with belief_scope("TranslationOrchestrator.cancel_run"): with belief_scope("TranslationOrchestrator.cancel_run"):
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first() run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
@@ -661,7 +680,8 @@ class TranslationOrchestrator:
run.completed_at = datetime.now(timezone.utc) run.completed_at = datetime.now(timezone.utc)
self.db.flush() self.db.flush()
self.event_log.log_event( # Record terminal event directly — we are the source of the terminal event
self.event_log._create_event_raw(
job_id=run.job_id, job_id=run.job_id,
run_id=run.id, run_id=run.id,
event_type="RUN_CANCELLED", event_type="RUN_CANCELLED",
@@ -676,12 +696,12 @@ class TranslationOrchestrator:
"run_id": run_id, "run_id": run_id,
}) })
return run return run
# [/DEF:cancel_run:Function] # #endregion cancel_run
# [DEF:get_run_status:Function] # #region get_run_status [C:5] [TYPE Function] [SEMANTICS translate,run,status]
# @PURPOSE: Get run status with statistics. # @BRIEF Get run status with statistics and event invariant checks.
# @PRE: run_id exists. # @PRE run_id exists.
# @POST: Returns dict with run details. # @POST Returns dict with run details.
def get_run_status(self, run_id: str) -> Dict[str, Any]: def get_run_status(self, run_id: str) -> Dict[str, Any]:
with belief_scope("TranslationOrchestrator.get_run_status"): with belief_scope("TranslationOrchestrator.get_run_status"):
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first() run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
@@ -695,8 +715,8 @@ class TranslationOrchestrator:
.count() .count()
) )
# Get event summary # Get event invariants (lightweight — only fetches event_type column)
event_summary = self.event_log.get_run_event_summary(run_id) invariants = self.event_log.get_run_event_invariants_lightweight(run_id)
return { return {
"id": run.id, "id": run.id,
@@ -712,20 +732,16 @@ class TranslationOrchestrator:
"insert_status": run.insert_status, "insert_status": run.insert_status,
"superset_execution_id": run.superset_execution_id, "superset_execution_id": run.superset_execution_id,
"batch_count": batch_count, "batch_count": batch_count,
"event_invariants": { "event_invariants": invariants,
"has_run_started": event_summary["has_run_started"],
"terminal_event_count": event_summary["terminal_event_count"],
"invariant_valid": event_summary["invariant_valid"],
},
"created_by": run.created_by, "created_by": run.created_by,
"created_at": run.created_at.isoformat() if run.created_at else None, "created_at": run.created_at.isoformat() if run.created_at else None,
} }
# [/DEF:get_run_status:Function] # #endregion get_run_status
# [DEF:get_run_records:Function] # #region get_run_records [C:5] [TYPE Function] [SEMANTICS translate,run,records]
# @PURPOSE: Get paginated records for a run. # @BRIEF Get paginated records for a run with optional status filter.
# @PRE: run_id exists. # @PRE run_id exists.
# @POST: Returns dict with records and pagination info. # @POST Returns dict with records and pagination info.
def get_run_records( def get_run_records(
self, self,
run_id: str, run_id: str,
@@ -771,12 +787,12 @@ class TranslationOrchestrator:
"page_size": page_size, "page_size": page_size,
"status_filter": status_filter, "status_filter": status_filter,
} }
# [/DEF:get_run_records:Function] # #endregion get_run_records
# [DEF:get_run_history:Function] # #region get_run_history [C:5] [TYPE Function] [SEMANTICS translate,run,history]
# @PURPOSE: Get run history for a job. # @BRIEF Get paginated run history for a job.
# @PRE: job_id exists. # @PRE job_id exists.
# @POST: Returns list of runs. # @POST Returns tuple of (total_count, list_of_runs).
def get_run_history( def get_run_history(
self, self,
job_id: str, job_id: str,
@@ -814,10 +830,10 @@ class TranslationOrchestrator:
} }
for r in runs for r in runs
] ]
# [/DEF:get_run_history:Function] # #endregion get_run_history
# [DEF:_compute_config_hash:Function] # #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison. # @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison.
@staticmethod @staticmethod
def _compute_config_hash(job: TranslationJob) -> str: def _compute_config_hash(job: TranslationJob) -> str:
import hashlib import hashlib
@@ -833,10 +849,10 @@ class TranslationOrchestrator:
"upsert_strategy": job.upsert_strategy, "upsert_strategy": job.upsert_strategy,
}, sort_keys=True) }, sort_keys=True)
return hashlib.sha256(config_str.encode()).hexdigest()[:16] return hashlib.sha256(config_str.encode()).hexdigest()[:16]
# [/DEF:_compute_config_hash:Function] # #endregion _compute_config_hash
# [DEF:_compute_dict_snapshot_hash:Function] # #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @PURPOSE: Compute a hash of dictionary state for snapshot comparison. # @BRIEF Compute a SHA-256 hash of dictionary state for snapshot comparison.
def _compute_dict_snapshot_hash(self, job_id: str) -> str: def _compute_dict_snapshot_hash(self, job_id: str) -> str:
import hashlib import hashlib
from ...models.translate import TranslationJobDictionary from ...models.translate import TranslationJobDictionary
@@ -848,8 +864,8 @@ class TranslationOrchestrator:
dict_ids = sorted([dl.dictionary_id for dl in dict_links]) dict_ids = sorted([dl.dictionary_id for dl in dict_links])
hash_input = ",".join(dict_ids) hash_input = ",".join(dict_ids)
return hashlib.sha256(hash_input.encode()).hexdigest()[:16] return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
# [/DEF:_compute_dict_snapshot_hash:Function] # #endregion _compute_dict_snapshot_hash
# [/DEF:TranslationOrchestrator:Class] # #endregion TranslationOrchestrator
# [/DEF:TranslationOrchestrator:Module] # #endregion TranslationOrchestrator

22
backend/src/plugins/translate/plugin.py
View File

@@ -1,19 +1,15 @@
# [DEF:TranslatePlugin:Module] # #region TranslatePlugin [C:3] [TYPE Module] [SEMANTICS plugin,translate,llm,sql,dashboard]
# @COMPLEXITY: 2 # @BRIEF TranslatePlugin skeleton for cross-dialect SQL and dashboard translation.
# @SEMANTICS: plugin, translate, llm, sql, dashboard # @LAYER Domain
# @PURPOSE: TranslatePlugin skeleton for cross-dialect SQL and dashboard translation. # @RELATION INHERITS -> [PluginBase]
# @LAYER: Domain
# @RELATION: INHERITS -> [PluginBase]
# @RATIONALE: Separate plugin avoids bloating LLMAnalysisPlugin beyond fractal limit (<400 lines).
# @REJECTED: Extending LLMAnalysisPlugin would conflate two distinct feature domains.
from typing import Dict, Any, Optional from typing import Dict, Any, Optional
from ...core.plugin_base import PluginBase from ...core.plugin_base import PluginBase
# [DEF:TranslatePlugin:Class] # #region TranslatePlugin [C:3] [TYPE Class] [SEMANTICS plugin,translate]
# @PURPOSE: Plugin for translating SQL queries and dashboard definitions across database dialects. # @BRIEF Plugin for translating SQL queries and dashboard definitions across database dialects.
# @RELATION: IMPLEMENTS -> backend.src.core.plugin_base.PluginBase # @RELATION IMPLEMENTS -> [PluginBase]
class TranslatePlugin(PluginBase): class TranslatePlugin(PluginBase):
@property @property
def id(self) -> str: def id(self) -> str:
@@ -57,6 +53,6 @@ class TranslatePlugin(PluginBase):
async def execute(self, params: Dict[str, Any], context: Optional[Any] = None): async def execute(self, params: Dict[str, Any], context: Optional[Any] = None):
"""Execute a translation job — implementation deferred to later phases.""" """Execute a translation job — implementation deferred to later phases."""
raise NotImplementedError("TranslatePlugin.execute not yet implemented") raise NotImplementedError("TranslatePlugin.execute not yet implemented")
# [/DEF:TranslatePlugin:Class] # #endregion TranslatePlugin
# [/DEF:TranslatePlugin:Module] # #endregion TranslatePlugin

209
backend/src/plugins/translate/preview.py
View File

@@ -1,21 +1,21 @@
# [DEF:TranslationPreview:Module] # #region TranslationPreview [C:5] [TYPE Module] [SEMANTICS translate,preview,llm,session]
# @COMPLEXITY: 4 # @BRIEF Preview session management: fetch sample rows from Superset, send to LLM with context + filtered dictionary, return side-by-side results.
# @SEMANTICS: translate, preview, llm, session # @LAYER Domain
# @PURPOSE: Preview session management: fetch sample rows from Superset, send to LLM with context + filtered dictionary, return side-by-side results. # @RELATION DEPENDS_ON -> [TranslationJob]
# @LAYER: Domain # @RELATION DEPENDS_ON -> [TranslationPreviewSession]
# @RELATION: DEPENDS_ON -> [TranslationJob:Class] # @RELATION DEPENDS_ON -> [TranslationPreviewRecord]
# @RELATION: DEPENDS_ON -> [TranslationPreviewSession:Class] # @RELATION DEPENDS_ON -> [LLMProviderService]
# @RELATION: DEPENDS_ON -> [TranslationPreviewRecord:Class] # @RELATION DEPENDS_ON -> [DictionaryManager]
# @RELATION: DEPENDS_ON -> [LLMProviderService] # @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION: DEPENDS_ON -> [DictionaryManager] # @RELATION DEPENDS_ON -> [render_prompt]
# @RELATION: DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [ConfigManager]
# @RELATION: DEPENDS_ON -> [render_prompt] # @PRE Database session and config manager are available.
# @RELATION: DEPENDS_ON -> [ConfigManager] # @POST Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected.
# @PRE: Database session and config manager are available. # @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates DB rows.
# @POST: Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected. # @DATA_CONTRACT Input[job_id, sample_size] -> Output[Dict with session, records, cost_estimate]
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows. # @INVARIANT Preview sessions expire after 24 hours; exactly one ACTIVE session per job at a time.
# @RATIONALE: C4 because preview is stateful with approve/edit/reject lifecycle and LLM API calls. # @RATIONALE C5 because preview is stateful with approve/edit/reject lifecycle and LLM API calls requiring structured error handling.
# @REJECTED: Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably. # @REJECTED Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably.
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -38,8 +38,7 @@ from ...services.llm_provider import LLMProviderService
from ...services.llm_prompt_templates import render_prompt from ...services.llm_prompt_templates import render_prompt
from .dictionary import DictionaryManager from .dictionary import DictionaryManager
# [DEF:DEFAULT_EXECUTION_PROMPT_TEMPLATE:Constant] # #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [C:1] [TYPE Constant] [SEMANTICS translate,prompt]
# @PURPOSE: Default prompt template for batch LLM translation execution (no context columns — faster).
DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = ( DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
"Translate the following database content from {source_language} to {target_language}.\n\n" "Translate the following database content from {source_language} to {target_language}.\n\n"
"Source dialect: {source_dialect}\n" "Source dialect: {source_dialect}\n"
@@ -52,11 +51,10 @@ DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n' '{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
"Each row_id must match the index provided. Return exactly {row_count} entries." "Each row_id must match the index provided. Return exactly {row_count} entries."
) )
# [/DEF:DEFAULT_EXECUTION_PROMPT_TEMPLATE:Constant] # #endregion DEFAULT_EXECUTION_PROMPT_TEMPLATE
# [DEF:DEFAULT_PREVIEW_PROMPT_TEMPLATE:Constant] # #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [C:1] [TYPE Constant] [SEMANTICS translate,prompt]
# @PURPOSE: Default prompt template for LLM translation preview.
DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = ( DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
"Translate the following database content from {source_language} to {target_language}.\n\n" "Translate the following database content from {source_language} to {target_language}.\n\n"
"Source dialect: {source_dialect}\n" "Source dialect: {source_dialect}\n"
@@ -70,13 +68,11 @@ DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n' '{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
"Each row_id must match the index provided. Return exactly {row_count} entries." "Each row_id must match the index provided. Return exactly {row_count} entries."
) )
# [/DEF:DEFAULT_PREVIEW_PROMPT_TEMPLATE:Constant] # #endregion DEFAULT_PREVIEW_PROMPT_TEMPLATE
# [DEF:TokenEstimator:Class] # #region TokenEstimator [C:2] [TYPE Class] [SEMANTICS translate,tokens,estimate]
# @PURPOSE: Estimate token counts and costs for LLM translation operations. # @BRIEF Estimate token counts and costs for LLM translation operations using heuristic chars/token ratio.
# @RATIONALE: Token estimation uses a heuristic (chars/token ratio) since exact tokenization depends on the LLM model.
# @REJECTED: Using an external tokenizer library would introduce a heavy dependency for estimation only.
class TokenEstimator: class TokenEstimator:
"""Estimate token counts and costs for LLM operations.""" """Estimate token counts and costs for LLM operations."""
@@ -84,46 +80,38 @@ class TokenEstimator:
OUTPUT_TOKENS_PER_ROW_ESTIMATE: int = 50 OUTPUT_TOKENS_PER_ROW_ESTIMATE: int = 50
TOKEN_COST_PER_1K: float = 0.002 # Default cost per 1K tokens TOKEN_COST_PER_1K: float = 0.002 # Default cost per 1K tokens
# [DEF:estimate_prompt_tokens:Function] # #region estimate_prompt_tokens [C:1] [TYPE Function] [SEMANTICS translate,tokens]
# @PURPOSE: Estimate token count for a prompt string.
# @PRE: prompt is a non-empty string.
# @POST: Returns estimated token count (integer).
@staticmethod @staticmethod
def estimate_prompt_tokens(prompt: str) -> int: def estimate_prompt_tokens(prompt: str) -> int:
if not prompt: if not prompt:
return 0 return 0
return max(1, int(len(prompt) / TokenEstimator.CHARS_PER_TOKEN_ESTIMATE)) return max(1, int(len(prompt) / TokenEstimator.CHARS_PER_TOKEN_ESTIMATE))
# [/DEF:estimate_prompt_tokens:Function] # #endregion estimate_prompt_tokens
# [DEF:estimate_output_tokens:Function] # #region estimate_output_tokens [C:1] [TYPE Function] [SEMANTICS translate,tokens]
# @PURPOSE: Estimate output token count for translating N rows.
# @PRE: row_count >= 0.
# @POST: Returns estimated output token count.
@staticmethod @staticmethod
def estimate_output_tokens(row_count: int) -> int: def estimate_output_tokens(row_count: int) -> int:
return row_count * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE return row_count * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE
# [/DEF:estimate_output_tokens:Function] # #endregion estimate_output_tokens
# [DEF:estimate_cost:Function] # #region estimate_cost [C:1] [TYPE Function] [SEMANTICS translate,cost]
# @PURPOSE: Estimate cost for a given number of tokens.
# @PRE: total_tokens >= 0.
# @POST: Returns estimated cost in USD.
@staticmethod @staticmethod
def estimate_cost(total_tokens: int, cost_per_1k: Optional[float] = None) -> float: def estimate_cost(total_tokens: int, cost_per_1k: Optional[float] = None) -> float:
rate = cost_per_1k if cost_per_1k is not None else TokenEstimator.TOKEN_COST_PER_1K rate = cost_per_1k if cost_per_1k is not None else TokenEstimator.TOKEN_COST_PER_1K
return round((total_tokens / 1000) * rate, 6) return round((total_tokens / 1000) * rate, 6)
# [/DEF:estimate_cost:Function] # #endregion estimate_cost
# [/DEF:TokenEstimator:Class] # #endregion TokenEstimator
# [DEF:TranslationPreview:Class] # #region TranslationPreview [C:5] [TYPE Class] [SEMANTICS translate,preview,lifecycle]
# @COMPLEXITY: 4 # @BRIEF Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate.
# @PURPOSE: Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate. # @PRE Database session and config manager are available.
# @PRE: Database session and config manager are available. # @POST Preview sessions created with persisted records; full execution gates on accepted session.
# @POST: Preview sessions created with persisted records; full execution gates on accepted session. # @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates DB rows.
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows. # @DATA_CONTRACT Input[job_id, sample_size] -> Output[Dict with session, records, cost_estimate]
# @INVARIANT Preview sessions expire after 24 hours; exactly one ACTIVE session per job at a time.
class TranslationPreview: class TranslationPreview:
def __init__( def __init__(
@@ -136,12 +124,11 @@ class TranslationPreview:
self.config_manager = config_manager self.config_manager = config_manager
self.current_user = current_user self.current_user = current_user
# [DEF:preview_rows:Function] # #region preview_rows [C:4] [TYPE Function] [SEMANTICS translate,preview,rows]
# @COMPLEXITY: 4 # @BRIEF Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.
# @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records. # @PRE job_id exists and job has source_datasource_id, translation_column configured.
# @PRE: job_id exists and job has source_datasource_id, translation_column configured. # @POST Returns TranslationPreviewResponse with records, cost estimation, and persistent session.
# @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session. # @SIDE_EFFECT Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.
# @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.
def preview_rows( def preview_rows(
self, self,
job_id: str, job_id: str,
@@ -303,6 +290,7 @@ class TranslationPreview:
source_object_name=f"Row {idx + 1}", source_object_name=f"Row {idx + 1}",
status=status, status=status,
feedback=feedback, feedback=feedback,
source_data=meta.get("context_data"),
created_at=datetime.now(timezone.utc), created_at=datetime.now(timezone.utc),
) )
self.db.add(record) self.db.add(record)
@@ -348,12 +336,13 @@ class TranslationPreview:
"sample_cost": sample_cost, "sample_cost": sample_cost,
}) })
return result return result
# [/DEF:preview_rows:Function] # #endregion preview_rows
# [DEF:update_preview_row:Function] # #region update_preview_row [C:4] [TYPE Function] [SEMANTICS translate,preview,row,update]
# @PURPOSE: Approve, edit, or reject an individual preview row. # @BRIEF Approve, edit, or reject an individual preview row.
# @PRE: session_id and row_id exist, session is ACTIVE. # @PRE session_id and row_id exist, session is ACTIVE.
# @POST: PreviewRecord status is updated. # @POST PreviewRecord status is updated (APPROVED, REJECTED, or APPROVED with edited translation).
# @SIDE_EFFECT DB write.
def update_preview_row( def update_preview_row(
self, self,
job_id: str, job_id: str,
@@ -417,13 +406,13 @@ class TranslationPreview:
"status": record.status, "status": record.status,
"feedback": record.feedback, "feedback": record.feedback,
} }
# [/DEF:update_preview_row:Function] # #endregion update_preview_row
# [DEF:accept_preview_session:Function] # #region accept_preview_session [C:4] [TYPE Function] [SEMANTICS translate,preview,accept]
# @PURPOSE: Mark a preview session as accepted, which gates full execution. # @BRIEF Mark a preview session as accepted (APPLIED), which gates full execution.
# @PRE: job_id has an ACTIVE preview session. # @PRE job_id has an ACTIVE preview session.
# @POST: Session status changes to APPLIED. # @POST Session status changes to APPLIED; future full execution calls check for accepted session.
# @SIDE_EFFECT: Future full execution calls will check for accepted session. # @SIDE_EFFECT DB write.
def accept_preview_session(self, job_id: str) -> Dict[str, Any]: def accept_preview_session(self, job_id: str) -> Dict[str, Any]:
with belief_scope("TranslationPreview.accept_preview_session"): with belief_scope("TranslationPreview.accept_preview_session"):
session = ( session = (
@@ -472,12 +461,11 @@ class TranslationPreview:
for r in records for r in records
], ],
} }
# [/DEF:accept_preview_session:Function] # #endregion accept_preview_session
# [DEF:get_preview_session:Function] # #region get_preview_session [C:3] [TYPE Function] [SEMANTICS translate,preview,get]
# @PURPOSE: Get the latest preview session for a job with its records. # @BRIEF Get the latest preview session for a job with its records.
# @PRE: job_id exists. # @RELATION DEPENDS_ON -> [TranslationPreviewSession]
# @POST: Returns session data with records or raises ValueError.
def get_preview_session(self, job_id: str) -> Dict[str, Any]: def get_preview_session(self, job_id: str) -> Dict[str, Any]:
with belief_scope("TranslationPreview.get_preview_session"): with belief_scope("TranslationPreview.get_preview_session"):
session = ( session = (
@@ -516,14 +504,13 @@ class TranslationPreview:
for r in records for r in records
], ],
} }
# [/DEF:get_preview_session:Function] # #endregion get_preview_session
# [DEF:_fetch_sample_rows:Function] # #region _fetch_sample_rows [C:4] [TYPE Function] [SEMANTICS translate,superset,sample]
# @COMPLEXITY: 4 # @BRIEF Fetch sample rows from the Superset dataset for preview.
# @PURPOSE: Fetch sample rows from the Superset dataset for preview. # @PRE job has source_datasource_id and translation_column.
# @PRE: job has source_datasource_id and translation_column. # @POST Returns list of dicts with row data from Superset chart data API.
# @POST: Returns list of dicts with row data. # @SIDE_EFFECT Calls Superset chart data endpoint with pagination.
# @SIDE_EFFECT: Calls Superset chart data endpoint.
def _fetch_sample_rows(self, job: TranslationJob, sample_size: int = 10, env_id: Optional[str] = None) -> List[Dict[str, Any]]: def _fetch_sample_rows(self, job: TranslationJob, sample_size: int = 10, env_id: Optional[str] = None) -> List[Dict[str, Any]]:
with belief_scope("TranslationPreview._fetch_sample_rows"): with belief_scope("TranslationPreview._fetch_sample_rows"):
# Determine environment: prefer explicit env_id, then job.environment_id, then job.source_dialect (legacy) # Determine environment: prefer explicit env_id, then job.environment_id, then job.source_dialect (legacy)
@@ -602,12 +589,11 @@ class TranslationPreview:
) )
return rows return rows
# [/DEF:_fetch_sample_rows:Function] # #endregion _fetch_sample_rows
# [DEF:_extract_data_rows:Function] # #region _extract_data_rows [C:3] [TYPE Function] [SEMANTICS translate,superset,data]
# @PURPOSE: Extract data rows from Superset chart data response. # @BRIEF Extract data rows from Superset chart data response, trying multiple response formats.
# @PRE: response is a dict from Superset API. # @RELATION DEPENDS_ON -> [SupersetClient]
# @POST: Returns list of row dicts.
@staticmethod @staticmethod
def _extract_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]: def _extract_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]:
with belief_scope("TranslationPreview._extract_data_rows"): with belief_scope("TranslationPreview._extract_data_rows"):
@@ -636,14 +622,13 @@ class TranslationPreview:
return result return result
return [] return []
# [/DEF:_extract_data_rows:Function] # #endregion _extract_data_rows
# [DEF:_call_llm:Function] # #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call]
# @COMPLEXITY: 4 # @BRIEF Call the configured LLM provider with the preview prompt.
# @PURPOSE: Call the configured LLM provider with the preview prompt. # @PRE job has a valid provider_id.
# @PRE: job has a valid provider_id. # @POST Returns raw LLM response string.
# @POST: Returns raw LLM response string. # @SIDE_EFFECT Makes HTTP call to LLM provider.
# @SIDE_EFFECT: Makes HTTP call to LLM provider.
def _call_llm(self, job: TranslationJob, prompt: str) -> str: def _call_llm(self, job: TranslationJob, prompt: str) -> str:
with belief_scope("TranslationPreview._call_llm"): with belief_scope("TranslationPreview._call_llm"):
if not job.provider_id: if not job.provider_id:
@@ -679,13 +664,13 @@ class TranslationPreview:
"response_length": len(response_text), "response_length": len(response_text),
}) })
return response_text return response_text
# [/DEF:_call_llm:Function] # #endregion _call_llm
# [DEF:_call_openai_compatible:Function] # #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai]
# @PURPOSE: Call an OpenAI-compatible API for translation. # @BRIEF Call an OpenAI-compatible API for translation with structured output support.
# @PRE: base_url, api_key, model, prompt are valid. # @PRE base_url, api_key, model, prompt are valid.
# @POST: Returns response text. # @POST Returns response text.
# @SIDE_EFFECT: Makes HTTP POST to LLM API. # @SIDE_EFFECT Makes HTTP POST to LLM API.
@staticmethod @staticmethod
def _call_openai_compatible( def _call_openai_compatible(
base_url: str, base_url: str,
@@ -741,12 +726,13 @@ class TranslationPreview:
raise ValueError("LLM returned empty content") raise ValueError("LLM returned empty content")
return content return content
# [/DEF:_call_openai_compatible:Function] # #endregion _call_openai_compatible
# [DEF:_parse_llm_response:Function] # #region _parse_llm_response [C:4] [TYPE Function] [SEMANTICS translate,llm,parse]
# @PURPOSE: Parse the LLM JSON response into a dict of row_id -> translation. # @BRIEF Parse the LLM JSON response into a dict of row_id -> translation, with markdown code block fallback.
# @PRE: response_text is valid JSON with {"rows": [...]} structure. # @PRE response_text is valid JSON with {"rows": [...]} structure.
# @POST: Returns dict mapping string row_id to translation text. # @POST Returns dict mapping string row_id to translation text; warns if fewer translations than expected.
# @SIDE_EFFECT Logs warnings for short responses.
@staticmethod @staticmethod
def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]: def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:
with belief_scope("TranslationPreview._parse_llm_response"): with belief_scope("TranslationPreview._parse_llm_response"):
@@ -785,10 +771,10 @@ class TranslationPreview:
) )
return translations return translations
# [/DEF:_parse_llm_response:Function] # #endregion _parse_llm_response
# [DEF:_compute_config_hash:Function] # #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison. # @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison.
@staticmethod @staticmethod
def _compute_config_hash(job: TranslationJob) -> str: def _compute_config_hash(job: TranslationJob) -> str:
config_str = json.dumps({ config_str = json.dumps({
@@ -803,10 +789,10 @@ class TranslationPreview:
"upsert_strategy": job.upsert_strategy, "upsert_strategy": job.upsert_strategy,
}, sort_keys=True) }, sort_keys=True)
return hashlib.sha256(config_str.encode()).hexdigest()[:16] return hashlib.sha256(config_str.encode()).hexdigest()[:16]
# [/DEF:_compute_config_hash:Function] # #endregion _compute_config_hash
# [DEF:_compute_dict_snapshot_hash:Function] # #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @PURPOSE: Compute a hash of the dictionary state for snapshot comparison. # @BRIEF Compute a SHA-256 hash of the dictionary state for snapshot comparison.
def _compute_dict_snapshot_hash(self, job_id: str) -> str: def _compute_dict_snapshot_hash(self, job_id: str) -> str:
dict_links = ( dict_links = (
self.db.query(TranslationJobDictionary) self.db.query(TranslationJobDictionary)
@@ -816,9 +802,8 @@ class TranslationPreview:
dict_ids = sorted([dl.dictionary_id for dl in dict_links]) dict_ids = sorted([dl.dictionary_id for dl in dict_links])
hash_input = ",".join(dict_ids) hash_input = ",".join(dict_ids)
return hashlib.sha256(hash_input.encode()).hexdigest()[:16] return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
# [/DEF:_compute_dict_snapshot_hash:Function] # #endregion _compute_dict_snapshot_hash
# [/DEF:TranslationPreview:Class] # #endregion TranslationPreview
# #endregion TranslationPreview
# [/DEF:TranslationPreview:Module]

128
backend/src/plugins/translate/scheduler.py
View File

@@ -1,18 +1,18 @@
# [DEF:TranslationScheduler:Module] # #region TranslationScheduler [C:5] [TYPE Module] [SEMANTICS translate,scheduler,apscheduler,cron]
# @COMPLEXITY: 4 # @BRIEF Manage TranslationSchedule rows and register them with core SchedulerService for cron-based execution.
# @SEMANTICS: translate, scheduler, apscheduler, cron # @LAYER Domain
# @PURPOSE: Manage TranslationSchedule rows and register them with core SchedulerService. # @RELATION DEPENDS_ON -> [TranslationSchedule]
# @LAYER: Domain # @RELATION DEPENDS_ON -> [SchedulerService]
# @RELATION: DEPENDS_ON -> [TranslationSchedule:Class] # @RELATION DEPENDS_ON -> [TranslationOrchestrator]
# @RELATION: DEPENDS_ON -> [SchedulerService:Class] # @RELATION DEPENDS_ON -> [TranslationEventLog]
# @RELATION: DEPENDS_ON -> [TranslationOrchestrator:Class] # @PRE Database session and SchedulerService are available.
# @RELATION: DEPENDS_ON -> [TranslationEventLog:Class] # @POST TranslationSchedule CRUD persisted; translation runs triggered on schedule.
# @PRE: Database session and SchedulerService are available. # @SIDE_EFFECT Registers APScheduler jobs; runs translations on trigger; creates events.
# @POST: TranslationSchedule CRUD persisted; APScheduler jobs registered/updated/removed. # @DATA_CONTRACT Input[job_id, cron_expression] -> Output[TranslationSchedule]
# @SIDE_EFFECT: Registers APScheduler jobs; runs translations on trigger; creates events. # @INVARIANT Concurrency guard: max 1 pending/running run per job at scheduled trigger time.
# @RATIONALE: Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance. # @RATIONALE Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance.
# @REJECTED: Separate scheduler instance would create resource contention. # @REJECTED Separate scheduler instance would create resource contention.
# @REJECTED: Polling-based approach — event-driven APScheduler is more precise. # @REJECTED Polling-based approach — event-driven APScheduler is more precise.
import uuid import uuid
from datetime import datetime, timezone, timedelta from datetime import datetime, timezone, timedelta
@@ -26,9 +26,13 @@ from ...models.translate import TranslationSchedule, TranslationJob, Translation
from .events import TranslationEventLog from .events import TranslationEventLog
# [DEF:TranslationScheduler:Class] # #region TranslationScheduler [C:5] [TYPE Class] [SEMANTICS translate,scheduler,crud]
# @COMPLEXITY: 4 # @BRIEF CRUD for TranslationSchedule rows + APScheduler registration wrappers.
# @PURPOSE: CRUD for TranslationSchedule rows + APScheduler registration wrappers. # @PRE Database session and ConfigManager are available.
# @POST Schedule rows created/updated/deleted with event logging.
# @SIDE_EFFECT Writes TranslationSchedule rows; logs SCHEDULE_CREATED/UPDATED/DELETED events.
# @DATA_CONTRACT Input[job_id, cron_expression, timezone] -> Output[TranslationSchedule]
# @INVARIANT Exactly one schedule per job (upsert pattern).
class TranslationScheduler: class TranslationScheduler:
def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None): def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None):
@@ -37,10 +41,11 @@ class TranslationScheduler:
self.current_user = current_user self.current_user = current_user
self.event_log = TranslationEventLog(db) self.event_log = TranslationEventLog(db)
# [DEF:create_schedule:Function] # #region create_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,create]
# @PURPOSE: Create a new schedule for a job. # @BRIEF Create a new schedule for a job with cron expression and event logging.
# @PRE: job_id exists. cron_expression is valid. # @PRE job_id exists. cron_expression is valid.
# @POST: TranslationSchedule row created. # @POST TranslationSchedule row created; SCHEDULE_CREATED event logged.
# @SIDE_EFFECT DB write; event logging.
def create_schedule( def create_schedule(
self, self,
job_id: str, job_id: str,
@@ -79,12 +84,13 @@ class TranslationScheduler:
logger.reflect("Schedule created", {"schedule_id": schedule.id, "job_id": job_id}) logger.reflect("Schedule created", {"schedule_id": schedule.id, "job_id": job_id})
return schedule return schedule
# [/DEF:create_schedule:Function] # #endregion create_schedule
# [DEF:update_schedule:Function] # #region update_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,update]
# @PURPOSE: Update an existing schedule. # @BRIEF Update an existing schedule's cron expression, timezone, or active state.
# @PRE: job_id has an existing schedule. # @PRE job_id has an existing schedule.
# @POST: Schedule updated. # @POST Schedule updated; SCHEDULE_UPDATED event logged.
# @SIDE_EFFECT DB write; event logging.
def update_schedule( def update_schedule(
self, self,
job_id: str, job_id: str,
@@ -123,12 +129,13 @@ class TranslationScheduler:
logger.reflect("Schedule updated", {"schedule_id": schedule.id, "job_id": job_id}) logger.reflect("Schedule updated", {"schedule_id": schedule.id, "job_id": job_id})
return schedule return schedule
# [/DEF:update_schedule:Function] # #endregion update_schedule
# [DEF:delete_schedule:Function] # #region delete_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,delete]
# @PURPOSE: Delete a schedule for a job. # @BRIEF Delete a schedule for a job with event logging.
# @PRE: job_id has an existing schedule. # @PRE job_id has an existing schedule.
# @POST: Schedule deleted. # @POST Schedule deleted; SCHEDULE_DELETED event logged.
# @SIDE_EFFECT DB write; event logging.
def delete_schedule(self, job_id: str) -> None: def delete_schedule(self, job_id: str) -> None:
with belief_scope("TranslationScheduler.delete_schedule"): with belief_scope("TranslationScheduler.delete_schedule"):
schedule = self.db.query(TranslationSchedule).filter( schedule = self.db.query(TranslationSchedule).filter(
@@ -149,12 +156,13 @@ class TranslationScheduler:
) )
logger.reflect("Schedule deleted", {"schedule_id": schedule_id, "job_id": job_id}) logger.reflect("Schedule deleted", {"schedule_id": schedule_id, "job_id": job_id})
# [/DEF:delete_schedule:Function] # #endregion delete_schedule
# [DEF:enable_disable_schedule:Function] # #region set_schedule_active [C:4] [TYPE Function] [SEMANTICS translate,schedule,activate]
# @PURPOSE: Enable or disable a schedule. # @BRIEF Enable or disable a schedule by setting is_active flag.
# @PRE: job_id has an existing schedule. # @PRE job_id has an existing schedule.
# @POST: Schedule is_active updated. # @POST Schedule is_active updated.
# @SIDE_EFFECT DB write.
def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule: def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule:
with belief_scope("TranslationScheduler.set_schedule_active"): with belief_scope("TranslationScheduler.set_schedule_active"):
schedule = self.db.query(TranslationSchedule).filter( schedule = self.db.query(TranslationSchedule).filter(
@@ -174,12 +182,10 @@ class TranslationScheduler:
"is_active": is_active, "is_active": is_active,
}) })
return schedule return schedule
# [/DEF:enable_disable_schedule:Function] # #endregion set_schedule_active
# [DEF:get_schedule:Function] # #region get_schedule [C:2] [TYPE Function] [SEMANTICS translate,schedule,get]
# @PURPOSE: Get schedule for a job. # @BRIEF Get the schedule for a job by job_id.
# @PRE: job_id exists.
# @POST: Returns TranslationSchedule or raises ValueError.
def get_schedule(self, job_id: str) -> TranslationSchedule: def get_schedule(self, job_id: str) -> TranslationSchedule:
with belief_scope("TranslationScheduler.get_schedule"): with belief_scope("TranslationScheduler.get_schedule"):
schedule = self.db.query(TranslationSchedule).filter( schedule = self.db.query(TranslationSchedule).filter(
@@ -188,11 +194,10 @@ class TranslationScheduler:
if not schedule: if not schedule:
raise ValueError(f"No schedule found for job '{job_id}'") raise ValueError(f"No schedule found for job '{job_id}'")
return schedule return schedule
# [/DEF:get_schedule:Function] # #endregion get_schedule
# [DEF:list_active_schedules:Function] # #region list_active_schedules [C:2] [TYPE Function] [SEMANTICS translate,schedule,list]
# @PURPOSE: List all active schedules. # @BRIEF List all active schedules.
# @POST: Returns list of active TranslationSchedule rows.
@staticmethod @staticmethod
def list_active_schedules(db: Session) -> List[TranslationSchedule]: def list_active_schedules(db: Session) -> List[TranslationSchedule]:
return ( return (
@@ -200,13 +205,10 @@ class TranslationScheduler:
.filter(TranslationSchedule.is_active == True) .filter(TranslationSchedule.is_active == True)
.all() .all()
) )
# [/DEF:list_active_schedules:Function] # #endregion list_active_schedules
# [DEF:get_next_executions:Function] # #region get_next_executions [C:2] [TYPE Function] [SEMANTICS translate,schedule,next]
# @PURPOSE: Compute next N execution times from cron expression. # @BRIEF Compute next N execution times from a cron expression using APScheduler's CronTrigger.
# @PRE: cron_expression is valid.
# @POST: Returns list of ISO datetime strings.
@staticmethod
def get_next_executions(cron_expression: str, timezone_str: str = "UTC", n: int = 3) -> List[str]: def get_next_executions(cron_expression: str, timezone_str: str = "UTC", n: int = 3) -> List[str]:
from zoneinfo import ZoneInfo from zoneinfo import ZoneInfo
from apscheduler.triggers.cron import CronTrigger from apscheduler.triggers.cron import CronTrigger
@@ -231,18 +233,20 @@ class TranslationScheduler:
prev = ft prev = ft
next_time = ft next_time = ft
return results return results
# [/DEF:get_next_executions:Function] # #endregion get_next_executions
# [/DEF:TranslationScheduler:Class] # #endregion TranslationScheduler
# [DEF:execute_scheduled_translation:Function] # #region execute_scheduled_translation [C:5] [TYPE Function] [SEMANTICS translate,schedule,execute]
# @PURPOSE: APScheduler job handler — runs scheduled translation with concurrency check and new-key-only fallback. # @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and baseline expiry detection.
# @PRE: schedule_id is valid. # @PRE schedule_id is valid and job exists.
# @POST: Translation run created and executed if no concurrent run exists. # @POST Translation run created and executed if no concurrent run exists.
# @SIDE_EFFECT: DB writes; LLM calls; Superset API calls. # @SIDE_EFFECT DB writes; LLM calls; Superset API calls; event logging.
# @RATIONALE: New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full. # @DATA_CONTRACT Input[schedule_id, job_id, db_session_maker, config_manager] -> Output[None]
# @INVARIANT Concurrency guard: max 1 pending/running run per job at trigger time.
# @RATIONALE New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.
def execute_scheduled_translation( def execute_scheduled_translation(
schedule_id: str, schedule_id: str,
job_id: str, job_id: str,
@@ -357,5 +361,5 @@ def execute_scheduled_translation(
logger.error(f"[scheduled_translation] Unexpected error: {e}") logger.error(f"[scheduled_translation] Unexpected error: {e}")
finally: finally:
db.close() db.close()
# [/DEF:execute_scheduled_translation:Function] # #endregion execute_scheduled_translation
# [/DEF:TranslationScheduler:Module] # #endregion TranslationScheduler

219
backend/src/plugins/translate/service.py
View File

@@ -1,16 +1,16 @@
# [DEF:TranslateJobService:Module] # #region TranslateJobService [C:5] [TYPE Module] [SEMANTICS translate,service,crud,validation]
# @COMPLEXITY: 4 # @BRIEF Service layer for translation job CRUD with datasource column validation and database dialect detection.
# @SEMANTICS: translate, service, crud, validation # @LAYER Domain
# @PURPOSE: Service layer for translation job CRUD with datasource column validation and database dialect detection. # @RELATION DEPENDS_ON -> [TranslationJob]
# @LAYER: Domain # @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION: DEPENDS_ON -> [TranslationJob] # @RELATION DEPENDS_ON -> [ConfigManager]
# @RELATION: DEPENDS_ON -> [SupersetClient] # @PRE Database session and config manager are available.
# @RELATION: DEPENDS_ON -> [ConfigManager] # @POST Translation jobs are created/updated/deleted with column validation and dialect caching.
# @PRE: Database session and config manager are available. # @SIDE_EFFECT Queries Superset for column metadata and database dialect at save time.
# @POST: Translation jobs are created/updated/deleted with column validation and dialect caching. # @DATA_CONTRACT Input[TranslateJobCreate/Update] -> Output[TranslationJob/TranslateJobResponse]
# @SIDE_EFFECT: Queries Superset for column metadata and database dialect at save time. # @INVARIANT Snapshot isolation: in-progress runs use config snapshot; config edits affect future runs only.
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. # @RATIONALE Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity. # @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity.
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -37,12 +37,16 @@ SUPPORTED_DIALECTS = {
} }
# [DEF:get_dialect_from_database:Function] # #region get_dialect_from_database [C:4] [TYPE Function] [SEMANTICS translate,dialect,validation]
# @PURPOSE: Extract normalized dialect string from a Superset database record. # @BRIEF Extract normalized dialect string from a Superset database record and validate it is supported.
# @PRE: database_record is a dict from Superset API. # @PRE database_record is a dict from Superset API with 'backend' or 'engine' key.
# @POST: Returns normalized dialect string or raises ValueError if unsupported. # @POST Returns normalized dialect string or raises ValueError if unsupported.
# @SIDE_EFFECT None — pure data extraction.
def get_dialect_from_database(database_record: Dict[str, Any]) -> str: def get_dialect_from_database(database_record: Dict[str, Any]) -> str:
"""Extract and validate dialect from Superset database record.""" """Extract and validate dialect from Superset database record."""
logger.reason("Extracting dialect from database record", {
"has_backend": bool(database_record.get("backend") or database_record.get("engine")),
})
backend = ( backend = (
database_record.get("backend") database_record.get("backend")
or database_record.get("engine") or database_record.get("engine")
@@ -50,6 +54,9 @@ def get_dialect_from_database(database_record: Dict[str, Any]) -> str:
).lower().strip() ).lower().strip()
if not backend: if not backend:
logger.explore("Could not determine database dialect", {
"record_keys": list(database_record.keys()),
})
raise ValueError("Could not determine database dialect from connection") raise ValueError("Could not determine database dialect from connection")
# Map Superset backend names to normalized dialect # Map Superset backend names to normalized dialect
@@ -75,28 +82,36 @@ def get_dialect_from_database(database_record: Dict[str, Any]) -> str:
normalized = dialect_map.get(backend, backend) normalized = dialect_map.get(backend, backend)
if normalized not in SUPPORTED_DIALECTS: if normalized not in SUPPORTED_DIALECTS:
logger.explore("Unsupported dialect", {"backend": backend, "normalized": normalized})
raise ValueError( raise ValueError(
f"Unsupported database dialect: '{backend}'. " f"Unsupported database dialect: '{backend}'. "
f"Supported dialects: {', '.join(sorted(SUPPORTED_DIALECTS))}" f"Supported dialects: {', '.join(sorted(SUPPORTED_DIALECTS))}"
) )
logger.reflect("Dialect validated", {"dialect": normalized})
return normalized return normalized
# [/DEF:get_dialect_from_database:Function] # #endregion get_dialect_from_database
# [DEF:fetch_datasource_metadata:Function] # #region fetch_datasource_metadata [C:4] [TYPE Function] [SEMANTICS translate,datasource,metadata]
# @PURPOSE: Fetch datasource columns and database dialect from Superset. # @BRIEF Fetch datasource columns and database dialect from Superset for a given dataset.
# @PRE: dataset_id is a valid Superset dataset ID and environment has valid credentials. # @PRE dataset_id is a valid Superset dataset ID and environment has valid credentials.
# @POST: Returns (columns_list, dialect_string) or raises on failure. # @POST Returns (columns_list, dialect_string) or raises on failure.
# @SIDE_EFFECT Queries Superset API for dataset detail and database info.
def fetch_datasource_metadata( def fetch_datasource_metadata(
dataset_id: int, dataset_id: int,
env_id: str, env_id: str,
config_manager: ConfigManager, config_manager: ConfigManager,
) -> Tuple[List[Dict[str, Any]], str]: ) -> Tuple[List[Dict[str, Any]], str]:
"""Fetch column metadata and database dialect for a datasource from Superset.""" """Fetch column metadata and database dialect for a datasource from Superset."""
logger.reason("Fetching datasource metadata", {
"dataset_id": dataset_id,
"env_id": env_id,
})
# Find environment config # Find environment config
environments = config_manager.get_environments() environments = config_manager.get_environments()
env_config = next((e for e in environments if e.id == env_id), None) env_config = next((e for e in environments if e.id == env_id), None)
if not env_config: if not env_config:
logger.explore("Environment not found", {"env_id": env_id})
raise ValueError(f"Superset environment '{env_id}' not found in configuration") raise ValueError(f"Superset environment '{env_id}' not found in configuration")
# Create Superset client and fetch dataset detail # Create Superset client and fetch dataset detail
@@ -133,25 +148,29 @@ def fetch_datasource_metadata(
else: else:
raise ValueError("No database information available for this datasource") raise ValueError("No database information available for this datasource")
except Exception as e: except Exception as e:
logger.warning(f"[translate_service] Could not fetch database dialect: {e}") logger.explore("Could not fetch database dialect", {"error": str(e)})
raise ValueError(f"Could not determine database dialect: {e}") raise ValueError(f"Could not determine database dialect: {e}")
logger.reflect("Metadata fetched", {"columns": len(columns), "dialect": dialect})
return columns, dialect return columns, dialect
# [/DEF:fetch_datasource_metadata:Function] # #endregion fetch_datasource_metadata
# [DEF:detect_virtual_columns:Function] # #region detect_virtual_columns [C:2] [TYPE Function] [SEMANTICS translate,columns]
# @PURPOSE: Identify virtual (calculated) columns from column metadata. # @BRIEF Identify virtual (calculated) columns from column metadata by filtering non-physical entries.
# @PRE: columns is a list of dicts with 'is_physical' key.
# @POST: Returns list of virtual column names.
def detect_virtual_columns(columns: List[Dict[str, Any]]) -> List[str]: def detect_virtual_columns(columns: List[Dict[str, Any]]) -> List[str]:
"""Return names of columns that are virtual (not physical).""" """Return names of columns that are virtual (not physical)."""
return [col["name"] for col in columns if not col.get("is_physical", True)] return [col["name"] for col in columns if not col.get("is_physical", True)]
# [/DEF:detect_virtual_columns:Function] # #endregion detect_virtual_columns
# [DEF:TranslateJobService:Class] # #region TranslateJobService [C:5] [TYPE Class] [SEMANTICS translate,service,crud]
# @PURPOSE: Service for translation job CRUD with validation and Superset integration. # @BRIEF Service for translation job CRUD with validation and Superset integration.
# @PRE Database session and config manager are available.
# @POST Translation jobs are managed with full lifecycle (create/update/delete/duplicate).
# @SIDE_EFFECT Queries Superset for dialect detection and column validation.
# @DATA_CONTRACT Input[TranslateJobCreate/Update] -> Output[TranslationJob]
# @INVARIANT Snapshot isolation: in-progress runs unaffected by config edits.
class TranslateJobService: class TranslateJobService:
def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None): def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None):
@@ -159,9 +178,8 @@ class TranslateJobService:
self.config_manager = config_manager self.config_manager = config_manager
self.current_user = current_user self.current_user = current_user
# [DEF:list_jobs:Function] # #region list_jobs [C:2] [TYPE Function] [SEMANTICS translate,jobs,query]
# @PURPOSE: List translation jobs with optional status filter and pagination. # @BRIEF List translation jobs with optional status filter and pagination.
# @POST: Returns tuple of (total_count, list_of_jobs).
def list_jobs( def list_jobs(
self, self,
page: int = 1, page: int = 1,
@@ -178,29 +196,30 @@ class TranslateJobService:
jobs = query.order_by(TranslationJob.created_at.desc()).offset(offset).limit(page_size).all() jobs = query.order_by(TranslationJob.created_at.desc()).offset(offset).limit(page_size).all()
return total, jobs return total, jobs
# [/DEF:list_jobs:Function] # #endregion list_jobs
# [DEF:get_job:Function] # #region get_job [C:2] [TYPE Function] [SEMANTICS translate,jobs,query]
# @PURPOSE: Get a single translation job by ID. # @BRIEF Get a single translation job by ID, raising ValueError if not found.
# @POST: Returns TranslationJob or raises ValueError.
def get_job(self, job_id: str) -> TranslationJob: def get_job(self, job_id: str) -> TranslationJob:
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
if not job: if not job:
raise ValueError(f"Translation job '{job_id}' not found") raise ValueError(f"Translation job '{job_id}' not found")
return job return job
# [/DEF:get_job:Function] # #endregion get_job
# [DEF:create_job:Function] # #region create_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,create]
# @PURPOSE: Create a new translation job with column validation. # @BRIEF Create a new translation job with column validation and dialect detection.
# @PRE: payload contains valid job configuration. # @PRE payload contains valid job configuration (name, upsert_strategy, etc).
# @POST: Returns the created TranslationJob with database_dialect cached. # @POST Returns the created TranslationJob with database_dialect cached.
# @SIDE_EFFECT: Validates columns via SupersetClient if source_datasource_id is provided. # @SIDE_EFFECT Validates columns via SupersetClient if source_datasource_id is provided; caches database_dialect.
# @SIDE_EFFECT: Caches database_dialect from Superset connection.
def create_job(self, payload: TranslateJobCreate) -> TranslationJob: def create_job(self, payload: TranslateJobCreate) -> TranslationJob:
logger.info(f"[TranslateJobService] Creating job '{payload.name}'") logger.reason("Creating translation job", {"name": payload.name})
# Validate: must have a translation column if datasource is configured # Validate: must have a translation column if datasource is configured
if payload.source_datasource_id and not payload.translation_column: if payload.source_datasource_id and not payload.translation_column:
logger.explore("Missing translation column", {
"source_datasource_id": payload.source_datasource_id,
})
raise ValueError("A translation column is required when a datasource is selected") raise ValueError("A translation column is required when a datasource is selected")
# Validate upsert strategy # Validate upsert strategy
@@ -225,7 +244,7 @@ class TranslateJobService:
) )
dialect = detected_dialect dialect = detected_dialect
except Exception as e: except Exception as e:
logger.warning(f"[TranslateJobService] Dialect detection failed: {e}") logger.explore("Dialect detection failed", {"error": str(e)})
dialect = payload.source_dialect dialect = payload.source_dialect
# Build job instance # Build job instance
@@ -249,6 +268,7 @@ class TranslateJobService:
batch_size=payload.batch_size, batch_size=payload.batch_size,
upsert_strategy=payload.upsert_strategy, upsert_strategy=payload.upsert_strategy,
environment_id=payload.environment_id, environment_id=payload.environment_id,
target_database_id=payload.target_database_id,
status="DRAFT", status="DRAFT",
created_by=self.current_user, created_by=self.current_user,
) )
@@ -267,17 +287,17 @@ class TranslateJobService:
self.db.commit() self.db.commit()
self.db.refresh(job) self.db.refresh(job)
logger.info(f"[TranslateJobService] Created job '{job.id}'") logger.reflect("Job created", {"job_id": job.id})
return job return job
# [/DEF:create_job:Function] # #endregion create_job
# [DEF:update_job:Function] # #region update_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,update]
# @PURPOSE: Update an existing translation job. # @BRIEF Update an existing translation job with optional dialect re-detection.
# @PRE: payload contains fields to update. # @PRE payload contains fields to update; job_id exists.
# @POST: Returns the updated TranslationJob. # @POST Returns the updated TranslationJob with re-detected dialect if datasource changed.
# @SIDE_EFFECT: Re-detects database_dialect if source_datasource_id changed. # @SIDE_EFFECT Re-detects database_dialect if source_datasource_id changed.
def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob: def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob:
logger.info(f"[TranslateJobService] Updating job '{job_id}'") logger.reason("Updating translation job", {"job_id": job_id})
job = self.get_job(job_id) job = self.get_job(job_id)
update_data = payload.model_dump(exclude_unset=True) update_data = payload.model_dump(exclude_unset=True)
@@ -298,7 +318,7 @@ class TranslateJobService:
) )
job.database_dialect = detected_dialect job.database_dialect = detected_dialect
except Exception as e: except Exception as e:
logger.warning(f"[TranslateJobService] Dialect re-detection failed: {e}") logger.explore("Dialect re-detection failed", {"error": str(e)})
job.updated_at = datetime.now(timezone.utc) job.updated_at = datetime.now(timezone.utc)
@@ -317,16 +337,17 @@ class TranslateJobService:
self.db.commit() self.db.commit()
self.db.refresh(job) self.db.refresh(job)
logger.info(f"[TranslateJobService] Updated job '{job_id}'") logger.reflect("Job updated", {"job_id": job_id})
return job return job
# [/DEF:update_job:Function] # #endregion update_job
# [DEF:delete_job:Function] # #region delete_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,delete]
# @PURPOSE: Delete a translation job and its associations. # @BRIEF Delete a translation job and its dictionary associations.
# @PRE: job_id must exist. # @PRE job_id must exist.
# @POST: Job and all related records are deleted. # @POST Job and all related TranslationJobDictionary records are deleted.
# @SIDE_EFFECT Deletes TranslationJob and TranslationJobDictionary rows.
def delete_job(self, job_id: str) -> None: def delete_job(self, job_id: str) -> None:
logger.info(f"[TranslateJobService] Deleting job '{job_id}'") logger.reason("Deleting translation job", {"job_id": job_id})
job = self.get_job(job_id) job = self.get_job(job_id)
# Delete dictionary associations # Delete dictionary associations
@@ -336,15 +357,16 @@ class TranslateJobService:
self.db.delete(job) self.db.delete(job)
self.db.commit() self.db.commit()
logger.info(f"[TranslateJobService] Deleted job '{job_id}'") logger.reflect("Job deleted", {"job_id": job_id})
# [/DEF:delete_job:Function] # #endregion delete_job
# [DEF:duplicate_job:Function] # #region duplicate_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,duplicate]
# @PURPOSE: Duplicate a translation job with a new name. # @BRIEF Duplicate a translation job with a new name, copying all fields and dictionary associations.
# @PRE: job_id must exist. # @PRE job_id must exist.
# @POST: Returns the duplicated TranslationJob with status DRAFT. # @POST Returns the duplicated TranslationJob with status DRAFT.
# @SIDE_EFFECT Creates new TranslationJob and TranslationJobDictionary rows.
def duplicate_job(self, job_id: str, new_name: Optional[str] = None) -> TranslationJob: def duplicate_job(self, job_id: str, new_name: Optional[str] = None) -> TranslationJob:
logger.info(f"[TranslateJobService] Duplicating job '{job_id}'") logger.reason("Duplicating translation job", {"job_id": job_id, "new_name": new_name})
source = self.get_job(job_id) source = self.get_job(job_id)
# Copy all fields except id, created_at, updated_at, status # Copy all fields except id, created_at, updated_at, status
@@ -367,6 +389,8 @@ class TranslateJobService:
provider_id=source.provider_id, provider_id=source.provider_id,
batch_size=source.batch_size, batch_size=source.batch_size,
upsert_strategy=source.upsert_strategy, upsert_strategy=source.upsert_strategy,
environment_id=source.environment_id,
target_database_id=source.target_database_id,
status="DRAFT", status="DRAFT",
created_by=self.current_user, created_by=self.current_user,
) )
@@ -387,22 +411,20 @@ class TranslateJobService:
self.db.commit() self.db.commit()
self.db.refresh(new_job) self.db.refresh(new_job)
logger.info(f"[TranslateJobService] Duplicated job '{job_id}' -> '{new_job.id}'") logger.reflect("Job duplicated", {"new_job_id": new_job.id, "source_job_id": job_id})
return new_job return new_job
# [/DEF:duplicate_job:Function] # #endregion duplicate_job
# [DEF:get_job_dictionary_ids:Function] # #region get_job_dictionary_ids [C:1] [TYPE Function] [SEMANTICS translate,jobs,dictionaries]
# @PURPOSE: Get dictionary IDs attached to a job.
# @POST: Returns list of dictionary IDs.
def get_job_dictionary_ids(self, job_id: str) -> List[str]: def get_job_dictionary_ids(self, job_id: str) -> List[str]:
assocs = self.db.query(TranslationJobDictionary).filter( assocs = self.db.query(TranslationJobDictionary).filter(
TranslationJobDictionary.job_id == job_id TranslationJobDictionary.job_id == job_id
).all() ).all()
return [a.dictionary_id for a in assocs] return [a.dictionary_id for a in assocs]
# [/DEF:get_job_dictionary_ids:Function] # #endregion get_job_dictionary_ids
# [DEF:fetch_available_datasources:Function] # #region fetch_available_datasources [C:2] [TYPE Function] [SEMANTICS translate,datasources,query]
# @PURPOSE: List available Superset datasets for translation job creation. # @BRIEF List available Superset datasets for translation job creation with optional search filter.
def fetch_available_datasources(self, env_id: str, search: Optional[str] = None) -> list: def fetch_available_datasources(self, env_id: str, search: Optional[str] = None) -> list:
"""List Superset datasets available for translation.""" """List Superset datasets available for translation."""
from ...core.superset_client import SupersetClient from ...core.superset_client import SupersetClient
@@ -428,12 +450,11 @@ class TranslateJobService:
"description": ds.get("description", ""), "description": ds.get("description", ""),
}) })
return result return result
# [/DEF:fetch_available_datasources:Function] # #endregion fetch_available_datasources
# [/DEF:TranslateJobService:Class] # #endregion TranslateJobService
# [DEF:_extract_dialect:Function] # #region _extract_dialect [C:1] [TYPE Function] [SEMANTICS translate,dialect]
# @PURPOSE: Extract database dialect from Superset backend URI.
def _extract_dialect(backend: str) -> str: def _extract_dialect(backend: str) -> str:
"""Extract dialect name from a Superset database backend URI (e.g. 'postgresql+psycopg2://...').""" """Extract dialect name from a Superset database backend URI (e.g. 'postgresql+psycopg2://...')."""
if not backend: if not backend:
@@ -444,10 +465,11 @@ def _extract_dialect(backend: str) -> str:
return dialect.lower() return dialect.lower()
except Exception: except Exception:
return "unknown" return "unknown"
# #endregion _extract_dialect
# [DEF:job_to_response:Function] # #region job_to_response [C:2] [TYPE Function] [SEMANTICS translate,serialization]
# @PURPOSE: Convert a TranslationJob ORM model to a TranslateJobResponse schema with dictionary_ids. # @BRIEF Convert a TranslationJob ORM model to a TranslateJobResponse schema with dictionary_ids.
def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -> TranslateJobResponse: def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -> TranslateJobResponse:
return TranslateJobResponse( return TranslateJobResponse(
id=job.id, id=job.id,
@@ -474,27 +496,29 @@ def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -
updated_at=job.updated_at, updated_at=job.updated_at,
dictionary_ids=dict_ids or [], dictionary_ids=dict_ids or [],
environment_id=job.environment_id, environment_id=job.environment_id,
target_database_id=job.target_database_id,
) )
# [/DEF:job_to_response:Function] # #endregion job_to_response
# [DEF:DatasourceColumnsService:Function] # #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS translate,datasource,columns]
# @PURPOSE: Fetch datasource column metadata from Superset and return structured response. # @BRIEF Fetch datasource column metadata from Superset and return structured DatasourceColumnsResponse.
# @PRE: datasource_id is a valid Superset dataset ID. # @PRE datasource_id is a valid Superset dataset ID.
# @POST: Returns DatasourceColumnsResponse with column metadata and database dialect. # @POST Returns DatasourceColumnsResponse with column metadata and database dialect.
# @SIDE_EFFECT: Queries Superset API for dataset detail and database info. # @SIDE_EFFECT Queries Superset API for dataset detail and database info.
def get_datasource_columns( def get_datasource_columns(
datasource_id: int, datasource_id: int,
env_id: str, env_id: str,
config_manager: ConfigManager, config_manager: ConfigManager,
) -> DatasourceColumnsResponse: ) -> DatasourceColumnsResponse:
"""Fetch and return column metadata for a given Superset datasource.""" """Fetch and return column metadata for a given Superset datasource."""
logger.info(f"[get_datasource_columns] Fetching columns for datasource {datasource_id}") logger.reason("Fetching datasource columns", {"datasource_id": datasource_id, "env_id": env_id})
# Find environment config # Find environment config
environments = config_manager.get_environments() environments = config_manager.get_environments()
env_config = next((e for e in environments if e.id == env_id), None) env_config = next((e for e in environments if e.id == env_id), None)
if not env_config: if not env_config:
logger.explore("Environment not found", {"env_id": env_id})
raise ValueError(f"Superset environment '{env_id}' not found") raise ValueError(f"Superset environment '{env_id}' not found")
# Create Superset client # Create Superset client
@@ -516,6 +540,7 @@ def get_datasource_columns(
db_result = db_record.get("result", db_record) if isinstance(db_record, dict) else db_record db_result = db_record.get("result", db_record) if isinstance(db_record, dict) else db_record
dialect = get_dialect_from_database(db_result) dialect = get_dialect_from_database(db_result)
else: else:
logger.explore("Could not determine dialect", {"datasource_id": datasource_id})
raise ValueError("Could not determine database dialect for this datasource") raise ValueError("Could not determine database dialect for this datasource")
# Extract columns # Extract columns
@@ -533,13 +558,19 @@ def get_datasource_columns(
description=col.get("description", ""), description=col.get("description", ""),
)) ))
return DatasourceColumnsResponse( result = DatasourceColumnsResponse(
datasource_id=datasource_id, datasource_id=datasource_id,
datasource_name=dataset_detail.get("table_name"), datasource_name=dataset_detail.get("table_name"),
schema_name=dataset_detail.get("schema"), schema_name=dataset_detail.get("schema"),
database_dialect=dialect, database_dialect=dialect,
columns=columns, columns=columns,
) )
# [/DEF:DatasourceColumnsService:Function] logger.reflect("Datasource columns fetched", {
"datasource_id": datasource_id,
"columns_count": len(columns),
"dialect": dialect,
})
return result
# #endregion get_datasource_columns
# [/DEF:TranslateJobService:Module] # #endregion TranslateJobService

179
backend/src/plugins/translate/sql_generator.py
View File

@@ -1,18 +1,13 @@
# [DEF:SQLGenerator:Module] # #region SQLGenerator [C:4] [TYPE Module] [SEMANTICS translate,sql,generator,dialect]
# @COMPLEXITY: 3 # @BRIEF Dialect-aware safe SQL generation for INSERT/UPSERT operations with identifier quoting and value encoding.
# @SEMANTICS: translate, sql, generator, dialect # @LAYER Domain
# @PURPOSE: Dialect-aware safe SQL generation for INSERT/UPSERT operations. # @RELATION DEPENDS_ON -> [TranslationJob]
# @LAYER: Domain # @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION: DEPENDS_ON -> [TranslationJob] # @PRE Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS.
# @RELATION: DEPENDS_ON -> [TranslationRun] # @POST Returns safe SQL strings for the target dialect.
# @PRE: Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS.
# @POST: Returns safe SQL strings for the target dialect.
# @SIDE_EFFECT: None — pure code generation.
# @RATIONALE: Dialect-aware SQL uses ON CONFLICT for PostgreSQL; plain INSERT for ClickHouse with documented limitations.
# @REJECTED: UPDATE statements — source is append-only; UPSERT covers overwrite case.
# @REJECTED: ORM-based insert bypasses Superset's SQL Lab audit trail.
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple
from datetime import datetime, timezone
from ...core.logger import logger, belief_scope from ...core.logger import logger, belief_scope
# PostgreSQL/Greenplum dialects that support ON CONFLICT # PostgreSQL/Greenplum dialects that support ON CONFLICT
@@ -21,10 +16,45 @@ POSTGRESQL_DIALECTS = {"postgresql", "redshift", "greenplum"}
CLICKHOUSE_DIALECTS = {"clickhouse"} CLICKHOUSE_DIALECTS = {"clickhouse"}
# [DEF:_quote_identifier:Function] # #region _normalize_timestamp_value [C:2] [TYPE Function] [SEMANTICS translate,sql,timestamp]
# @PURPOSE: Quote an identifier per dialect rules. PostgreSQL uses double quotes; ClickHouse uses backticks. # @BRIEF Detect Unix timestamp strings (seconds or millis) and convert to 'YYYY-MM-DD' for Date columns.
# @PRE: identifier is a non-empty string. def _normalize_timestamp_value(value: Any) -> Optional[str]:
# @POST: Returns safely quoted identifier. """Detect Unix timestamp values and convert to date string.
Handles:
- Integer/float Unix timestamps (seconds or milliseconds)
- String representations of Unix timestamps (e.g. '1726358400000.0')
Returns 'YYYY-MM-DD' if conversion succeeds, None if value is not a timestamp.
"""
# Try numeric conversion first
try:
ts = float(value)
except (ValueError, TypeError):
return None
# Heuristic: Unix timestamps in seconds are ~10 digits (1e9 range for 2001-2033)
# Unix timestamps in milliseconds are ~13 digits (1e12 range for 2001-2033)
if 1e9 <= ts < 1e12:
# Already in seconds
pass
elif 1e12 <= ts < 1e15:
# Milliseconds — convert to seconds
ts = ts / 1000.0
else:
# Not a plausible Unix timestamp
return None
try:
dt = datetime.fromtimestamp(ts, tz=timezone.utc)
return dt.strftime("%Y-%m-%d")
except (OSError, OverflowError, ValueError):
return None
# #endregion _normalize_timestamp_value
# #region _quote_identifier [C:2] [TYPE Function] [SEMANTICS translate,sql,quoting]
# @BRIEF Quote a SQL identifier per dialect rules — double quotes for PostgreSQL, backticks for ClickHouse.
def _quote_identifier(identifier: str, dialect: str) -> str: def _quote_identifier(identifier: str, dialect: str) -> str:
"""Quote a SQL identifier per dialect rules.""" """Quote a SQL identifier per dialect rules."""
if not identifier: if not identifier:
@@ -38,50 +68,67 @@ def _quote_identifier(identifier: str, dialect: str) -> str:
else: else:
# Generic ANSI double-quote # Generic ANSI double-quote
return f'"{cleaned}"' return f'"{cleaned}"'
# [/DEF:_quote_identifier:Function] # #endregion _quote_identifier
# [DEF:_encode_sql_value:Function] # #region _encode_sql_value [C:2] [TYPE Function] [SEMANTICS translate,sql,encoding]
# @PURPOSE: Encode a Python value into a SQL-safe literal for INSERT VALUES. # @BRIEF Encode a Python value into a SQL-safe literal for INSERT VALUES, with ClickHouse timestamp normalization.
# @PRE: value is a Python primitive or None. def _encode_sql_value(value: Any, dialect: Optional[str] = None) -> str:
# @POST: Returns SQL-safe string literal representation. """Encode a Python value into a SQL-safe literal.
def _encode_sql_value(value: Any) -> str:
"""Encode a Python value into a SQL-safe literal.""" For ClickHouse dialect, attempts to detect Unix timestamp strings
and convert them to 'YYYY-MM-DD' format for Date column compatibility.
"""
if value is None: if value is None:
return "NULL" return "NULL"
if isinstance(value, bool): if isinstance(value, bool):
return "TRUE" if value else "FALSE" return "TRUE" if value else "FALSE"
# For ClickHouse: try to normalize timestamp-like string values
if dialect in CLICKHOUSE_DIALECTS and isinstance(value, str) and value:
normalized = _normalize_timestamp_value(value)
if normalized:
return f"'{normalized}'"
if isinstance(value, (int, float)): if isinstance(value, (int, float)):
return str(value) return str(value)
# String — escape single quotes by doubling them # String — escape single quotes by doubling them
escaped = str(value).replace("'", "''") escaped = str(value).replace("'", "''")
return f"'{escaped}'" return f"'{escaped}'"
# [/DEF:_encode_sql_value:Function] # #endregion _encode_sql_value
# [DEF:_build_values_clause:Function] # #region _build_values_clause [C:2] [TYPE Function] [SEMANTICS translate,sql,values]
# @PURPOSE: Build a VALUES clause for multiple rows. # @BRIEF Build a VALUES clause for multiple rows with per-column value encoding and dialect-aware quoting.
# @PRE: columns list is non-empty; rows is a list of dicts. def _build_values_clause(columns: List[str], rows: List[Dict[str, Any]], dialect: Optional[str] = None) -> str:
# @POST: Returns SQL VALUES clause string. """Build VALUES (...) clause for multiple rows.
def _build_values_clause(columns: List[str], rows: List[Dict[str, Any]]) -> str:
"""Build VALUES (...) clause for multiple rows.""" NOTE: columns may be quoted (e.g. '"col"' or '`col`') for the SQL column list,
but row dicts have UNQUOTED keys. Strip quotes before value lookup.
"""
value_groups = [] value_groups = []
for row in rows: for row in rows:
values = [_encode_sql_value(row.get(col)) for col in columns] values = []
for col in columns:
# Strip quoting for value lookup — row keys are unquoted
lookup_key = col.strip('"').strip('`').strip('[]')
val = row.get(lookup_key)
values.append(_encode_sql_value(val, dialect=dialect))
value_groups.append(f"({', '.join(values)})") value_groups.append(f"({', '.join(values)})")
return ",\n".join(value_groups) return ",\n".join(value_groups)
# [/DEF:_build_values_clause:Function] # #endregion _build_values_clause
# [DEF:generate_insert_sql:Function] # #region generate_insert_sql [C:3] [TYPE Function] [SEMANTICS translate,sql,insert]
# @PURPOSE: Generate a dialect-aware INSERT SQL statement. # @BRIEF Generate a dialect-aware plain INSERT SQL statement for the given table, columns, and rows.
# @PRE: target_table, columns are non-empty. # @RELATION DEPENDS_ON -> [_quote_identifier]
# @POST: Returns SQL string or raises ValueError on invalid input. # @RELATION DEPENDS_ON -> [_build_values_clause]
def generate_insert_sql( def generate_insert_sql(
target_schema: Optional[str], target_schema: Optional[str],
target_table: str, target_table: str,
columns: List[str], columns: List[str],
rows: List[Dict[str, Any]], rows: List[Dict[str, Any]],
dialect: Optional[str] = None,
) -> str: ) -> str:
"""Generate a plain INSERT SQL.""" """Generate a plain INSERT SQL."""
with belief_scope("generate_insert_sql"): with belief_scope("generate_insert_sql"):
@@ -93,7 +140,7 @@ def generate_insert_sql(
raise ValueError("At least one row is required for INSERT SQL generation") raise ValueError("At least one row is required for INSERT SQL generation")
col_list = ", ".join(columns) col_list = ", ".join(columns)
values = _build_values_clause(columns, rows) values = _build_values_clause(columns, rows, dialect=dialect)
table_ref = target_table table_ref = target_table
if target_schema: if target_schema:
@@ -101,19 +148,20 @@ def generate_insert_sql(
sql = f"INSERT INTO {table_ref} ({col_list})\nVALUES\n{values};" sql = f"INSERT INTO {table_ref} ({col_list})\nVALUES\n{values};"
return sql return sql
# [/DEF:generate_insert_sql:Function] # #endregion generate_insert_sql
# [DEF:generate_upsert_sql:Function] # #region generate_upsert_sql [C:3] [TYPE Function] [SEMANTICS translate,sql,upsert]
# @PURPOSE: Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE. # @BRIEF Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE / DO NOTHING.
# @PRE: dialect is postgresql-compatible. target_table, columns, key_columns are non-empty. # @RELATION DEPENDS_ON -> [_quote_identifier]
# @POST: Returns UPSERT SQL string or raises ValueError. # @RELATION DEPENDS_ON -> [_build_values_clause]
def generate_upsert_sql( def generate_upsert_sql(
target_schema: Optional[str], target_schema: Optional[str],
target_table: str, target_table: str,
columns: List[str], columns: List[str],
key_columns: List[str], key_columns: List[str],
rows: List[Dict[str, Any]], rows: List[Dict[str, Any]],
dialect: Optional[str] = None,
) -> str: ) -> str:
"""Generate INSERT ... ON CONFLICT (key_cols) DO UPDATE SET ... SQL.""" """Generate INSERT ... ON CONFLICT (key_cols) DO UPDATE SET ... SQL."""
with belief_scope("generate_upsert_sql"): with belief_scope("generate_upsert_sql"):
@@ -128,7 +176,7 @@ def generate_upsert_sql(
col_list = ", ".join(columns) col_list = ", ".join(columns)
key_list = ", ".join(key_columns) key_list = ", ".join(key_columns)
values = _build_values_clause(columns, rows) values = _build_values_clause(columns, rows, dialect=dialect)
table_ref = target_table table_ref = target_table
if target_schema: if target_schema:
@@ -150,21 +198,23 @@ def generate_upsert_sql(
f"ON CONFLICT ({key_list}) {conflict_action};" f"ON CONFLICT ({key_list}) {conflict_action};"
) )
return sql return sql
# [/DEF:generate_upsert_sql:Function] # #endregion generate_upsert_sql
# [DEF:SQLGenerator:Class] # #region SQLGenerator [C:4] [TYPE Class] [SEMANTICS translate,sql,generator]
# @COMPLEXITY: 3 # @BRIEF Generate safe, dialect-appropriate SQL INSERT/UPSERT statements with batching support.
# @PURPOSE: Generate safe, dialect-appropriate SQL INSERT/UPSERT statements. # @PRE Job has target_schema, target_table, key columns configured.
# @PRE: Job has target_schema, target_table, key columns configured. # @POST Returns generated SQL string for the target dialect.
# @POST: Returns generated SQL string for the target dialect. # @SIDE_EFFECT None — pure SQL generation.
# @RELATION DEPENDS_ON -> [generate_insert_sql]
# @RELATION DEPENDS_ON -> [generate_upsert_sql]
class SQLGenerator: class SQLGenerator:
# [DEF:SQLGenerator.generate:Function] # #region SQLGenerator.generate [C:4] [TYPE Function] [SEMANTICS translate,sql,generate]
# @PURPOSE: Generate SQL for a set of rows, detecting dialect from the job configuration. # @BRIEF Generate SQL for a set of rows, detecting dialect from the job configuration.
# @PRE: dialect is a supported database dialect. columns list is non-empty. rows is non-empty. # @PRE dialect is a supported database dialect. columns list is non-empty. rows is non-empty.
# @POST: Returns tuple of (sql_string, statement_count). # @POST Returns tuple of (sql_string, statement_count).
# @SIDE_EFFECT: None — pure SQL generation. # @SIDE_EFFECT None — pure SQL generation.
@staticmethod @staticmethod
def generate( def generate(
dialect: str, dialect: str,
@@ -236,6 +286,7 @@ class SQLGenerator:
columns=quoted_columns, columns=quoted_columns,
key_columns=quoted_key_columns, key_columns=quoted_key_columns,
rows=rows, rows=rows,
dialect=dialect,
) )
else: else:
sql = generate_insert_sql( sql = generate_insert_sql(
@@ -243,6 +294,7 @@ class SQLGenerator:
target_table=table_ref, target_table=table_ref,
columns=quoted_columns, columns=quoted_columns,
rows=rows, rows=rows,
dialect=dialect,
) )
elif dialect in CLICKHOUSE_DIALECTS: elif dialect in CLICKHOUSE_DIALECTS:
# ClickHouse: plain INSERT, no ON CONFLICT support # ClickHouse: plain INSERT, no ON CONFLICT support
@@ -251,6 +303,7 @@ class SQLGenerator:
target_table=table_ref, target_table=table_ref,
columns=quoted_columns, columns=quoted_columns,
rows=rows, rows=rows,
dialect=dialect,
) )
if use_upsert: if use_upsert:
logger.reason("ClickHouse UPSERT not supported; using plain INSERT", { logger.reason("ClickHouse UPSERT not supported; using plain INSERT", {
@@ -263,6 +316,7 @@ class SQLGenerator:
target_table=table_ref, target_table=table_ref,
columns=quoted_columns, columns=quoted_columns,
rows=rows, rows=rows,
dialect=dialect,
) )
logger.reflect("SQL generated", { logger.reflect("SQL generated", {
@@ -271,12 +325,11 @@ class SQLGenerator:
"sql_length": len(sql), "sql_length": len(sql),
}) })
return sql, len(rows) return sql, len(rows)
# [/DEF:SQLGenerator.generate:Function] # #endregion SQLGenerator.generate
# [DEF:SQLGenerator.generate_batch:Function] # #region SQLGenerator.generate_batch [C:3] [TYPE Function] [SEMANTICS translate,sql,batch]
# @PURPOSE: Generate separate INSERT statements for each row (batch-safe version). # @BRIEF Generate separate INSERT statements for each row chunk (batch-safe version).
# @PRE: Same as generate(). # @RELATION DEPENDS_ON -> [SQLGenerator.generate]
# @POST: Returns list of (sql_string, row_index) tuples.
@staticmethod @staticmethod
def generate_batch( def generate_batch(
dialect: str, dialect: str,
@@ -312,8 +365,8 @@ class SQLGenerator:
statements.append((sql, count)) statements.append((sql, count))
return statements return statements
# [/DEF:SQLGenerator.generate_batch:Function] # #endregion SQLGenerator.generate_batch
# [/DEF:SQLGenerator:Class] # #endregion SQLGenerator
# [/DEF:SQLGenerator:Module] # #endregion SQLGenerator

180
backend/src/plugins/translate/superset_executor.py
View File

@@ -1,17 +1,17 @@
# [DEF:SupersetSqlLabExecutor:Module] # #region SupersetSqlLabExecutor [C:5] [TYPE Module] [SEMANTICS translate,superset,sqllab,execute]
# @COMPLEXITY: 4 # @BRIEF Submit SQL to Superset SQL Lab API and poll execution status with retry.
# @SEMANTICS: translate, superset, sqllab, execute # @LAYER Infra
# @PURPOSE: Submit SQL to Superset SQL Lab API and poll execution status. # @RELATION DEPENDS_ON -> [SupersetClient]
# @LAYER: Infra # @RELATION DEPENDS_ON -> [ConfigManager]
# @RELATION: DEPENDS_ON -> [SupersetClient] # @PRE Valid Superset environment configuration and authenticated client.
# @RELATION: DEPENDS_ON -> [ConfigManager] # @POST SQL is submitted to Superset SQL Lab; execution reference is returned.
# @PRE: Valid Superset environment configuration and authenticated client. # @SIDE_EFFECT Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status.
# @POST: SQL is submitted to Superset SQL Lab; execution reference is returned. # @DATA_CONTRACT Input[sql:str, database_id:Optional] -> Output[Dict with query_id, status, error_message]
# @SIDE_EFFECT: Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status. # @INVARIANT Polling respects max_polls limit; timeout returns structured error.
# @RATIONALE: Direct SQL Lab API submission provides audit trail and execution monitoring within Superset. # @RATIONALE Direct SQL Lab API submission provides audit trail and execution monitoring within Superset.
# @REJECTED: Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC. # @REJECTED Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC.
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple, Union
from datetime import datetime, timezone from datetime import datetime, timezone
import json import json
import time import time
@@ -22,25 +22,26 @@ from ...core.config_manager import ConfigManager
from ...core.superset_client import SupersetClient from ...core.superset_client import SupersetClient
# [DEF:SupersetSqlLabExecutor:Class] # #region SupersetSqlLabExecutor [C:5] [TYPE Class] [SEMANTICS translate,superset,sqllab]
# @COMPLEXITY: 4 # @BRIEF Submit SQL to Superset SQL Lab API with polling and status tracking.
# @PURPOSE: Submit SQL to Superset SQL Lab API with polling and status tracking. # @PRE Valid environment ID and ConfigManager.
# @PRE: Valid environment ID and ConfigManager. # @POST SQL submitted to Superset; execution reference recorded; polling completes.
# @POST: SQL submitted to Superset; execution reference recorded. # @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status.
# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status. # @DATA_CONTRACT Input[env_id:str, database_id:Optional] -> Output[Dict with query_id, status]
# @INVARIANT database_id resolved lazily; polling capped at max_polls.
class SupersetSqlLabExecutor: class SupersetSqlLabExecutor:
def __init__(self, config_manager: ConfigManager, env_id: str): def __init__(self, config_manager: ConfigManager, env_id: str, database_id: Optional[int] = None):
self.config_manager = config_manager self.config_manager = config_manager
self.env_id = env_id self.env_id = env_id
self._client: Optional[SupersetClient] = None self._client: Optional[SupersetClient] = None
self._database_id: Optional[int] = None self._database_id: Optional[int] = database_id
# [DEF:_get_client:Function] # #region _get_client [C:4] [TYPE Function] [SEMANTICS translate,superset,client]
# @PURPOSE: Lazy-initialize SupersetClient for the configured environment. # @BRIEF Lazy-initialize SupersetClient for the configured environment.
# @PRE: env_id must correspond to a valid environment config. # @PRE env_id must correspond to a valid environment config.
# @POST: Returns authenticated SupersetClient. # @POST Returns authenticated SupersetClient.
# @SIDE_EFFECT: Authenticates against Superset API. # @SIDE_EFFECT Authenticates against Superset API on first call.
def _get_client(self) -> SupersetClient: def _get_client(self) -> SupersetClient:
with belief_scope("SupersetSqlLabExecutor._get_client"): with belief_scope("SupersetSqlLabExecutor._get_client"):
if self._client is not None: if self._client is not None:
@@ -53,13 +54,13 @@ class SupersetSqlLabExecutor:
self._client = SupersetClient(env_config) self._client = SupersetClient(env_config)
return self._client return self._client
# [/DEF:_get_client:Function] # #endregion _get_client
# [DEF:resolve_database_id:Function] # #region resolve_database_id [C:4] [TYPE Function] [SEMANTICS translate,database,resolve]
# @PURPOSE: Resolve the target database ID from the environment. # @BRIEF Resolve the target database ID from the environment by name or default.
# @PRE: database_name or database_id should be known. # @PRE Superset environment is reachable and has databases.
# @POST: Returns database_id integer or raises ValueError. # @POST Returns database_id integer or raises ValueError.
# @SIDE_EFFECT: Fetches databases list from Superset. # @SIDE_EFFECT Fetches databases list from Superset API.
def resolve_database_id(self, database_name: Optional[str] = None) -> int: def resolve_database_id(self, database_name: Optional[str] = None) -> int:
with belief_scope("SupersetSqlLabExecutor.resolve_database_id"): with belief_scope("SupersetSqlLabExecutor.resolve_database_id"):
client = self._get_client() client = self._get_client()
@@ -89,37 +90,71 @@ class SupersetSqlLabExecutor:
"database_id": self._database_id, "database_id": self._database_id,
}) })
return self._database_id return self._database_id
# [/DEF:resolve_database_id:Function] # #endregion resolve_database_id
# [DEF:execute_sql:Function] # #region resolve_database_uuid [C:4] [TYPE Function] [SEMANTICS translate,database,uuid]
# @PURPOSE: Submit SQL to Superset SQL Lab and return execution tracking info. # @BRIEF Resolve a database UUID to a numeric database ID from Superset.
# @PRE: sql is valid SQL string. database_id is a valid Superset DB ID. # @PRE uuid_str is a valid Superset database UUID.
# @POST: Returns execution result dict with query_id and status. # @POST Returns numeric database_id or raises ValueError if not found.
# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/. # @SIDE_EFFECT Fetches databases list from Superset API.
def resolve_database_uuid(self, uuid_str: str) -> int:
with belief_scope("SupersetSqlLabExecutor.resolve_database_uuid"):
client = self._get_client()
_, databases = client.get_databases(
query={"columns": ["id", "uuid", "database_name"]}
)
if not databases:
raise ValueError("No databases found in Superset environment")
for db in databases:
db_uuid = db.get("uuid") or ""
if db_uuid.lower() == uuid_str.lower():
db_id = db["id"]
self._database_id = db_id
logger.reason("Resolved database ID by UUID", {
"uuid": uuid_str,
"database_id": db_id,
"database_name": db.get("database_name"),
})
return db_id
raise ValueError(
f"Database with UUID '{uuid_str}' not found in Superset environment"
)
# #endregion resolve_database_uuid
# #region execute_sql [C:4] [TYPE Function] [SEMANTICS translate,sql,execute]
# @BRIEF Submit SQL to Superset SQL Lab and return execution tracking info.
# @PRE sql is valid SQL string. database_id is a valid Superset DB ID.
# @POST Returns execution result dict with query_id and status.
# @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/.
def execute_sql( def execute_sql(
self, self,
sql: str, sql: str,
database_id: Optional[int] = None, database_id: Optional[Union[int, str]] = None,
run_async: bool = True,
) -> Dict[str, Any]: ) -> Dict[str, Any]:
with belief_scope("SupersetSqlLabExecutor.execute_sql"): with belief_scope("SupersetSqlLabExecutor.execute_sql"):
client = self._get_client() client = self._get_client()
db_id = database_id or self._database_id db_id = database_id or self._database_id
if not db_id: if db_id is None:
db_id = self.resolve_database_id() db_id = self.resolve_database_id()
# Handle UUID string passed as database_id — resolve to numeric ID
if isinstance(db_id, str):
try:
db_id = int(db_id)
except (ValueError, TypeError):
db_id = self.resolve_database_uuid(db_id)
logger.reason("Submitting SQL to Superset SQL Lab", { logger.reason("Submitting SQL to Superset SQL Lab", {
"database_id": db_id, "database_id": db_id,
"sql_length": len(sql), "sql_length": len(sql),
"run_async": run_async,
}) })
payload = { payload = {
"database_id": db_id, "database_id": db_id,
"sql": sql, "sql": sql,
"runAsync": run_async, "client_id": uuid.uuid4().hex[:10],
"schema": None, "schema": None,
"tab": "translation-insert",
} }
try: try:
@@ -133,14 +168,22 @@ class SupersetSqlLabExecutor:
logger.explore("SQL Lab execute failed", {"error": str(e)}) logger.explore("SQL Lab execute failed", {"error": str(e)})
raise ValueError(f"Superset SQL Lab execute failed: {e}") raise ValueError(f"Superset SQL Lab execute failed: {e}")
# Parse response # Parse response — try multiple formats
result = response if isinstance(response, dict) else {} result = response if isinstance(response, dict) else {}
query_id = result.get("query_id") or result.get("id") query_id = (result.get("query_id")
or result.get("id")
or result.get("sql_lab_session_ref")
or (result.get("result") or {}).get("query_id")
or (result.get("result") or {}).get("id"))
status = result.get("status", "unknown") status = result.get("status", "unknown")
logger.reason("SQL Lab execute response", { logger.reason("SQL Lab execute response", {
"query_id": query_id, "query_id": query_id,
"status": status, "status": status,
"has_query_id": query_id is not None,
"has_errors": "errors" in result or "error" in result,
"response_keys": list(result.keys()) if isinstance(result, dict) else type(result).__name__,
"response_preview": str(result)[:1000] if isinstance(result, dict) else str(result)[:1000],
}) })
return { return {
@@ -149,13 +192,13 @@ class SupersetSqlLabExecutor:
"raw_response": result, "raw_response": result,
"database_id": db_id, "database_id": db_id,
} }
# [/DEF:execute_sql:Function] # #endregion execute_sql
# [DEF:poll_execution_status:Function] # #region poll_execution_status [C:4] [TYPE Function] [SEMANTICS translate,poll,status]
# @PURPOSE: Poll Superset for SQL execution status until completion or timeout. # @BRIEF Poll Superset for SQL execution status until completion or timeout.
# @PRE: query_id is a valid Superset query ID. # @PRE query_id is a valid Superset query ID.
# @POST: Returns final execution status dict. # @POST Returns final execution status dict with success/failure/timeout.
# @SIDE_EFFECT: Makes HTTP GET requests to Superset. # @SIDE_EFFECT Makes HTTP GET requests to Superset API.
def poll_execution_status( def poll_execution_status(
self, self,
query_id: str, query_id: str,
@@ -240,17 +283,17 @@ class SupersetSqlLabExecutor:
"error_message": f"Polling timed out after {max_polls} attempts", "error_message": f"Polling timed out after {max_polls} attempts",
"results": None, "results": None,
} }
# [/DEF:poll_execution_status:Function] # #endregion poll_execution_status
# [DEF:execute_and_poll:Function] # #region execute_and_poll [C:4] [TYPE Function] [SEMANTICS translate,sql,execute,poll]
# @PURPOSE: Execute SQL and wait for completion. One-shot convenience method. # @BRIEF Execute SQL and wait for completion. One-shot convenience method.
# @PRE: sql is valid SQL. # @PRE sql is valid SQL.
# @POST: Returns final execution result. # @POST Returns final execution result.
# @SIDE_EFFECT: Makes HTTP calls to Superset API. # @SIDE_EFFECT Makes HTTP calls to Superset API.
def execute_and_poll( def execute_and_poll(
self, self,
sql: str, sql: str,
database_id: Optional[int] = None, database_id: Optional[Union[int, str]] = None,
max_polls: int = 60, max_polls: int = 60,
poll_interval_seconds: float = 2.0, poll_interval_seconds: float = 2.0,
) -> Dict[str, Any]: ) -> Dict[str, Any]:
@@ -258,7 +301,6 @@ class SupersetSqlLabExecutor:
exec_result = self.execute_sql( exec_result = self.execute_sql(
sql=sql, sql=sql,
database_id=database_id, database_id=database_id,
run_async=True,
) )
query_id = exec_result.get("query_id") query_id = exec_result.get("query_id")
@@ -277,13 +319,13 @@ class SupersetSqlLabExecutor:
max_polls=max_polls, max_polls=max_polls,
poll_interval_seconds=poll_interval_seconds, poll_interval_seconds=poll_interval_seconds,
) )
# [/DEF:execute_and_poll:Function] # #endregion execute_and_poll
# [DEF:get_query_results:Function] # #region get_query_results [C:4] [TYPE Function] [SEMANTICS translate,query,results]
# @PURPOSE: Fetch the results of a completed query. # @BRIEF Fetch the results of a completed query from Superset.
# @PRE: query_id is a valid Superset query ID. # @PRE query_id is a valid Superset query ID.
# @POST: Returns query results if available. # @POST Returns query results if available, or error dict.
# @SIDE_EFFECT: Makes HTTP GET to Superset. # @SIDE_EFFECT Makes HTTP GET to Superset API.
def get_query_results(self, query_id: str) -> Dict[str, Any]: def get_query_results(self, query_id: str) -> Dict[str, Any]:
with belief_scope("SupersetSqlLabExecutor.get_query_results"): with belief_scope("SupersetSqlLabExecutor.get_query_results"):
client = self._get_client() client = self._get_client()
@@ -309,8 +351,8 @@ class SupersetSqlLabExecutor:
"error_message": str(e), "error_message": str(e),
"results": None, "results": None,
} }
# [/DEF:get_query_results:Function] # #endregion get_query_results
# [/DEF:SupersetSqlLabExecutor:Class] # #endregion SupersetSqlLabExecutor
# [/DEF:SupersetSqlLabExecutor:Module] # #endregion SupersetSqlLabExecutor

5
backend/src/schemas/__init__.py
View File

@@ -1,3 +1,2 @@
# [DEF:SchemasPackage:Package] # #region SchemasPackage [C:1] [TYPE Package]
# @PURPOSE: API schema package root. # #endregion SchemasPackage
# [/DEF:SchemasPackage:Package]

172
backend/src/schemas/translate.py
View File

@@ -1,9 +1,6 @@
# [DEF:TranslateSchemas:Module] # #region TranslateSchemas [C:2] [TYPE Module]
# @COMPLEXITY: 2 # @BRIEF Pydantic v2 schemas for translation API request/response serialization.
# @SEMANTICS: translate, schemas, pydantic # @LAYER Domain
# @PURPOSE: Pydantic v2 schemas for translation API request/response serialization.
# @LAYER: API
# @RELATION: DEPENDS_ON -> pydantic
from datetime import datetime from datetime import datetime
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
@@ -11,8 +8,7 @@ from pydantic import BaseModel, Field
import json import json
# [DEF:TranslateJobCreate:Class] # #region TranslateJobCreate [C:1] [TYPE Class]
# @PURPOSE: Schema for creating a new translation job.
class TranslateJobCreate(BaseModel): class TranslateJobCreate(BaseModel):
name: str name: str
description: Optional[str] = None description: Optional[str] = None
@@ -33,11 +29,11 @@ class TranslateJobCreate(BaseModel):
upsert_strategy: str = Field("MERGE", description="UPSERT strategy: MERGE, INSERT, UPDATE") upsert_strategy: str = Field("MERGE", description="UPSERT strategy: MERGE, INSERT, UPDATE")
dictionary_ids: Optional[List[str]] = Field(default_factory=list, description="Associated terminology dictionary IDs") dictionary_ids: Optional[List[str]] = Field(default_factory=list, description="Associated terminology dictionary IDs")
environment_id: Optional[str] = Field(None, description="Superset environment ID") environment_id: Optional[str] = Field(None, description="Superset environment ID")
# [/DEF:TranslateJobCreate:Class] target_database_id: Optional[str] = Field(None, description="Superset database ID for INSERT via SQL Lab")
# #endregion TranslateJobCreate
# [DEF:TranslateJobUpdate:Class] # #region TranslateJobUpdate [C:1] [TYPE Class]
# @PURPOSE: Schema for updating an existing translation job.
class TranslateJobUpdate(BaseModel): class TranslateJobUpdate(BaseModel):
name: Optional[str] = None name: Optional[str] = None
description: Optional[str] = None description: Optional[str] = None
@@ -59,11 +55,11 @@ class TranslateJobUpdate(BaseModel):
status: Optional[str] = None status: Optional[str] = None
dictionary_ids: Optional[List[str]] = None dictionary_ids: Optional[List[str]] = None
environment_id: Optional[str] = None environment_id: Optional[str] = None
# [/DEF:TranslateJobUpdate:Class] target_database_id: Optional[str] = None
# #endregion TranslateJobUpdate
# [DEF:TranslateJobResponse:Class] # #region TranslateJobResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for translation job API responses.
class TranslateJobResponse(BaseModel): class TranslateJobResponse(BaseModel):
id: str id: str
name: str name: str
@@ -89,24 +85,24 @@ class TranslateJobResponse(BaseModel):
updated_at: Optional[datetime] = None updated_at: Optional[datetime] = None
dictionary_ids: Optional[List[str]] = None dictionary_ids: Optional[List[str]] = None
environment_id: Optional[str] = None environment_id: Optional[str] = None
target_database_id: Optional[str] = None
class Config: class Config:
from_attributes = True from_attributes = True
# [/DEF:TranslateJobResponse:Class] # #endregion TranslateJobResponse
# [DEF:DatasourceColumnResponse:Class] # #region DatasourceColumnResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for datasource column metadata response.
class DatasourceColumnResponse(BaseModel): class DatasourceColumnResponse(BaseModel):
name: str name: str
type: Optional[str] = None type: Optional[str] = None
is_physical: bool = True is_physical: bool = True
is_dttm: bool = False is_dttm: bool = False
description: Optional[str] = None description: Optional[str] = None
# #endregion DatasourceColumnResponse
# [DEF:DatasourceColumnsResponse:Class] # #region DatasourceColumnsResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for datasource columns endpoint response.
class DatasourceColumnsResponse(BaseModel): class DatasourceColumnsResponse(BaseModel):
datasource_id: int datasource_id: int
datasource_name: Optional[str] = None datasource_name: Optional[str] = None
@@ -116,11 +112,10 @@ class DatasourceColumnsResponse(BaseModel):
class Config: class Config:
from_attributes = True from_attributes = True
# [/DEF:DatasourceColumnsResponse:Class] # #endregion DatasourceColumnsResponse
# [DEF:DuplicateJobResponse:Class] # #region DuplicateJobResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for duplicate job response.
class DuplicateJobResponse(BaseModel): class DuplicateJobResponse(BaseModel):
id: str id: str
name: str name: str
@@ -128,32 +123,29 @@ class DuplicateJobResponse(BaseModel):
class Config: class Config:
from_attributes = True from_attributes = True
# [/DEF:DuplicateJobResponse:Class] # #endregion DuplicateJobResponse
# [DEF:DictionaryCreate:Class] # #region DictionaryCreate [C:1] [TYPE Class]
# @PURPOSE: Schema for creating a new terminology dictionary.
class DictionaryCreate(BaseModel): class DictionaryCreate(BaseModel):
name: str name: str
description: Optional[str] = None description: Optional[str] = None
source_dialect: str source_dialect: str
target_dialect: str target_dialect: str
is_active: bool = True is_active: bool = True
# [/DEF:DictionaryCreate:Class] # #endregion DictionaryCreate
# [DEF:DictionaryImport:Class] # #region DictionaryImport [C:1] [TYPE Class]
# @PURPOSE: Schema for importing entries into a terminology dictionary.
class DictionaryImport(BaseModel): class DictionaryImport(BaseModel):
content: str = Field(..., description="CSV or TSV content as raw string") content: str = Field(..., description="CSV or TSV content as raw string")
delimiter: Optional[str] = Field(None, description="Detected or forced delimiter: ',' or '\\t'. Auto-detect if omitted.") delimiter: Optional[str] = Field(None, description="Detected or forced delimiter: ',' or '\\t'. Auto-detect if omitted.")
on_conflict: str = Field("overwrite", description="'overwrite' or 'keep_existing' or 'cancel'") on_conflict: str = Field("overwrite", description="'overwrite' or 'keep_existing' or 'cancel'")
preview_only: bool = Field(False, description="If true, return preview without applying") preview_only: bool = Field(False, description="If true, return preview without applying")
# [/DEF:DictionaryImport:Class] # #endregion DictionaryImport
# [DEF:DictionaryResponse:Class] # #region DictionaryResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for terminology dictionary API responses.
class DictionaryResponse(BaseModel): class DictionaryResponse(BaseModel):
id: str id: str
name: str name: str
@@ -168,11 +160,10 @@ class DictionaryResponse(BaseModel):
class Config: class Config:
from_attributes = True from_attributes = True
# [/DEF:DictionaryResponse:Class] # #endregion DictionaryResponse
# [DEF:DictionaryEntryCreate:Class] # #region DictionaryEntryCreate [C:1] [TYPE Class]
# @PURPOSE: Schema for adding/editing a dictionary entry.
class DictionaryEntryCreate(BaseModel): class DictionaryEntryCreate(BaseModel):
source_term: str = Field(..., description="Source term to translate") source_term: str = Field(..., description="Source term to translate")
target_term: str = Field(..., description="Target/translated term") target_term: str = Field(..., description="Target/translated term")
@@ -180,11 +171,10 @@ class DictionaryEntryCreate(BaseModel):
class Config: class Config:
from_attributes = True from_attributes = True
# [/DEF:DictionaryEntryCreate:Class] # #endregion DictionaryEntryCreate
# [DEF:DictionaryEntryResponse:Class] # #region DictionaryEntryResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for dictionary entry API responses.
class DictionaryEntryResponse(BaseModel): class DictionaryEntryResponse(BaseModel):
id: str id: str
dictionary_id: str dictionary_id: str
@@ -197,11 +187,10 @@ class DictionaryEntryResponse(BaseModel):
class Config: class Config:
from_attributes = True from_attributes = True
# [/DEF:DictionaryEntryResponse:Class] # #endregion DictionaryEntryResponse
# [DEF:DictionaryImportResult:Class] # #region DictionaryImportResult [C:1] [TYPE Class]
# @PURPOSE: Schema for dictionary import result.
class DictionaryImportResult(BaseModel): class DictionaryImportResult(BaseModel):
total: int = 0 total: int = 0
created: int = 0 created: int = 0
@@ -209,28 +198,26 @@ class DictionaryImportResult(BaseModel):
skipped: int = 0 skipped: int = 0
errors: List[Dict[str, Any]] = Field(default_factory=list) errors: List[Dict[str, Any]] = Field(default_factory=list)
preview: List[Dict[str, Any]] = Field(default_factory=list, description="Preview rows with conflict flags") preview: List[Dict[str, Any]] = Field(default_factory=list, description="Preview rows with conflict flags")
# [/DEF:DictionaryImportResult:Class] # #endregion DictionaryImportResult
# [DEF:PreviewRequest:Class] # #region PreviewRequest [C:1] [TYPE Class]
# @PURPOSE: Schema for triggering a translation preview.
class PreviewRequest(BaseModel): class PreviewRequest(BaseModel):
sample_size: int = Field(10, ge=1, le=100, description="Number of sample rows to preview") sample_size: int = Field(10, ge=1, le=100, description="Number of sample rows to preview")
prompt_template: Optional[str] = Field(None, description="Optional custom prompt template") prompt_template: Optional[str] = Field(None, description="Optional custom prompt template")
env_id: Optional[str] = Field(None, description="Superset environment ID for preview data fetch") env_id: Optional[str] = Field(None, description="Superset environment ID for preview data fetch")
# #endregion PreviewRequest
# [DEF:PreviewRowUpdate:Class] # #region PreviewRowUpdate [C:1] [TYPE Class]
# @PURPOSE: Schema for approving/editing/rejecting a preview row.
class PreviewRowUpdate(BaseModel): class PreviewRowUpdate(BaseModel):
action: str = Field(..., description="'approve', 'reject', or 'edit'") action: str = Field(..., description="'approve', 'reject', or 'edit'")
translation: Optional[str] = Field(None, description="Edited translation (required for 'edit' action)") translation: Optional[str] = Field(None, description="Edited translation (required for 'edit' action)")
feedback: Optional[str] = Field(None, description="Optional feedback/comment") feedback: Optional[str] = Field(None, description="Optional feedback/comment")
# [/DEF:PreviewRowUpdate:Class] # #endregion PreviewRowUpdate
# [DEF:PreviewAcceptResponse:Class] # #region PreviewAcceptResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for preview accept response.
class PreviewAcceptResponse(BaseModel): class PreviewAcceptResponse(BaseModel):
id: str id: str
job_id: str job_id: str
@@ -242,11 +229,10 @@ class PreviewAcceptResponse(BaseModel):
class Config: class Config:
from_attributes = True from_attributes = True
# [/DEF:PreviewAcceptResponse:Class] # #endregion PreviewAcceptResponse
# [DEF:CostEstimate:Class] # #region CostEstimate [C:1] [TYPE Class]
# @PURPOSE: Schema for cost estimation in preview response.
class CostEstimate(BaseModel): class CostEstimate(BaseModel):
sample_size: int = 0 sample_size: int = 0
sample_prompt_tokens: int = 0 sample_prompt_tokens: int = 0
@@ -256,11 +242,10 @@ class CostEstimate(BaseModel):
estimated_total_rows: int = 0 estimated_total_rows: int = 0
estimated_tokens: int = 0 estimated_tokens: int = 0
estimated_cost: float = 0.0 estimated_cost: float = 0.0
# [/DEF:CostEstimate:Class] # #endregion CostEstimate
# [DEF:TermCorrectionSubmit:Class] # #region TermCorrectionSubmit [C:1] [TYPE Class]
# @PURPOSE: Schema for submitting a term correction in a translation preview.
class TermCorrectionSubmit(BaseModel): class TermCorrectionSubmit(BaseModel):
source_term: str source_term: str
incorrect_target_term: str incorrect_target_term: str
@@ -268,29 +253,26 @@ class TermCorrectionSubmit(BaseModel):
dictionary_id: Optional[str] = Field(None, description="Target dictionary ID (language-filtered)") dictionary_id: Optional[str] = Field(None, description="Target dictionary ID (language-filtered)")
origin_run_id: Optional[str] = Field(None, description="Run ID from which this correction originated") origin_run_id: Optional[str] = Field(None, description="Run ID from which this correction originated")
origin_row_key: Optional[str] = Field(None, description="Row key within the run") origin_row_key: Optional[str] = Field(None, description="Row key within the run")
# [/DEF:TermCorrectionSubmit:Class] # #endregion TermCorrectionSubmit
# [DEF:TermCorrectionBulkSubmit:Class] # #region TermCorrectionBulkSubmit [C:1] [TYPE Class]
# @PURPOSE: Schema for submitting multiple term corrections atomically.
class TermCorrectionBulkSubmit(BaseModel): class TermCorrectionBulkSubmit(BaseModel):
corrections: List[TermCorrectionSubmit] corrections: List[TermCorrectionSubmit]
dictionary_id: str = Field(..., description="Target dictionary ID for all corrections") dictionary_id: str = Field(..., description="Target dictionary ID for all corrections")
# [/DEF:TermCorrectionBulkSubmit:Class] # #endregion TermCorrectionBulkSubmit
# [DEF:CorrectionConflictResult:Class] # #region CorrectionConflictResult [C:1] [TYPE Class]
# @PURPOSE: Schema for reporting conflicts in correction submission.
class CorrectionConflictResult(BaseModel): class CorrectionConflictResult(BaseModel):
source_term: str source_term: str
existing_target_term: str existing_target_term: str
submitted_target_term: str submitted_target_term: str
action: str = "keep_existing" action: str = "keep_existing"
# [/DEF:CorrectionConflictResult:Class] # #endregion CorrectionConflictResult
# [DEF:CorrectionSubmitResponse:Class] # #region CorrectionSubmitResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for correction submission response.
class CorrectionSubmitResponse(BaseModel): class CorrectionSubmitResponse(BaseModel):
entry_id: Optional[str] = None entry_id: Optional[str] = None
action: str # "created", "updated", "conflict_detected", "skipped" action: str # "created", "updated", "conflict_detected", "skipped"
@@ -298,20 +280,18 @@ class CorrectionSubmitResponse(BaseModel):
target_term: str target_term: str
conflict: Optional[CorrectionConflictResult] = None conflict: Optional[CorrectionConflictResult] = None
message: Optional[str] = None message: Optional[str] = None
# [/DEF:CorrectionSubmitResponse:Class] # #endregion CorrectionSubmitResponse
# [DEF:ScheduleConfig:Class] # #region ScheduleConfig [C:1] [TYPE Class]
# @PURPOSE: Schema for configuring a recurring translation schedule.
class ScheduleConfig(BaseModel): class ScheduleConfig(BaseModel):
cron_expression: str = Field(..., description="Cron expression for scheduling (e.g. '0 2 * * *')") cron_expression: str = Field(..., description="Cron expression for scheduling (e.g. '0 2 * * *')")
timezone: str = Field("UTC", description="Timezone for the cron schedule (e.g. 'UTC', 'Europe/Moscow')") timezone: str = Field("UTC", description="Timezone for the cron schedule (e.g. 'UTC', 'Europe/Moscow')")
is_active: bool = True is_active: bool = True
# [/DEF:ScheduleConfig:Class] # #endregion ScheduleConfig
# [DEF:ScheduleResponse:Class] # #region ScheduleResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for schedule API responses.
class ScheduleResponse(BaseModel): class ScheduleResponse(BaseModel):
id: str id: str
job_id: str job_id: str
@@ -326,21 +306,19 @@ class ScheduleResponse(BaseModel):
class Config: class Config:
from_attributes = True from_attributes = True
# [/DEF:ScheduleResponse:Class] # #endregion ScheduleResponse
# [DEF:NextExecutionResponse:Class] # #region NextExecutionResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for next execution preview.
class NextExecutionResponse(BaseModel): class NextExecutionResponse(BaseModel):
job_id: str job_id: str
cron_expression: str cron_expression: str
timezone: str timezone: str
next_executions: List[str] = [] next_executions: List[str] = []
# [/DEF:NextExecutionResponse:Class] # #endregion NextExecutionResponse
# [DEF:TranslationRunResponse:Class] # #region TranslationRunResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for translation run API responses.
class TranslationRunResponse(BaseModel): class TranslationRunResponse(BaseModel):
id: str id: str
job_id: str job_id: str
@@ -364,22 +342,20 @@ class TranslationRunResponse(BaseModel):
class Config: class Config:
from_attributes = True from_attributes = True
# [/DEF:TranslationRunResponse:Class] # #endregion TranslationRunResponse
# [DEF:RunDetailResponse:Class] # #region RunDetailResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for detailed run response including records, events, and config snapshot.
class RunDetailResponse(BaseModel): class RunDetailResponse(BaseModel):
run: TranslationRunResponse run: TranslationRunResponse
records: List[Dict[str, Any]] = Field(default_factory=list, description="Paginated translation records") records: List[Dict[str, Any]] = Field(default_factory=list, description="Paginated translation records")
events: List[Dict[str, Any]] = Field(default_factory=list, description="Run events") events: List[Dict[str, Any]] = Field(default_factory=list, description="Run events")
event_invariants: Optional[Dict[str, Any]] = None event_invariants: Optional[Dict[str, Any]] = None
batch_count: int = 0 batch_count: int = 0
# [/DEF:RunDetailResponse:Class> # #endregion RunDetailResponse
# [DEF:RunHistoryFilter:Class] # #region RunHistoryFilter [C:1] [TYPE Class]
# @PURPOSE: Schema for filtering run history.
class RunHistoryFilter(BaseModel): class RunHistoryFilter(BaseModel):
job_id: Optional[str] = None job_id: Optional[str] = None
status: Optional[str] = None status: Optional[str] = None
@@ -389,21 +365,19 @@ class RunHistoryFilter(BaseModel):
date_to: Optional[datetime] = None date_to: Optional[datetime] = None
page: int = 1 page: int = 1
page_size: int = 20 page_size: int = 20
# [/DEF:RunHistoryFilter:Class> # #endregion RunHistoryFilter
# [DEF:RunListResponse:Class] # #region RunListResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for paginated run list response.
class RunListResponse(BaseModel): class RunListResponse(BaseModel):
items: List[TranslationRunResponse] = [] items: List[TranslationRunResponse] = []
total: int = 0 total: int = 0
page: int = 1 page: int = 1
page_size: int = 20 page_size: int = 20
# [/DEF:RunListResponse:Class> # #endregion RunListResponse
# [DEF:AggregatedMetricsResponse:Class] # #region AggregatedMetricsResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for aggregated per-job metrics.
class AggregatedMetricsResponse(BaseModel): class AggregatedMetricsResponse(BaseModel):
job_id: str job_id: str
total_runs: int = 0 total_runs: int = 0
@@ -419,11 +393,10 @@ class AggregatedMetricsResponse(BaseModel):
avg_duration_ms: Optional[float] = None avg_duration_ms: Optional[float] = None
last_run_at: Optional[datetime] = None last_run_at: Optional[datetime] = None
next_scheduled_run: Optional[datetime] = None next_scheduled_run: Optional[datetime] = None
# [/DEF:AggregatedMetricsResponse:Class] # #endregion AggregatedMetricsResponse
# [DEF:PreviewRow:Class] # #region PreviewRow [C:1] [TYPE Class]
# @PURPOSE: A single row in a translation preview showing original and translated content side by side.
class PreviewRow(BaseModel): class PreviewRow(BaseModel):
id: str id: str
source_sql: Optional[str] = None source_sql: Optional[str] = None
@@ -433,11 +406,10 @@ class PreviewRow(BaseModel):
source_object_name: Optional[str] = None source_object_name: Optional[str] = None
status: str = "PENDING" status: str = "PENDING"
feedback: Optional[str] = None feedback: Optional[str] = None
# [/DEF:PreviewRow:Class] # #endregion PreviewRow
# [DEF:TranslationPreviewResponse:Class] # #region TranslationPreviewResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for translation preview session API responses.
class TranslationPreviewResponse(BaseModel): class TranslationPreviewResponse(BaseModel):
id: str id: str
job_id: str job_id: str
@@ -450,11 +422,10 @@ class TranslationPreviewResponse(BaseModel):
class Config: class Config:
from_attributes = True from_attributes = True
# [/DEF:TranslationPreviewResponse:Class] # #endregion TranslationPreviewResponse
# [DEF:TranslationBatchResponse:Class] # #region TranslationBatchResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for translation batch API responses.
class TranslationBatchResponse(BaseModel): class TranslationBatchResponse(BaseModel):
id: str id: str
run_id: str run_id: str
@@ -469,11 +440,10 @@ class TranslationBatchResponse(BaseModel):
class Config: class Config:
from_attributes = True from_attributes = True
# [/DEF:TranslationBatchResponse:Class] # #endregion TranslationBatchResponse
# [DEF:MetricsResponse:Class] # #region MetricsResponse [C:1] [TYPE Class]
# @PURPOSE: Schema for translation metrics API responses.
class MetricsResponse(BaseModel): class MetricsResponse(BaseModel):
job_id: str job_id: str
snapshot_date: datetime snapshot_date: datetime
@@ -490,6 +460,6 @@ class MetricsResponse(BaseModel):
class Config: class Config:
from_attributes = True from_attributes = True
# [/DEF:MetricsResponse:Class] # #endregion MetricsResponse
# [/DEF:TranslateSchemas:Module] # #endregion TranslateSchemas