busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(health): suppress 404 when health monitor disabled + fix audit test assertion

Browse Source 
- Sidebar.svelte: defer healthStore.refresh() until feature flags confirm
  health_monitor is enabled; skip entirely when disabled
- health.js: remove throw error from catch block — log and return null
  instead, preventing 'Uncaught (in promise)' on expected 404
- test_report_audit_immutability.py: fix mock assertions —
  audit_service uses logger.reason/reflect/explore, not logger.info
- HealthStore and Sidebar now produce zero network noise when
  FEATURES__HEALTH_MONITOR=false
This commit is contained in:
busya
2026-05-17 14:18:02 +03:00
parent 58ac89c21e
commit cd868df261
141 changed files with 9631 additions and 10165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

258
backend/src/plugins/git_plugin.py
View File

@@ -1,4 +1,4 @@
# #region GitPluginModule [TYPE Module] [SEMANTICS git, versioning, deploy, sync, superset]
# #region GitPluginModule [C:4] [TYPE Module] [SEMANTICS git, versioning, deploy, sync, superset, backup, transactional]
#
# @BRIEF Предоставляет плагин для версионирования и развертывания дашбордов Superset.
# @LAYER: Plugin
@@ -10,10 +10,13 @@
#
# @INVARIANT: Все операции с Git должны выполняться через GitService.
# @CONSTRAINT: Плагин работает только с распакованными YAML-экспортами Superset.
# @INVARIANT: _handle_sync сохраняет backup управляемых директорий перед удалением;
# при ошибке распаковки backup восстанавливается.
import io
import os
import shutil
import time
import zipfile
from pathlib import Path
from typing import Any
@@ -62,52 +65,33 @@ class GitPlugin(PluginBase):
# endregion __init__
@property
# region id [TYPE Function]
# @PURPOSE: Returns the plugin identifier.
# @PRE: GitPlugin is initialized.
# @POST: Returns 'git-integration'.
# region id [C:1] [TYPE Function]
def id(self) -> str:
with belief_scope("GitPlugin.id"):
return "git-integration"
return "git-integration"
# endregion id
@property
# region name [TYPE Function]
# @PURPOSE: Returns the plugin name.
# @PRE: GitPlugin is initialized.
# @POST: Returns the human-readable name.
# region name [C:1] [TYPE Function]
def name(self) -> str:
with belief_scope("GitPlugin.name"):
return "Git Integration"
return "Git Integration"
# endregion name
@property
# region description [TYPE Function]
# @PURPOSE: Returns the plugin description.
# @PRE: GitPlugin is initialized.
# @POST: Returns the plugin's purpose description.
# region description [C:1] [TYPE Function]
def description(self) -> str:
with belief_scope("GitPlugin.description"):
return "Version control for Superset dashboards"
return "Version control for Superset dashboards"
# endregion description
@property
# region version [TYPE Function]
# @PURPOSE: Returns the plugin version.
# @PRE: GitPlugin is initialized.
# @POST: Returns the version string.
# region version [C:1] [TYPE Function]
def version(self) -> str:
with belief_scope("GitPlugin.version"):
return "0.1.0"
return "0.1.0"
# endregion version
@property
# region ui_route [TYPE Function]
# @PURPOSE: Returns the frontend route for the git plugin.
# @RETURN: str - "/git"
# region ui_route [C:1] [TYPE Function]
def ui_route(self) -> str:
with belief_scope("GitPlugin.ui_route"):
return "/git"
return "/git"
# endregion ui_route
# region get_schema [TYPE Function]
@@ -129,21 +113,13 @@ class GitPlugin(PluginBase):
}
# endregion get_schema
# region initialize [TYPE Function]
# @PURPOSE: Выполняет начальную настройку плагина.
# @PRE: GitPlugin is initialized.
# @POST: Плагин готов к выполнению задач.
# region initialize [C:1] [TYPE Function]
async def initialize(self):
with belief_scope("GitPlugin.initialize"):
app_logger.reason("Initializing Git Integration Plugin logic.", extra={"src": "GitPlugin.initialize"})
app_logger.reason("Initializing Git Integration Plugin logic.", extra={"src": "GitPlugin.initialize"})
# endregion initialize
# region execute [TYPE Function]
# @PURPOSE: Основной метод выполнения задач плагина с поддержкой TaskContext.
# @PRE: task_data содержит 'operation' и 'dashboard_id'.
# @POST: Возвращает результат выполнения операции.
# @PARAM: task_data (Dict[str, Any]) - Данные задачи.
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @RETURN: Dict[str, Any] - Статус и сообщение.
# region execute [C:3] [TYPE Function]
# @PURPOSE: Main task executor with TaskContext support.
# @RELATION: CALLS -> self._handle_sync
# @RELATION: CALLS -> self._handle_deploy
async def execute(self, task_data: dict[str, Any], context: TaskContext | None = None) -> dict[str, Any]:
@@ -176,41 +152,48 @@ class GitPlugin(PluginBase):
return result
# endregion execute
# region _handle_sync [TYPE Function]
# @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий.
# region _handle_sync [C:4] [TYPE Function] [SEMANTICS git,sync,backup,transactional]
# @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий с backup/restore.
# @PRE: Репозиторий для дашборда должен существовать.
# @POST: Файлы в репозитории обновлены до текущего состояния в Superset.
# @PARAM: dashboard_id (int) - ID дашборда.
# @PARAM: source_env_id (Optional[str]) - ID исходного окружения.
# @RETURN: Dict[str, str] - Результат синхронизации.
# Управляемые директории backup-ируются перед удалением; при ошибке распаковки backup восстанавливается.
# @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория.
# Создаёт временный backup в /tmp/ss-tools-backup-{dashboard_id}-{timestamp}/.
# @RETURN: Dict[str, str] - Результат синхронизации.
# @RELATION: CALLS -> src.services.git_service.GitService.get_repo
# @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard
async def _handle_sync(self, dashboard_id: int, source_env_id: str | None = None, log=None, git_log=None, superset_log=None) -> dict[str, str]:
with belief_scope("GitPlugin._handle_sync"):
try:
# 1. Получение репозитория
repo = self.git_service.get_repo(dashboard_id)
repo_path = Path(repo.working_dir)
git_log.info(f"Target repo path: {repo_path}")
# 2. Настройка клиента Superset
env = self._get_env(source_env_id)
client = SupersetClient(env)
client.authenticate()
# 3. Экспорт дашборда
superset_log.info(f"Exporting dashboard {dashboard_id} from {env.name}")
zip_bytes, _ = client.export_dashboard(dashboard_id)
# 4. Распаковка с выравниванием структуры (flattening)
git_log.info(f"Unpacking export to {repo_path}")
# Список папок/файлов, которые мы ожидаем от Superset
managed_dirs = ["dashboards", "charts", "datasets", "databases"]
managed_files = ["metadata.yaml"]
# Очистка старых данных перед распаковкой, чтобы не оставалось "призраков"
# Fix 1: Create backup before deleting managed files
backup_dir = Path(f"/tmp/ss-tools-backup-{dashboard_id}-{time.time_ns()}")
for d in managed_dirs:
d_path = repo_path / d
if d_path.exists() and d_path.is_dir():
shutil.copytree(d_path, backup_dir / d)
for f in managed_files:
f_path = repo_path / f
if f_path.exists():
(backup_dir / f).parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(f_path, backup_dir / f)
# Delete old managed files
for d in managed_dirs:
d_path = repo_path / d
if d_path.exists() and d_path.is_dir():
@@ -220,30 +203,45 @@ class GitPlugin(PluginBase):
if f_path.exists():
f_path.unlink()
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
# Superset экспортирует всё в подпапку dashboard_export_timestamp/
# Нам нужно найти это имя папки
namelist = zf.namelist()
if not namelist:
raise ValueError("Export ZIP is empty")
try:
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
namelist = zf.namelist()
if not namelist:
raise ValueError("Export ZIP is empty")
root_folder = namelist[0].split('/')[0]
git_log.info(f"Detected root folder in ZIP: {root_folder}")
for member in zf.infolist():
if member.filename.startswith(root_folder + "/") and len(member.filename) > len(root_folder) + 1:
relative_path = member.filename[len(root_folder)+1:]
target_path = repo_path / relative_path
if member.is_dir():
target_path.mkdir(parents=True, exist_ok=True)
else:
target_path.parent.mkdir(parents=True, exist_ok=True)
with zf.open(member) as source, open(target_path, "wb") as target:
shutil.copyfileobj(source, target)
except Exception:
# Fix 1: Restore backup on unzip failure
if backup_dir.exists():
for d in managed_dirs:
src = backup_dir / d
if src.exists():
dst = repo_path / d
if dst.exists():
shutil.rmtree(dst)
shutil.copytree(src, dst)
for f in managed_files:
src = backup_dir / f
if src.exists():
dst = repo_path / f
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dst)
raise
root_folder = namelist[0].split('/')[0]
git_log.info(f"Detected root folder in ZIP: {root_folder}")
# Fix 1: Remove backup on success
if backup_dir.exists():
shutil.rmtree(backup_dir, ignore_errors=True)
for member in zf.infolist():
if member.filename.startswith(root_folder + "/") and len(member.filename) > len(root_folder) + 1:
# Убираем префикс папки
relative_path = member.filename[len(root_folder)+1:]
target_path = repo_path / relative_path
if member.is_dir():
target_path.mkdir(parents=True, exist_ok=True)
else:
target_path.parent.mkdir(parents=True, exist_ok=True)
with zf.open(member) as source, open(target_path, "wb") as target:
shutil.copyfileobj(source, target)
# 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff)
try:
repo.git.add(A=True)
app_logger.reason("Changes staged in git", extra={"src": "_handle_sync"})
@@ -258,35 +256,21 @@ class GitPlugin(PluginBase):
raise
# endregion _handle_sync
# region _handle_deploy [TYPE Function]
# @PURPOSE: Упаковывает репозиторий в ZIP и импортирует в целевое окружение Superset.
# @PRE: environment_id должен соответствовать настроенному окружению.
# @POST: Дашборд импортирован в целевой Superset.
# @PARAM: dashboard_id (int) - ID дашборда.
# @PARAM: env_id (str) - ID целевого окружения.
# @PARAM: log - Main logger instance.
# @PARAM: git_log - Git-specific logger instance.
# @PARAM: superset_log - Superset API-specific logger instance.
# @RETURN: Dict[str, Any] - Результат деплоя.
# @SIDE_EFFECT: Создает и удаляет временный ZIP-файл.
# region _handle_deploy [C:4] [TYPE Function] [SEMANTICS git,deploy,zip]
# @PURPOSE: Packages repository into ZIP and imports into target Superset environment.
# @POST: Dashboard imported into target Superset.
# @SIDE_EFFECT: Creates and removes temporary ZIP file.
# @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard
async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> dict[str, Any]:
with belief_scope("GitPlugin._handle_deploy"):
try:
if not env_id:
raise ValueError("Target environment ID required for deployment")
# 1. Получение репозитория
repo = self.git_service.get_repo(dashboard_id)
repo_path = Path(repo.working_dir)
# 2. Упаковка в ZIP
git_log.info(f"Packing repository {repo_path} for deployment.")
zip_buffer = io.BytesIO()
# Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/)
root_dir_name = f"dashboard_export_{dashboard_id}"
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
for root, dirs, files in os.walk(repo_path):
if ".git" in dirs:
@@ -295,21 +279,14 @@ class GitPlugin(PluginBase):
if file == ".git" or file.endswith(".zip"):
continue
file_path = Path(root) / file
# Prepend the root directory name to the archive path
arcname = Path(root_dir_name) / file_path.relative_to(repo_path)
zf.write(file_path, arcname)
zip_buffer.seek(0)
# 3. Настройка клиента Superset
env = self.config_manager.get_environment(env_id)
if not env:
raise ValueError(f"Environment {env_id} not found")
client = SupersetClient(env)
client.authenticate()
# 4. Импорт
temp_zip_path = repo_path / f"deploy_{dashboard_id}.zip"
git_log.info(f"Saving temporary zip to {temp_zip_path}")
with open(temp_zip_path, "wb") as f:
@@ -329,11 +306,12 @@ class GitPlugin(PluginBase):
raise
# endregion _handle_deploy
# region _get_env [TYPE Function]
# region _get_env [C:4] [TYPE Function] [SEMANTICS env,config,strict]
# @PURPOSE: Вспомогательный метод для получения конфигурации окружения.
# @PARAM: env_id (Optional[str]) - ID окружения.
# @PRE: env_id is a string or None.
# @POST: Returns an Environment object from config or DB.
# When env_id is explicitly provided and not found, raises ValueError (no fallback).
# @RETURN: Environment - Объект конфигурации окружения.
def _get_env(self, env_id: str | None = None):
with belief_scope("GitPlugin._get_env"):
@@ -346,54 +324,46 @@ class GitPlugin(PluginBase):
app_logger.reflect(f"Found environment by ID in ConfigManager: {env.name}", extra={"src": "_get_env"})
return env
# Priority 2: Database (DeploymentEnvironment)
from src.core.database import SessionLocal
from src.models.git import DeploymentEnvironment
# Priority 2: Database (DeploymentEnvironment)
from src.core.database import SessionLocal
from src.models.git import DeploymentEnvironment
db = SessionLocal()
try:
if env_id:
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()
else:
# If no ID, try to find active or any environment in DB
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active).first()
if not db_env:
db_env = db.query(DeploymentEnvironment).first()
db = SessionLocal()
try:
if env_id:
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()
else:
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active).first()
if not db_env:
db_env = db.query(DeploymentEnvironment).first()
if db_env:
app_logger.reflect(f"Found environment in DB: {db_env.name}", extra={"src": "_get_env"})
from src.core.config_models import Environment
# Use token as password for SupersetClient
return Environment(
id=db_env.id,
name=db_env.name,
url=db_env.superset_url,
username="admin",
password=db_env.superset_token,
verify_ssl=True
)
finally:
db.close()
if db_env:
app_logger.reflect(f"Found environment in DB: {db_env.name}", extra={"src": "_get_env"})
from src.core.config_models import Environment
return Environment(
id=db_env.id,
name=db_env.name,
url=db_env.superset_url,
username="admin",
password=db_env.superset_token,
verify_ssl=True
)
# Priority 3: ConfigManager Default (if no env_id provided)
envs = self.config_manager.get_environments()
if envs:
if env_id:
# If env_id was provided but not found in DB or specifically by ID in config,
# but we have other envs, maybe it's one of them?
env = next((e for e in envs if e.id == env_id), None)
if env:
app_logger.reflect(f"Found environment {env_id} in ConfigManager list", extra={"src": "_get_env"})
return env
# Fix 7: Strict check — if env_id was explicitly provided but not found, raise immediately
if env_id:
raise ValueError(f"Environment '{env_id}' not found in ConfigManager or database")
finally:
db.close()
if not env_id:
app_logger.reflect(f"Using first environment from ConfigManager: {envs[0].name}", extra={"src": "_get_env"})
return envs[0]
# Priority 3: fallback to first env only when no specific env_id was requested
envs = self.config_manager.get_environments()
if envs and not env_id:
app_logger.reflect(f"Using first environment from ConfigManager: {envs[0].name}", extra={"src": "_get_env"})
return envs[0]
app_logger.error(f"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}")
raise ValueError("No environments configured. Please add a Superset Environment in Settings.")
app_logger.error(f"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}")
raise ValueError("No environments configured. Please add a Superset Environment in Settings.")
# endregion _get_env
# endregion initialize
# #endregion GitPlugin
# #endregion GitPluginModule

1
backend/src/plugins/translate/__init__.py
View File

@@ -1,3 +1,2 @@
# #region TranslatePluginPackage [TYPE Package] [SEMANTICS translate, plugin, package, sql, dashboard]
# @BRIEF Translation plugin package for LLM-based cross-Superset SQL/dashboard translation.
# #endregion TranslatePluginPackage

44
backend/src/plugins/translate/_token_budget.py
View File

@@ -9,24 +9,24 @@
# Fixed batch_size of 50 — causes truncation on long-content rows.
# #region DEFAULT_CONTEXT_WINDOW [TYPE Constant]
# @BRIEF Default context window for DeepSeek v4 Flash: up to 64K tokens.
DEFAULT_CONTEXT_WINDOW = 64000
# #endregion DEFAULT_CONTEXT_WINDOW
# #region DEFAULT_MAX_OUTPUT_TOKENS [TYPE Constant]
# @BRIEF Default max_tokens setting for LLM output (16384 — sufficient for 50 rows x 4 languages).
# #region DEFAULT_MAX_OUTPUT_TOKENS [TYPE Constant]
DEFAULT_MAX_OUTPUT_TOKENS = 16384
# #endregion DEFAULT_MAX_OUTPUT_TOKENS
# #region REASONING_OVERHEAD [TYPE Constant]
# @BRIEF CoT reasoning overhead tokens for DeepSeek models (~2000 tokens for chain-of-thought).
# #region REASONING_OVERHEAD [TYPE Constant]
REASONING_OVERHEAD = 2000
# #endregion REASONING_OVERHEAD
# #region PROVIDER_DEFAULTS [TYPE Constant]
# @BRIEF Provider-aware defaults for context_window and max_output_tokens.
# Maps model name (or "default" fallback) to capacity limits.
# @RATIONALE: Different providers have drastically different context windows and
# @RATIONALE Different providers have drastically different context windows and
# output limits. Using a single default for all causes either wasted
# capacity (underestimation) or truncation (overestimation).
PROVIDER_DEFAULTS: dict[str, dict[str, int]] = {
@@ -38,46 +38,46 @@ PROVIDER_DEFAULTS: dict[str, dict[str, int]] = {
"default": {"context_window": 64000, "max_output_tokens": 16384},
}
# #endregion PROVIDER_DEFAULTS
# Increased from 60 to 120 because SQL/dashboard text and JSON structure need more.
# #region OUTPUT_PER_ROW_PER_LANG [TYPE Constant]
# @BRIEF Estimated output tokens per row per language in JSON response format.
# Increased from 60 to 120 because SQL/dashboard text and JSON structure need more.
OUTPUT_PER_ROW_PER_LANG = 120
# #endregion OUTPUT_PER_ROW_PER_LANG
# #endregion JSON_OVERHEAD_PER_ROW
# #region JSON_OVERHEAD_PER_ROW [TYPE Constant]
# @BRIEF Estimated overhead tokens for JSON keys, brackets, and formatting per row.
JSON_OVERHEAD_PER_ROW = 50
# #endregion JSON_OVERHEAD_PER_ROW
# #endregion PROMPT_BASE_TOKENS
# #region PROMPT_BASE_TOKENS [TYPE Constant]
# @BRIEF Base tokens for system prompt + instructions + dictionary section + JSON format specification.
# Increased from 300 to 600 to account for longer template, system msg, and dict preamble.
PROMPT_BASE_TOKENS = 600
# #endregion PROMPT_BASE_TOKENS
# @BRIEF Cap for dictionary tokens in estimation to avoid overestimating.
# #region DICT_TOKENS_PER_ENTRY [TYPE Constant]
# @BRIEF Estimated tokens per dictionary entry in the glossary section.
DICT_TOKENS_PER_ENTRY = 20
# #endregion DICT_TOKENS_PER_ENTRY
# #region DICT_TOKENS_MAX [TYPE Constant]
# @BRIEF Cap for dictionary tokens in estimation to avoid overestimating.
DICT_TOKENS_MAX = 5000
# #endregion DICT_TOKENS_MAX
MIN_MAX_TOKENS = 4096
# #region CHARS_PER_TOKEN_MIXED [TYPE Constant]
# @BRIEF Characters per token for mixed Russian/English text (empirical ~2.2 for mixed, ~2 for Russian, ~4 for English).
CHARS_PER_TOKEN_MIXED = 2.2
# #endregion CHARS_PER_TOKEN_MIXED
MAX_OUTPUT_HEADROOM = 3000
# #region MIN_MAX_TOKENS [TYPE Constant]
# @BRIEF Minimum max_tokens to allow (4096 ensures reasonable output even for small batches).
MIN_MAX_TOKENS = 4096
# #endregion MIN_MAX_TOKENS
# #endregion MIN_MAX_TOKENS
# #region MAX_OUTPUT_HEADROOM [TYPE Constant]
# @BRIEF Extra headroom added to max_output_needed beyond the estimate (3000 = 10-20% for variance).
# Increased from 1000 to 3000 because SQL/dashboard text output varies significantly.
MAX_OUTPUT_HEADROOM = 3000
# #endregion MAX_OUTPUT_HEADROOM

139
backend/src/plugins/translate/dictionary.py
View File

@@ -1,13 +1,16 @@
# #region DictionaryManagerModule [C:4] [TYPE Module] [SEMANTICS sqlalchemy, translate, dictionary, batch, term]
# @BRIEF Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TerminologyDictionary:Class]
# @RELATION DEPENDS_ON -> [DictionaryEntry:Class]
# @RELATION DEPENDS_ON -> [TranslationJobDictionary:Class]
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationJob:Class]
# @RATIONALE: C4 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term.
# @REJECTED: Pure C3 CRUD without state guards would allow orphaned job-dictionary links.
# @REJECTED: "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing.
# @RELATION DEPENDS_ON -> [TranslationJob:Class]
# @RELATION DEPENDS_ON -> [TranslationJob:Class]
# @RELATION DEPENDS_ON -> [TranslationJob:Class]
# @RATIONALE C4 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term.
# @REJECTED "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing.
# @REJECTED "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing.
# @POST Dictionary and entry mutations are persisted with conflict detection.
# @PRE Database session is open and valid. Dictionary manager is properly initialized.
# @SIDE_EFFECT Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
import csv
import io
@@ -42,19 +45,18 @@ def _validate_bcp47(tag: str, field_name: str) -> None:
f"{field_name} is not a valid BCP-47 tag: '{tag}'. "
"Expected format like 'en', 'ru', 'zh-CN', 'pt-BR'."
)
# #endregion _validate_bcp47
# #region DictionaryManager [C:4] [TYPE Class]
# @BRIEF Manages terminology dictionaries and their entries with referential integrity.
# @PRE: Database session is open and valid.
# @POST: Dictionary and entry mutations are persisted with conflict detection.
# @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
# @PRE Database session is open and valid.
# @POST Dictionary and entry mutations are persisted with conflict detection.
# @SIDE_EFFECT Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
# @RELATION DEPENDS_ON -> [TerminologyDictionary:Class], DEPENDS_ON -> [DictionaryEntry:Class], DEPENDS_ON -> [TranslationJobDictionary:Class], DEPENDS_ON -> [TranslationJob:Class]
class DictionaryManager:
# region DictionaryManager.create_dictionary [TYPE Function]
# @PURPOSE: Create a new terminology dictionary (language independent at dictionary level; language pairs are per-entry).
# @PRE: name is provided.
# @POST: New TerminologyDictionary row is created and returned.
# @PURPOSE Create a new terminology dictionary (language independent at dictionary level; language pairs are per-entry).
# @PRE name is provided.
# @POST New TerminologyDictionary row is created and returned.
@staticmethod
def create_dictionary(
db: Session, name: str,
@@ -81,9 +83,9 @@ class DictionaryManager:
# endregion DictionaryManager.create_dictionary
# region DictionaryManager.update_dictionary [TYPE Function]
# @PURPOSE: Update an existing terminology dictionary.
# @PRE: dict_id exists in terminology_dictionaries table.
# @POST: Dictionary metadata is updated and returned.
# @PURPOSE Update an existing terminology dictionary.
# @PRE dict_id exists in terminology_dictionaries table.
# @POST Dictionary metadata is updated and returned.
@staticmethod
def update_dictionary(
db: Session, dict_id: str, name: str | None = None,
@@ -112,10 +114,10 @@ class DictionaryManager:
# endregion DictionaryManager.update_dictionary
# region DictionaryManager.delete_dictionary [TYPE Function]
# @PURPOSE: Delete a dictionary, blocked if attached to active/scheduled jobs.
# @PRE: dict_id exists.
# @POST: Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.
# @SIDE_EFFECT: Deletes TerminologyDictionary row and all DictionaryEntry rows.
# @PURPOSE Delete a dictionary, blocked if attached to active/scheduled jobs.
# @PRE dict_id exists.
# @POST Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.
# @SIDE_EFFECT Deletes TerminologyDictionary row and all DictionaryEntry rows.
@staticmethod
def delete_dictionary(db: Session, dict_id: str) -> None:
with belief_scope("DictionaryManager.delete_dictionary"):
@@ -155,9 +157,9 @@ class DictionaryManager:
# endregion DictionaryManager.delete_dictionary
# region DictionaryManager.get_dictionary [TYPE Function]
# @PURPOSE: Get a single dictionary by ID with entry count.
# @PRE: dict_id exists.
# @POST: Returns dict with dictionary + entry_count or raises ValueError.
# @PURPOSE Get a single dictionary by ID with entry count.
# @PRE dict_id exists.
# @POST Returns dict with dictionary + entry_count or raises ValueError.
@staticmethod
def get_dictionary(db: Session, dict_id: str) -> TerminologyDictionary:
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
@@ -167,9 +169,9 @@ class DictionaryManager:
# endregion DictionaryManager.get_dictionary
# region DictionaryManager.list_dictionaries [TYPE Function]
# @PURPOSE: List dictionaries with pagination and entry counts.
# @PRE: page >= 1, page_size between 1 and 100.
# @POST: Returns (list of dicts, total_count).
# @PURPOSE List dictionaries with pagination and entry counts.
# @PRE page >= 1, page_size between 1 and 100.
# @POST Returns (list of dicts, total_count).
@staticmethod
def list_dictionaries(
db: Session, page: int = 1, page_size: int = 20,
@@ -186,9 +188,9 @@ class DictionaryManager:
# endregion DictionaryManager.list_dictionaries
# region DictionaryManager.add_entry [TYPE Function]
# @PURPOSE: Add an entry to a dictionary with language-pair-aware uniqueness validation.
# @PRE: dict_id exists. source_term, target_term are non-empty. source_language, target_language are valid BCP-47.
# @POST: New DictionaryEntry row is created or raises on duplicate.
# @PURPOSE Add an entry to a dictionary with language-pair-aware uniqueness validation.
# @PRE dict_id exists. source_term, target_term are non-empty. source_language, target_language are valid BCP-47.
# @POST New DictionaryEntry row is created or raises on duplicate.
@staticmethod
def add_entry(
db: Session, dict_id: str, source_term: str, target_term: str,
@@ -237,9 +239,9 @@ class DictionaryManager:
# endregion DictionaryManager.add_entry
# region DictionaryManager.edit_entry [TYPE Function]
# @PURPOSE: Edit an existing dictionary entry with language-pair-aware duplicate check.
# @PRE: entry_id exists.
# @POST: Entry fields are updated.
# @PURPOSE Edit an existing dictionary entry with language-pair-aware duplicate check.
# @PRE entry_id exists.
# @POST Entry fields are updated.
@staticmethod
def edit_entry(
db: Session, entry_id: str, source_term: str | None = None,
@@ -293,9 +295,9 @@ class DictionaryManager:
# endregion DictionaryManager.edit_entry
# region DictionaryManager.delete_entry [TYPE Function]
# @PURPOSE: Delete a single dictionary entry.
# @PRE: entry_id exists.
# @POST: Entry is deleted.
# @PURPOSE Delete a single dictionary entry.
# @PRE entry_id exists.
# @POST Entry is deleted.
@staticmethod
def delete_entry(db: Session, entry_id: str) -> None:
with belief_scope("DictionaryManager.delete_entry"):
@@ -309,9 +311,9 @@ class DictionaryManager:
# endregion DictionaryManager.delete_entry
# region DictionaryManager.clear_entries [TYPE Function]
# @PURPOSE: Delete all entries for a dictionary.
# @PRE: dict_id exists.
# @POST: All entries for the dictionary are deleted.
# @PURPOSE Delete all entries for a dictionary.
# @PRE dict_id exists.
# @POST All entries for the dictionary are deleted.
@staticmethod
def clear_entries(db: Session, dict_id: str) -> int:
with belief_scope("DictionaryManager.clear_entries"):
@@ -326,9 +328,9 @@ class DictionaryManager:
# endregion DictionaryManager.clear_entries
# region DictionaryManager.list_entries [TYPE Function]
# @PURPOSE: List entries for a dictionary with pagination.
# @PRE: dict_id exists.
# @POST: Returns (list of entries, total_count).
# @PURPOSE List entries for a dictionary with pagination.
# @PRE dict_id exists.
# @POST Returns (list of entries, total_count).
@staticmethod
def list_entries(
db: Session, dict_id: str, page: int = 1, page_size: int = 50,
@@ -351,10 +353,10 @@ class DictionaryManager:
# endregion DictionaryManager.list_entries
# region DictionaryManager.import_entries [TYPE Function]
# @PURPOSE: Import entries from CSV/TSV content with duplicate detection and conflict resolution.
# @PRE: content is valid CSV or TSV. dict_id exists.
# @POST: Entries are created/updated/skipped per conflict mode. Returns result summary.
# @SIDE_EFFECT: Batch-inserts or updates DictionaryEntry rows.
# @PURPOSE Import entries from CSV/TSV content with duplicate detection and conflict resolution.
# @PRE content is valid CSV or TSV. dict_id exists.
# @POST Entries are created/updated/skipped per conflict mode. Returns result summary.
# @SIDE_EFFECT Batch-inserts or updates DictionaryEntry rows.
@staticmethod
def import_entries(
db: Session, dict_id: str, content: str,
@@ -504,9 +506,9 @@ class DictionaryManager:
# endregion DictionaryManager.import_entries
# region DictionaryManager.export_entries [TYPE Function]
# @PURPOSE: Export all entries as CSV string with language columns.
# @PRE: dict_id exists.
# @POST: Returns CSV string with header: source_term, target_term, source_language, target_language, context_notes, context_data, usage_notes.
# @PURPOSE Export all entries as CSV string with language columns.
# @PRE dict_id exists.
# @POST Returns CSV string with header: source_term, target_term, source_language, target_language, context_notes, context_data, usage_notes.
@staticmethod
def export_entries(
db: Session, dict_id: str, delimiter: str = ",",
@@ -549,10 +551,10 @@ class DictionaryManager:
# endregion DictionaryManager.export_entries
# region DictionaryManager.migrate_old_entries [TYPE Function]
# @PURPOSE: Migrate existing single-language dictionary entries to populate source_language and target_language.
# @PRE: db session is open.
# @POST: Entries with "und" source_language or target_language are updated with inferred values.
# @SIDE_EFFECT: Updates DictionaryEntry rows in bulk.
# @PURPOSE Migrate existing single-language dictionary entries to populate source_language and target_language.
# @PRE db session is open.
# @POST Entries with "und" source_language or target_language are updated with inferred values.
# @SIDE_EFFECT Updates DictionaryEntry rows in bulk.
@staticmethod
def migrate_old_entries(db: Session) -> dict[str, int]:
with belief_scope("DictionaryManager.migrate_old_entries"):
@@ -601,12 +603,12 @@ class DictionaryManager:
# endregion DictionaryManager.migrate_old_entries
# region DictionaryManager.filter_for_batch [TYPE Function]
# @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job,
# @PURPOSE Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job,
# optionally filtered by language pair. When row_context is provided, computes context-aware priority flags.
# @PRE: job_id exists and source_texts is a list of strings.
# @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries.
# @PRE job_id exists and source_texts is a list of strings.
# @POST Returns list of matched entries with match info, sorted by priority across attached dictionaries.
# When row_context is given, each result includes a 'priority_match' boolean.
# @SIDE_EFFECT: Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.
# @SIDE_EFFECT Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.
@staticmethod
def filter_for_batch(
db: Session, source_texts: list[str], job_id: str,
@@ -738,10 +740,10 @@ class DictionaryManager:
# region DictionaryManager.submit_correction [TYPE Function]
# @PURPOSE: Submit a term correction from a run result. Validates language match, detects conflicts.
# @PRE: source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.
# @POST: Entry created or updated; origin tracking populated. Returns action + conflict info.
# @SIDE_EFFECT: Creates/updates DictionaryEntry row.
# @PURPOSE Submit a term correction from a run result. Validates language match, detects conflicts.
# @PRE source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.
# @POST Entry created or updated; origin tracking populated. Returns action + conflict info.
# @SIDE_EFFECT Creates/updates DictionaryEntry row.
@staticmethod
def submit_correction(
db: Session,
@@ -868,10 +870,10 @@ class DictionaryManager:
# endregion DictionaryManager.submit_correction
# region DictionaryManager.submit_bulk_corrections [TYPE Function]
# @PURPOSE: Submit multiple term corrections atomically — all succeed or all fail.
# @PRE: corrections list is non-empty. dict_id exists.
# @POST: All corrections applied or none applied with conflict list.
# @SIDE_EFFECT: Creates/updates DictionaryEntry rows; commits once.
# @PURPOSE Submit multiple term corrections atomically — all succeed or all fail.
# @PRE corrections list is non-empty. dict_id exists.
# @POST All corrections applied or none applied with conflict list.
# @SIDE_EFFECT Creates/updates DictionaryEntry rows; commits once.
@staticmethod
def submit_bulk_corrections(
db: Session,
@@ -998,5 +1000,8 @@ class DictionaryManager:
# endregion DictionaryManager.submit_bulk_corrections
# #endregion DictionaryManager
# #endregion DictionaryManager
# #endregion DictionaryManagerModule

32
backend/src/plugins/translate/executor.py
View File

@@ -194,11 +194,12 @@ def _check_translation_cache(
# #endregion _check_translation_cache
# #region estimate_row_tokens [C:2] [TYPE Function] [SEMANTICS translate, token, estimation]
# #region estimate_row_tokens [C:4] [TYPE Function] [SEMANTICS translate, token, estimation]
# @BRIEF Estimate token count for a single source row including context fields.
# @PRE: source_text is a string.
# @POST: Returns estimated token count >= 1.
# @PRE source_text is a string.
# @POST Returns estimated token count >= 1.
# @RELATION DEPENDS_ON -> [_estimate_tokens_for_text]
def estimate_row_tokens(
source_text: str,
source_data: dict | None,
@@ -1745,30 +1746,47 @@ class TranslationExecutor:
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
}
# Reasoning-safe system prompt: always forbid chain-of-thought in output.
# Reasoning models (DeepSeek, QwQ, etc.) may leak reasoning into content
# when response_format=json_object is set. Explicit suppression in the
# system message prevents this regardless of the disable_reasoning flag.
system_content = (
"You are a database content translation assistant. "
"Translate the provided text accurately, preserving data semantics. "
"Respond directly with ONLY the JSON result. "
"Do NOT include any reasoning, thinking, chain-of-thought, analysis, "
"or explanation. Output ONLY valid JSON."
)
payload = {
"model": model,
"messages": [
{"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics."},
{"role": "system", "content": system_content},
{"role": "user", "content": prompt},
],
"temperature": 0.1,
"max_tokens": max_tokens,
}
# Structured output — OpenRouter and Kilo also support response_format, but some
# upstream providers (e.g. StepFun) reject it. We try with response_format and
# fall back on 400 "structured_outputs is not supported".
# NOTE: Reasoning models (deepseek, qwen with reasoning, etc.) often break with
# response_format=json_object, producing reasoning text instead of JSON.
# When disable_reasoning is set, we remove response_format and rely on the
# system prompt to enforce JSON-only output.
if provider_type in ("openai", "openai_compatible", "kilo", "openrouter", "litellm"):
payload["response_format"] = {"type": "json_object"}
if not disable_reasoning:
payload["response_format"] = {"type": "json_object"}
# Suppress Chain of Thought reasoning to save output tokens
# NOTE: Kilo/OpenRouter/LiteLLM do NOT support disabling reasoning (returns 400)
if disable_reasoning:
if provider_type not in ("kilo", "openrouter", "litellm"):
payload["reasoning_effort"] = "none"
payload.pop("response_format", None)
# response_format already skipped above when disable_reasoning is True
# Use caller-provided max_tokens instead of hardcoded 8192
payload["max_tokens"] = max_tokens
payload["messages"][0] = {"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics. Respond directly with ONLY the JSON result. Do NOT include any reasoning, thinking, chain-of-thought, analysis, or explanation. Output ONLY valid JSON."}
logger.reason(
f"LLM request url={base_url} model={payload.get('model')} "

27
backend/src/plugins/translate/metrics.py
View File

@@ -1,14 +1,15 @@
# #region TranslationMetrics [C:3] [TYPE Module] [SEMANTICS sqlalchemy, translate, metrics, job, statistics]
# @BRIEF Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationEvent:Class]
# @RELATION DEPENDS_ON -> [MetricSnapshot:Class]
# @RELATION DEPENDS_ON -> [TranslationRun:Class]
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @PRE: Database session is open.
# @POST: Metrics are aggregated and returned; no side effects.
# @RATIONALE: Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture.
# @REJECTED: Querying all events for all time — would be slow; MetricSnapshot provides historical summary.
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @PRE Database session is open.
# @POST Metrics are aggregated and returned; no side effects.
# @RATIONALE Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture.
# @REJECTED Querying all events for all time — would be slow; MetricSnapshot provides historical summary.
# @COMPLEXITY 4
from datetime import UTC, datetime
from typing import Any
@@ -34,9 +35,9 @@ class TranslationMetrics:
self.db = db
# region get_job_metrics [TYPE Function]
# @PURPOSE: Get aggregated metrics for a specific job.
# @PRE: job_id exists.
# @POST: Returns dict with metrics from events + latest snapshot.
# @PURPOSE Get aggregated metrics for a specific job.
# @PRE job_id exists.
# @POST Returns dict with metrics from events + latest snapshot.
def get_job_metrics(self, job_id: str) -> dict[str, Any]:
with belief_scope("TranslationMetrics.get_job_metrics"):
# Run counts from TranslationRun
@@ -174,8 +175,8 @@ class TranslationMetrics:
# endregion get_job_metrics
# region get_all_metrics [TYPE Function]
# @PURPOSE: Get aggregated metrics for all jobs.
# @POST: Returns list of per-job metrics.
# @PURPOSE Get aggregated metrics for all jobs.
# @POST Returns list of per-job metrics.
def get_all_metrics(self) -> list[dict[str, Any]]:
with belief_scope("TranslationMetrics.get_all_metrics"):
job_ids = (

10
backend/src/plugins/translate/orchestrator.py
View File

@@ -238,6 +238,15 @@ class TranslationOrchestrator:
"run_id": run.id,
"error": str(e),
})
# Rollback the session if it's in a broken state (e.g. after a
# failed flush from a previous exception in executor.execute_run).
# Without this, subsequent flush()/commit() calls raise
# "This Session's transaction has been rolled back".
try:
self.db.rollback()
except Exception:
pass
run = self.db.merge(run)
run.status = "FAILED"
run.error_message = f"Translation execution failed: {e}"
run.completed_at = datetime.now(UTC)
@@ -887,6 +896,7 @@ class TranslationOrchestrator:
"id": run.id,
"job_id": run.job_id,
"status": run.status,
"trigger_type": run.trigger_type,
"started_at": run.started_at.isoformat() if run.started_at else None,
"completed_at": run.completed_at.isoformat() if run.completed_at else None,
"error_message": run.error_message,

6
backend/src/plugins/translate/plugin.py
View File

@@ -1,9 +1,9 @@
# #region TranslatePlugin [C:2] [TYPE Module] [SEMANTICS clickhouse, translate, sql, dashboard, dialect]
# @BRIEF TranslatePlugin skeleton for cross-dialect SQL and dashboard translation.
# @LAYER: Domain
# @LAYER Domain
# @RELATION INHERITS -> [PluginBase]
# @RATIONALE: Separate plugin avoids bloating LLMAnalysisPlugin beyond fractal limit (<400 lines).
# @REJECTED: Extending LLMAnalysisPlugin would conflate two distinct feature domains.
# @RATIONALE Separate plugin avoids bloating LLMAnalysisPlugin beyond fractal limit (<400 lines).
# @REJECTED Extending LLMAnalysisPlugin would conflate two distinct feature domains.
from typing import Any

33
backend/src/plugins/translate/preview.py
View File

@@ -41,7 +41,7 @@ from ._token_budget import DEFAULT_CONTEXT_WINDOW, estimate_token_budget
from .dictionary import DictionaryManager
# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [TYPE Constant]
# @BRIEF Default prompt template for batch LLM translation execution (no context columns — faster).
# Supports both single-language and multi-language modes via {target_languages} placeholder.
DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
"Translate the following database content from {source_language} to the following language(s): {target_languages}.\n\n"
@@ -63,9 +63,9 @@ DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
)
# #endregion DEFAULT_EXECUTION_PROMPT_TEMPLATE
# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [TYPE Constant]
# @BRIEF Default prompt template for LLM translation preview.
# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [TYPE Constant]
DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
"Translate the following database content.\n\n"
"Source dialect: {source_dialect}\n"
@@ -1038,19 +1038,37 @@ class TranslationPreview:
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
}
# Reasoning-safe system prompt: always forbid chain-of-thought in output.
# Reasoning models (DeepSeek, QwQ, etc.) may leak reasoning into content
# when response_format=json_object is set. Explicit suppression in the
# system message prevents this regardless of the disable_reasoning flag.
system_content = (
"You are a database content translation assistant. "
"Translate the provided text accurately, preserving data semantics. "
"Respond directly with ONLY the JSON result. "
"Do NOT include any reasoning, thinking, chain-of-thought, analysis, "
"or explanation. Output ONLY valid JSON."
)
payload = {
"model": model,
"messages": [
{"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics."},
{"role": "system", "content": system_content},
{"role": "user", "content": prompt},
],
"temperature": 0.1,
"max_tokens": max_tokens,
}
# Structured output — Kilo gateway supports response_format, but upstream providers
# (e.g. StepFun) may reject it. We try with response_format and fall back on 400.
# NOTE: Reasoning models (deepseek variants, qwen with reasoning) often break with
# response_format=json_object, producing reasoning text instead of JSON.
# When disable_reasoning is set, we skip response_format and rely on the
# system prompt to enforce JSON-only output.
if provider_type in ("openai", "openai_compatible", "kilo", "openrouter", "litellm"):
payload["response_format"] = {"type": "json_object"}
if not disable_reasoning:
payload["response_format"] = {"type": "json_object"}
# Suppress Chain of Thought reasoning to save output tokens
# NOTE: Kilo/OpenRouter/LiteLLM reject reasoning_effort — only use for native OpenAI-compatible
@@ -1058,13 +1076,10 @@ class TranslationPreview:
# Kilo/OpenRouter/LiteLLM reject reasoning_effort — only use for native OpenAI-compatible
if provider_type not in ("kilo", "openrouter", "litellm"):
payload["reasoning_effort"] = "none"
payload.pop("response_format", None) # JSON mode triggers reasoning on some models
# response_format already skipped above when disable_reasoning is True
# Use caller-provided max_tokens instead of hardcoded 8192
# This ensures multi-language batches with large output get enough token budget.
payload["max_tokens"] = max_tokens
# Universal instruction — all models understand "respond directly without reasoning"
system_content = "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics. Respond directly with ONLY the JSON result. Do NOT include any reasoning, thinking, chain-of-thought, analysis, or explanation. Output ONLY valid JSON."
payload["messages"][0] = {"role": "system", "content": system_content}
logger.reason(
f"LLM request url={base_url} model={payload.get('model')} "

7
backend/src/plugins/translate/prompt_builder.py
View File

@@ -1,9 +1,10 @@
# #region ContextAwarePromptBuilder [C:2] [TYPE Module] [SEMANTICS translate, prompt, context, dictionary]
# @BRIEF Pure-function prompt builder that enhances dictionary entries with context annotations.
# @LAYER: Domain
# @LAYER Domain
# @RELATION DEPENDS_ON -> [DictionaryEntry:Class]
# @RATIONALE: Pure functions only — no I/O, no DB access. Separated from executor for testability.
# @REJECTED: Embedding context inline in the executor would make it untestable without mocking DB.
# @RATIONALE Pure functions only — no I/O, no DB access. Separated from executor for testability.
# @REJECTED Embedding context inline in the executor would make it untestable without mocking DB.
#
# Typical workflow:
# 1. Call build_context_entries(dictionary_entries, row_context) to get annotated, prioritized entries

12
backend/src/plugins/translate/scheduler.py
View File

@@ -240,12 +240,14 @@ class TranslationScheduler:
# #endregion TranslationScheduler
# #region execute_scheduled_translation [TYPE Function]
# #region execute_scheduled_translation [C:4] [TYPE Function]
# @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and new-key-only fallback.
# @PRE: schedule_id is valid.
# @POST: Translation run created and executed if no concurrent run exists.
# @SIDE_EFFECT: DB writes; LLM calls; Superset API calls.
# @RATIONALE: New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.
# @PRE schedule_id is valid.
# @POST Translation run created and executed if no concurrent run exists.
# @SIDE_EFFECT DB writes; LLM calls; Superset API calls.
# @RATIONALE New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.
# @COMPLEXITY 4
def execute_scheduled_translation(
schedule_id: str,
job_id: str,

16
backend/src/plugins/translate/service.py
View File

@@ -38,7 +38,7 @@ SUPPORTED_DIALECTS = {
}
# #region get_dialect_from_database [TYPE Function]
# #region get_dialect_from_database [C:4] [TYPE Function]
# @BRIEF Extract normalized dialect string from a Superset database record.
# @PRE: database_record is a dict from Superset API.
# @POST: Returns normalized dialect string or raises ValueError if unsupported.
@@ -84,7 +84,7 @@ def get_dialect_from_database(database_record: dict[str, Any]) -> str:
# #endregion get_dialect_from_database
# #region fetch_datasource_metadata [TYPE Function]
# #region fetch_datasource_metadata [C:4] [TYPE Function]
# @BRIEF Fetch datasource columns and database dialect from Superset.
# @PRE: dataset_id is a valid Superset dataset ID and environment has valid credentials.
# @POST: Returns (columns_list, dialect_string) or raises on failure.
@@ -141,7 +141,7 @@ def fetch_datasource_metadata(
# #endregion fetch_datasource_metadata
# #region detect_virtual_columns [TYPE Function]
# #region detect_virtual_columns [C:4] [TYPE Function]
# @BRIEF Identify virtual (calculated) columns from column metadata.
# @PRE: columns is a list of dicts with 'is_physical' key.
# @POST: Returns list of virtual column names.
@@ -519,11 +519,13 @@ def job_to_response(job: TranslationJob, dict_ids: list[str] | None = None) -> T
# #endregion job_to_response
# #region DatasourceColumnsService [TYPE Function]
# #region DatasourceColumnsService [C:4] [TYPE Function]
# @BRIEF Fetch datasource column metadata from Superset and return structured response.
# @PRE: datasource_id is a valid Superset dataset ID.
# @POST: Returns DatasourceColumnsResponse with column metadata and database dialect.
# @SIDE_EFFECT: Queries Superset API for dataset detail and database info.
# @PRE datasource_id is a valid Superset dataset ID.
# @POST Returns DatasourceColumnsResponse with column metadata and database dialect.
# @SIDE_EFFECT Queries Superset API for dataset detail and database info.
# @COMPLEXITY 4
def get_datasource_columns(
datasource_id: int,
env_id: str,

10
backend/src/plugins/translate/sql_generator.py
View File

@@ -58,7 +58,7 @@ def _normalize_timestamp_value(value: Any) -> str | None:
# #endregion _normalize_timestamp_value
# #region _quote_identifier [TYPE Function]
# #region _quote_identifier [C:4] [TYPE Function]
# @BRIEF Quote an identifier per dialect rules. PostgreSQL uses double quotes; ClickHouse uses backticks.
# @PRE: identifier is a non-empty string.
# @POST: Returns safely quoted identifier.
@@ -159,10 +159,12 @@ def generate_insert_sql(
# #endregion generate_insert_sql
# #region generate_upsert_sql [TYPE Function]
# #region generate_upsert_sql [C:4] [TYPE Function]
# @BRIEF Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE.
# @PRE: dialect is postgresql-compatible. target_table, columns, key_columns are non-empty.
# @POST: Returns UPSERT SQL string or raises ValueError.
# @PRE dialect is postgresql-compatible. target_table, columns, key_columns are non-empty.
# @POST Returns UPSERT SQL string or raises ValueError.
# @COMPLEXITY 4
def generate_upsert_sql(
target_schema: str | None,
target_table: str,

72
backend/src/plugins/translate/superset_executor.py
View File

@@ -1,13 +1,13 @@
# #region SupersetSqlLabExecutor [C:4] [TYPE Module] [SEMANTICS translate, superset, sqllab, execute, query]
# @BRIEF Submit SQL to Superset SQL Lab API and poll execution status.
# @LAYER: Infra
# @RELATION DEPENDS_ON -> [SupersetClient]
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [ConfigManager]
# @PRE: Valid Superset environment configuration and authenticated client.
# @POST: SQL is submitted to Superset SQL Lab; execution reference is returned.
# @SIDE_EFFECT: Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status.
# @RATIONALE: Direct SQL Lab API submission provides audit trail and execution monitoring within Superset.
# @REJECTED: Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC.
# @RELATION DEPENDS_ON -> [ConfigManager]
# @PRE Valid Superset environment configuration and authenticated client.
# @POST SQL is submitted to Superset SQL Lab; execution reference is returned.
# @SIDE_EFFECT Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status.
# @RATIONALE Direct SQL Lab API submission provides audit trail and execution monitoring within Superset.
# @REJECTED Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC.
import json
import time
@@ -21,9 +21,9 @@ from ...core.superset_client import SupersetClient
# #region SupersetSqlLabExecutor [C:4] [TYPE Class]
# @BRIEF Submit SQL to Superset SQL Lab API with polling and status tracking.
# @PRE: Valid environment ID and ConfigManager.
# @POST: SQL submitted to Superset; execution reference recorded.
# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status.
# @PRE Valid environment ID and ConfigManager.
# @POST SQL submitted to Superset; execution reference recorded.
# @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status.
class SupersetSqlLabExecutor:
def __init__(self, config_manager: ConfigManager, env_id: str):
@@ -34,10 +34,10 @@ class SupersetSqlLabExecutor:
self._database_backend: str | None = None
# region _get_client [TYPE Function]
# @PURPOSE: Lazy-initialize SupersetClient for the configured environment.
# @PRE: env_id must correspond to a valid environment config.
# @POST: Returns authenticated SupersetClient.
# @SIDE_EFFECT: Authenticates against Superset API.
# @PURPOSE Lazy-initialize SupersetClient for the configured environment.
# @PRE env_id must correspond to a valid environment config.
# @POST Returns authenticated SupersetClient.
# @SIDE_EFFECT Authenticates against Superset API.
def _get_client(self) -> SupersetClient:
with belief_scope("SupersetSqlLabExecutor._get_client"):
if self._client is not None:
@@ -53,10 +53,10 @@ class SupersetSqlLabExecutor:
# endregion _get_client
# region resolve_database_id [TYPE Function]
# @PURPOSE: Resolve the target database ID from the environment and fetch its backend engine.
# @PRE: database_name or database_id should be known.
# @POST: Returns database_id integer or raises ValueError. Also sets self._database_backend.
# @SIDE_EFFECT: Fetches databases list from Superset; optionally fetches single DB for backend.
# @PURPOSE Resolve the target database ID from the environment and fetch its backend engine.
# @PRE database_name or database_id should be known.
# @POST Returns database_id integer or raises ValueError. Also sets self._database_backend.
# @SIDE_EFFECT Fetches databases list from Superset; optionally fetches single DB for backend.
def resolve_database_id(
self,
database_name: str | None = None,
@@ -121,17 +121,17 @@ class SupersetSqlLabExecutor:
# endregion resolve_database_id
# region get_database_backend [TYPE Function]
# @PURPOSE: Return the cached database backend/engine string.
# @POST: Returns backend string or None if not yet resolved.
# @PURPOSE Return the cached database backend/engine string.
# @POST Returns backend string or None if not yet resolved.
def get_database_backend(self) -> str | None:
return self._database_backend
# endregion get_database_backend
# region execute_sql [TYPE Function]
# @PURPOSE: Submit SQL to Superset SQL Lab and return execution tracking info.
# @PRE: sql is valid SQL string. database_id is a valid Superset DB ID.
# @POST: Returns execution result dict with query_id and status.
# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/.
# @PURPOSE Submit SQL to Superset SQL Lab and return execution tracking info.
# @PRE sql is valid SQL string. database_id is a valid Superset DB ID.
# @POST Returns execution result dict with query_id and status.
# @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/.
def execute_sql(
self,
sql: str,
@@ -236,10 +236,10 @@ class SupersetSqlLabExecutor:
# endregion execute_sql
# region poll_execution_status [TYPE Function]
# @PURPOSE: Poll Superset for SQL execution status until completion or timeout.
# @PRE: query_id is a valid Superset query ID.
# @POST: Returns final execution status dict.
# @SIDE_EFFECT: Makes HTTP GET requests to Superset.
# @PURPOSE Poll Superset for SQL execution status until completion or timeout.
# @PRE query_id is a valid Superset query ID.
# @POST Returns final execution status dict.
# @SIDE_EFFECT Makes HTTP GET requests to Superset.
def poll_execution_status(
self,
query_id: str,
@@ -329,10 +329,10 @@ class SupersetSqlLabExecutor:
# endregion poll_execution_status
# region execute_and_poll [TYPE Function]
# @PURPOSE: Execute SQL and wait for completion. One-shot convenience method.
# @PRE: sql is valid SQL.
# @POST: Returns final execution result.
# @SIDE_EFFECT: Makes HTTP calls to Superset API.
# @PURPOSE Execute SQL and wait for completion. One-shot convenience method.
# @PRE sql is valid SQL.
# @POST Returns final execution result.
# @SIDE_EFFECT Makes HTTP calls to Superset API.
def execute_and_poll(
self,
sql: str,
@@ -384,10 +384,10 @@ class SupersetSqlLabExecutor:
# endregion execute_and_poll
# region get_query_results [TYPE Function]
# @PURPOSE: Fetch the results of a completed query.
# @PRE: query_id is a valid Superset query ID.
# @POST: Returns query results if available.
# @SIDE_EFFECT: Makes HTTP GET to Superset.
# @PURPOSE Fetch the results of a completed query.
# @PRE query_id is a valid Superset query ID.
# @POST Returns query results if available.
# @SIDE_EFFECT Makes HTTP GET to Superset.
def get_query_results(self, query_id: str) -> dict[str, Any]:
with belief_scope("SupersetSqlLabExecutor.get_query_results"):
client = self._get_client()