busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

semantics: complete DEF-to-region migration, fix regressions

Browse Source 
- Convert legacy [DEF🆔Type] anchors to #region/#endregion across 329 files
- Reinstate _normalize_timestamp_value in sql_generator.py
- Fix MarkerLogger→logger migration in events.py (molecular CoT markers)
- Fix dataset_review orchestrator dependencies (_build_execution_snapshot)
- Fix config_manager stale-record deletion (moved to save path only)
- Add 77 missing [/DEF:] closers in 5 unbalanced test files
- Update assistant_chat.integration.test.js for #region format
- Apply molecular-cot-logging markers (REASON/REFLECT/EXPLORE) via logger.* methods
This commit is contained in:
busya
2026-05-12 23:54:55 +03:00
parent fe8978f716
commit 306c5ae742
331 changed files with 9630 additions and 10312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
backend/src/plugins/backup.py
View File

@@ -1,10 +1,10 @@
# #region BackupPlugin [TYPE Module] [SEMANTICS backup, superset, automation, dashboard, plugin]
# @BRIEF A plugin that provides functionality to back up Superset dashboards.
# @LAYER App
# @RELATION IMPLEMENTS -> [PluginBase]
# @RELATION DEPENDS_ON -> [superset_tool.client]
# @RELATION DEPENDS_ON -> [superset_tool.utils]
# @RELATION USES -> [TaskContext]
# @LAYER: App
# @RELATION IMPLEMENTS -> PluginBase
# @RELATION DEPENDS_ON -> superset_tool.client
# @RELATION DEPENDS_ON -> superset_tool.utils
# @RELATION USES -> TaskContext
from typing import Dict, Any, Optional
from pathlib import Path
@@ -33,63 +33,63 @@ class BackupPlugin(PluginBase):
"""
@property
# #region id [TYPE Function]
# @BRIEF Returns the unique identifier for the backup plugin.
# @PRE Plugin instance exists.
# @POST Returns string ID.
# @RETURN str - "superset-backup"
# [DEF:id:Function]
# @PURPOSE: Returns the unique identifier for the backup plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string ID.
# @RETURN: str - "superset-backup"
def id(self) -> str:
with belief_scope("id"):
return "superset-backup"
# #endregion id
# [/DEF:id:Function]
@property
# #region name [TYPE Function]
# @BRIEF Returns the human-readable name of the backup plugin.
# @PRE Plugin instance exists.
# @POST Returns string name.
# @RETURN str - Plugin name.
# [DEF:name:Function]
# @PURPOSE: Returns the human-readable name of the backup plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string name.
# @RETURN: str - Plugin name.
def name(self) -> str:
with belief_scope("name"):
return "Superset Dashboard Backup"
# #endregion name
# [/DEF:name:Function]
@property
# #region description [TYPE Function]
# @BRIEF Returns a description of the backup plugin.
# @PRE Plugin instance exists.
# @POST Returns string description.
# @RETURN str - Plugin description.
# [DEF:description:Function]
# @PURPOSE: Returns a description of the backup plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string description.
# @RETURN: str - Plugin description.
def description(self) -> str:
with belief_scope("description"):
return "Backs up all dashboards from a Superset instance."
# #endregion description
# [/DEF:description:Function]
@property
# #region version [TYPE Function]
# @BRIEF Returns the version of the backup plugin.
# @PRE Plugin instance exists.
# @POST Returns string version.
# @RETURN str - "1.0.0"
# [DEF:version:Function]
# @PURPOSE: Returns the version of the backup plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string version.
# @RETURN: str - "1.0.0"
def version(self) -> str:
with belief_scope("version"):
return "1.0.0"
# #endregion version
# [/DEF:version:Function]
@property
# #region ui_route [TYPE Function]
# @BRIEF Returns the frontend route for the backup plugin.
# @RETURN str - "/tools/backups"
# [DEF:ui_route:Function]
# @PURPOSE: Returns the frontend route for the backup plugin.
# @RETURN: str - "/tools/backups"
def ui_route(self) -> str:
with belief_scope("ui_route"):
return "/tools/backups"
# #endregion ui_route
# [/DEF:ui_route:Function]
# #region get_schema [TYPE Function]
# @BRIEF Returns the JSON schema for backup plugin parameters.
# @PRE Plugin instance exists.
# @POST Returns dictionary schema.
# @RETURN Dict[str, Any] - JSON schema.
# [DEF:get_schema:Function]
# @PURPOSE: Returns the JSON schema for backup plugin parameters.
# @PRE: Plugin instance exists.
# @POST: Returns dictionary schema.
# @RETURN: Dict[str, Any] - JSON schema.
def get_schema(self) -> Dict[str, Any]:
with belief_scope("get_schema"):
config_manager = get_config_manager()
@@ -108,10 +108,10 @@ class BackupPlugin(PluginBase):
},
"required": ["env"],
}
# #endregion get_schema
# [/DEF:get_schema:Function]
# #region execute [TYPE Function]
# @BRIEF Executes the dashboard backup logic with TaskContext support.
# [DEF:execute:Function]
# @PURPOSE: Executes the dashboard backup logic with TaskContext support.
# @PARAM: params (Dict[str, Any]) - Backup parameters (env, backup_path, dashboard_ids).
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE: Target environment must be configured. params must be a dictionary.
@@ -255,6 +255,6 @@ class BackupPlugin(PluginBase):
except (RequestException, IOError, KeyError) as e:
log.error(f"Fatal error during backup for {env}: {e}")
raise e
# #endregion execute
# [/DEF:execute:Function]
# #endregion BackupPlugin
# #endregion BackupPlugin

102
backend/src/plugins/debug.py
View File

@@ -1,16 +1,14 @@
# #region DebugPluginModule [TYPE Module] [SEMANTICS plugin, debug, api, database, superset]
# @BRIEF Implements a plugin for system diagnostics and debugging Superset API responses.
# @LAYER Plugins
# @LAYER: Plugins
# @RELATION Inherits from PluginBase. Uses SupersetClient from core.
# @RELATION USES -> [TaskContext]
# @RELATION USES -> TaskContext
# [SECTION: IMPORTS]
from typing import Dict, Any, Optional
from ..core.plugin_base import PluginBase
from ..core.superset_client import SupersetClient
from ..core.logger import logger, belief_scope
from ..core.task_manager.context import TaskContext
# [/SECTION]
# #region DebugPlugin [TYPE Class]
# @BRIEF Plugin for system diagnostics and debugging.
@@ -20,63 +18,63 @@ class DebugPlugin(PluginBase):
"""
@property
# #region id [TYPE Function]
# @BRIEF Returns the unique identifier for the debug plugin.
# @PRE Plugin instance exists.
# @POST Returns string ID.
# @RETURN str - "system-debug"
# [DEF:id:Function]
# @PURPOSE: Returns the unique identifier for the debug plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string ID.
# @RETURN: str - "system-debug"
def id(self) -> str:
with belief_scope("id"):
return "system-debug"
# #endregion id
# [/DEF:id:Function]
@property
# #region name [TYPE Function]
# @BRIEF Returns the human-readable name of the debug plugin.
# @PRE Plugin instance exists.
# @POST Returns string name.
# @RETURN str - Plugin name.
# [DEF:name:Function]
# @PURPOSE: Returns the human-readable name of the debug plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string name.
# @RETURN: str - Plugin name.
def name(self) -> str:
with belief_scope("name"):
return "System Debug"
# #endregion name
# [/DEF:name:Function]
@property
# #region description [TYPE Function]
# @BRIEF Returns a description of the debug plugin.
# @PRE Plugin instance exists.
# @POST Returns string description.
# @RETURN str - Plugin description.
# [DEF:description:Function]
# @PURPOSE: Returns a description of the debug plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string description.
# @RETURN: str - Plugin description.
def description(self) -> str:
with belief_scope("description"):
return "Run system diagnostics and debug Superset API responses."
# #endregion description
# [/DEF:description:Function]
@property
# #region version [TYPE Function]
# @BRIEF Returns the version of the debug plugin.
# @PRE Plugin instance exists.
# @POST Returns string version.
# @RETURN str - "1.0.0"
# [DEF:version:Function]
# @PURPOSE: Returns the version of the debug plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string version.
# @RETURN: str - "1.0.0"
def version(self) -> str:
with belief_scope("version"):
return "1.0.0"
# #endregion version
# [/DEF:version:Function]
@property
# #region ui_route [TYPE Function]
# @BRIEF Returns the frontend route for the debug plugin.
# @RETURN str - "/tools/debug"
# [DEF:ui_route:Function]
# @PURPOSE: Returns the frontend route for the debug plugin.
# @RETURN: str - "/tools/debug"
def ui_route(self) -> str:
with belief_scope("ui_route"):
return "/tools/debug"
# #endregion ui_route
# [/DEF:ui_route:Function]
# #region get_schema [TYPE Function]
# @BRIEF Returns the JSON schema for the debug plugin parameters.
# @PRE Plugin instance exists.
# @POST Returns dictionary schema.
# @RETURN Dict[str, Any] - JSON schema.
# [DEF:get_schema:Function]
# @PURPOSE: Returns the JSON schema for the debug plugin parameters.
# @PRE: Plugin instance exists.
# @POST: Returns dictionary schema.
# @RETURN: Dict[str, Any] - JSON schema.
def get_schema(self) -> Dict[str, Any]:
with belief_scope("get_schema"):
return {
@@ -111,10 +109,10 @@ class DebugPlugin(PluginBase):
},
"required": ["action"]
}
# #endregion get_schema
# [/DEF:get_schema:Function]
# #region execute [TYPE Function]
# @BRIEF Executes the debug logic with TaskContext support.
# [DEF:execute:Function]
# @PURPOSE: Executes the debug logic with TaskContext support.
# @PARAM: params (Dict[str, Any]) - Debug parameters.
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE: action must be provided in params.
@@ -138,12 +136,12 @@ class DebugPlugin(PluginBase):
else:
debug_log.error(f"Unknown action: {action}")
raise ValueError(f"Unknown action: {action}")
# #endregion execute
# [/DEF:execute:Function]
# #region _test_db_api [TYPE Function]
# @BRIEF Tests database API connectivity for source and target environments.
# @PRE source_env and target_env params exist in params.
# @POST Returns DB counts for both envs.
# [DEF:_test_db_api:Function]
# @PURPOSE: Tests database API connectivity for source and target environments.
# @PRE: source_env and target_env params exist in params.
# @POST: Returns DB counts for both envs.
# @PARAM: params (Dict) - Plugin parameters.
# @PARAM: log - Logger instance for superset_api source.
# @RETURN: Dict - Comparison results.
@@ -176,12 +174,12 @@ class DebugPlugin(PluginBase):
}
return results
# #endregion _test_db_api
# [/DEF:_test_db_api:Function]
# #region _get_dataset_structure [TYPE Function]
# @BRIEF Retrieves the structure of a dataset.
# @PRE env and dataset_id params exist in params.
# @POST Returns dataset JSON structure.
# [DEF:_get_dataset_structure:Function]
# @PURPOSE: Retrieves the structure of a dataset.
# @PRE: env and dataset_id params exist in params.
# @POST: Returns dataset JSON structure.
# @PARAM: params (Dict) - Plugin parameters.
# @PARAM: log - Logger instance for superset_api source.
# @RETURN: Dict - Dataset structure.
@@ -208,7 +206,7 @@ class DebugPlugin(PluginBase):
dataset_response = client.get_dataset(dataset_id)
log.debug(f"Retrieved dataset structure for {dataset_id}")
return dataset_response.get('result') or {}
# #endregion _get_dataset_structure
# [/DEF:_get_dataset_structure:Function]
# #endregion DebugPlugin
# #endregion DebugPluginModule
# #endregion DebugPluginModule

14
backend/src/plugins/git/llm_extension.py
View File

@@ -1,6 +1,8 @@
# #region backend/src/plugins/git/llm_extension [C:3] [TYPE Module] [SEMANTICS git, llm, commit]
# [DEF:backend/src/plugins/git/llm_extension:Module]
# @COMPLEXITY: 3
# @SEMANTICS: git, llm, commit
# @BRIEF LLM-based extensions for the Git plugin, specifically for commit message generation.
# @LAYER Domain
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [LLMClient]
from typing import List
@@ -15,8 +17,8 @@ class GitLLMExtension:
def __init__(self, client: LLMClient):
self.client = client
# #region suggest_commit_message [TYPE Function]
# @BRIEF Generates a suggested commit message based on a diff and history.
# [DEF:suggest_commit_message:Function]
# @PURPOSE: Generates a suggested commit message based on a diff and history.
# @PARAM: diff (str) - The git diff of staged changes.
# @PARAM: history (List[str]) - Recent commit messages for context.
# @RETURN: str - The suggested commit message.
@@ -56,7 +58,7 @@ class GitLLMExtension:
return "Update dashboard configurations (LLM generation failed)"
return response.choices[0].message.content.strip()
# #endregion suggest_commit_message
# [/DEF:suggest_commit_message:Function]
# #endregion GitLLMExtension
# #endregion backend/src/plugins/git/llm_extension
# [/DEF:backend/src/plugins/git/llm_extension:Module]

137
backend/src/plugins/git_plugin.py
View File

@@ -1,17 +1,16 @@
# #region GitPluginModule [TYPE Module] [SEMANTICS git, plugin, dashboard, version_control, sync, deploy]
#
# @BRIEF Предоставляет плагин для версионирования и развертывания дашбордов Superset.
# @LAYER Plugin
# @INVARIANT Все операции с Git должны выполняться через GitService.
# @RELATION INHERITS_FROM -> [src.core.plugin_base.PluginBase]
# @RELATION USES -> [src.services.git_service.GitService]
# @RELATION USES -> [src.core.superset_client.SupersetClient]
# @RELATION USES -> [src.core.config_manager.ConfigManager]
# @RELATION USES -> [TaskContext]
#
# @LAYER: Plugin
# @RELATION INHERITS_FROM -> src.core.plugin_base.PluginBase
# @RELATION USES -> src.services.git_service.GitService
# @RELATION USES -> src.core.superset_client.SupersetClient
# @RELATION USES -> src.core.config_manager.ConfigManager
# @RELATION USES -> TaskContext
#
# @INVARIANT: Все операции с Git должны выполняться через GitService.
# @CONSTRAINT: Плагин работает только с распакованными YAML-экспортами Superset.
# [SECTION: IMPORTS]
import os
import io
import shutil
@@ -21,27 +20,23 @@ from typing import Dict, Any, Optional
from src.core.plugin_base import PluginBase
from src.services.git_service import GitService
from src.core.logger import logger as app_logger, belief_scope
from src.core.cot_logger import MarkerLogger
log = MarkerLogger("GitPlugin")
from src.core.config_manager import ConfigManager
from src.core.superset_client import SupersetClient
from src.core.task_manager.context import TaskContext
from src.core.database import SessionLocal
from src.core.mapping_service import IdMappingService
# [/SECTION]
# #region GitPlugin [TYPE Class]
# @BRIEF Реализация плагина Git Integration для управления версиями дашбордов.
class GitPlugin(PluginBase):
# #region __init__ [TYPE Function]
# @BRIEF Инициализирует плагин и его зависимости.
# @PRE config.json exists or shared config_manager is available.
# @POST Инициализированы git_service и config_manager.
# [DEF:__init__:Function]
# @PURPOSE: Инициализирует плагин и его зависимости.
# @PRE: config.json exists or shared config_manager is available.
# @POST: Инициализированы git_service и config_manager.
def __init__(self):
with belief_scope("GitPlugin.__init__"):
log.reason("Initializing GitPlugin.")
app_logger.info("Initializing GitPlugin.")
self.git_service = GitService()
# Robust config path resolution:
@@ -56,69 +51,69 @@ class GitPlugin(PluginBase):
try:
from src.dependencies import config_manager
self.config_manager = config_manager
log.reflect("GitPlugin initialized using shared config_manager.")
app_logger.info("GitPlugin initialized using shared config_manager.")
return
except Exception:
config_path = "config.json"
self.config_manager = ConfigManager(config_path)
log.reflect(f"GitPlugin initialized with {config_path}")
# #endregion __init__
app_logger.info(f"GitPlugin initialized with {config_path}")
# [/DEF:__init__:Function]
@property
# #region id [TYPE Function]
# @BRIEF Returns the plugin identifier.
# @PRE GitPlugin is initialized.
# @POST Returns 'git-integration'.
# [DEF:id:Function]
# @PURPOSE: Returns the plugin identifier.
# @PRE: GitPlugin is initialized.
# @POST: Returns 'git-integration'.
def id(self) -> str:
with belief_scope("GitPlugin.id"):
return "git-integration"
# #endregion id
# [/DEF:id:Function]
@property
# #region name [TYPE Function]
# @BRIEF Returns the plugin name.
# @PRE GitPlugin is initialized.
# @POST Returns the human-readable name.
# [DEF:name:Function]
# @PURPOSE: Returns the plugin name.
# @PRE: GitPlugin is initialized.
# @POST: Returns the human-readable name.
def name(self) -> str:
with belief_scope("GitPlugin.name"):
return "Git Integration"
# #endregion name
# [/DEF:name:Function]
@property
# #region description [TYPE Function]
# @BRIEF Returns the plugin description.
# @PRE GitPlugin is initialized.
# @POST Returns the plugin's purpose description.
# [DEF:description:Function]
# @PURPOSE: Returns the plugin description.
# @PRE: GitPlugin is initialized.
# @POST: Returns the plugin's purpose description.
def description(self) -> str:
with belief_scope("GitPlugin.description"):
return "Version control for Superset dashboards"
# #endregion description
# [/DEF:description:Function]
@property
# #region version [TYPE Function]
# @BRIEF Returns the plugin version.
# @PRE GitPlugin is initialized.
# @POST Returns the version string.
# [DEF:version:Function]
# @PURPOSE: Returns the plugin version.
# @PRE: GitPlugin is initialized.
# @POST: Returns the version string.
def version(self) -> str:
with belief_scope("GitPlugin.version"):
return "0.1.0"
# #endregion version
# [/DEF:version:Function]
@property
# #region ui_route [TYPE Function]
# @BRIEF Returns the frontend route for the git plugin.
# @RETURN str - "/git"
# [DEF:ui_route:Function]
# @PURPOSE: Returns the frontend route for the git plugin.
# @RETURN: str - "/git"
def ui_route(self) -> str:
with belief_scope("GitPlugin.ui_route"):
return "/git"
# #endregion ui_route
# [/DEF:ui_route:Function]
# #region get_schema [TYPE Function]
# @BRIEF Возвращает JSON-схему параметров для выполнения задач плагина.
# @PRE GitPlugin is initialized.
# @POST Returns a JSON schema dictionary.
# @RETURN Dict[str, Any] - Схема параметров.
# [DEF:get_schema:Function]
# @PURPOSE: Возвращает JSON-схему параметров для выполнения задач плагина.
# @PRE: GitPlugin is initialized.
# @POST: Returns a JSON schema dictionary.
# @RETURN: Dict[str, Any] - Схема параметров.
def get_schema(self) -> Dict[str, Any]:
with belief_scope("GitPlugin.get_schema"):
return {
@@ -131,15 +126,15 @@ class GitPlugin(PluginBase):
},
"required": ["operation", "dashboard_id"]
}
# #endregion get_schema
# [/DEF:get_schema:Function]
# #region initialize [TYPE Function]
# @BRIEF Выполняет начальную настройку плагина.
# @PRE GitPlugin is initialized.
# @POST Плагин готов к выполнению задач.
# [DEF:initialize:Function]
# @PURPOSE: Выполняет начальную настройку плагина.
# @PRE: GitPlugin is initialized.
# @POST: Плагин готов к выполнению задач.
async def initialize(self):
with belief_scope("GitPlugin.initialize"):
log.reason("Initializing Git Integration Plugin logic")
app_logger.info("[GitPlugin.initialize][Action] Initializing Git Integration Plugin logic.")
# [DEF:execute:Function]
# @PURPOSE: Основной метод выполнения задач плагина с поддержкой TaskContext.
@@ -250,15 +245,15 @@ class GitPlugin(PluginBase):
# 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff)
try:
repo.git.add(A=True)
log.reason("Changes staged in git")
app_logger.info("[_handle_sync][Action] Changes staged in git")
except Exception as ge:
log.explore("Failed to stage changes", error=str(ge))
app_logger.warning(f"[_handle_sync][Action] Failed to stage changes: {ge}")
log.reflect("Dashboard synced successfully", payload={"dashboard_id": dashboard_id})
app_logger.info(f"[_handle_sync][Coherence:OK] Dashboard {dashboard_id} synced successfully.")
return {"status": "success", "message": "Dashboard synced and flattened in local repository"}
except Exception as e:
log.explore("Sync failed", error=str(e))
app_logger.error(f"[_handle_sync][Coherence:Failed] Sync failed: {e}")
raise
# [/DEF:_handle_sync:Function]
@@ -320,16 +315,16 @@ class GitPlugin(PluginBase):
f.write(zip_buffer.getvalue())
try:
log.reason("Importing dashboard", payload={"environment": env.name})
app_logger.info(f"[_handle_deploy][Action] Importing dashboard to {env.name}")
result = client.import_dashboard(temp_zip_path)
log.reflect("Deployment successful", payload={"dashboard_id": dashboard_id})
app_logger.info(f"[_handle_deploy][Coherence:OK] Deployment successful for dashboard {dashboard_id}.")
return {"status": "success", "message": f"Dashboard deployed to {env.name}", "details": result}
finally:
if temp_zip_path.exists():
os.remove(temp_zip_path)
except Exception as e:
log.explore("Deployment failed", error=str(e))
app_logger.error(f"[_handle_deploy][Coherence:Failed] Deployment failed: {e}")
raise
# [/DEF:_handle_deploy:Function]
@@ -341,13 +336,13 @@ class GitPlugin(PluginBase):
# @RETURN: Environment - Объект конфигурации окружения.
def _get_env(self, env_id: Optional[str] = None):
with belief_scope("GitPlugin._get_env"):
log.reason("Fetching environment", payload={"env_id": env_id})
app_logger.info(f"[_get_env][Entry] Fetching environment for ID: {env_id}")
# Priority 1: ConfigManager (config.json)
if env_id:
env = self.config_manager.get_environment(env_id)
if env:
log.reflect("Found environment by ID in ConfigManager", payload={"name": env.name})
app_logger.info(f"[_get_env][Exit] Found environment by ID in ConfigManager: {env.name}")
return env
# Priority 2: Database (DeploymentEnvironment)
@@ -365,7 +360,7 @@ class GitPlugin(PluginBase):
db_env = db.query(DeploymentEnvironment).first()
if db_env:
log.reflect("Found environment in DB", payload={"name": db_env.name})
app_logger.info(f"[_get_env][Exit] Found environment in DB: {db_env.name}")
from src.core.config_models import Environment
# Use token as password for SupersetClient
return Environment(
@@ -387,17 +382,17 @@ class GitPlugin(PluginBase):
# but we have other envs, maybe it's one of them?
env = next((e for e in envs if e.id == env_id), None)
if env:
log.reflect("Found environment in ConfigManager list", payload={"env_id": env_id})
app_logger.info(f"[_get_env][Exit] Found environment {env_id} in ConfigManager list")
return env
if not env_id:
log.reflect("Using first environment from ConfigManager", payload={"name": envs[0].name})
app_logger.info(f"[_get_env][Exit] Using first environment from ConfigManager: {envs[0].name}")
return envs[0]
log.explore("No environments configured", payload={"env_id": env_id}, error="No Superset environments found in config or database")
app_logger.error(f"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}")
raise ValueError("No environments configured. Please add a Superset Environment in Settings.")
# [/DEF:_get_env:Function]
# #endregion initialize
# [/DEF:initialize:Function]
# #endregion GitPlugin
# #endregion GitPluginModule
# #endregion GitPluginModule

4
backend/src/plugins/llm_analysis/__init__.py
View File

@@ -1,4 +1,4 @@
# #region backend/src/plugins/llm_analysis/__init__.py [TYPE Module]
# [DEF:backend/src/plugins/llm_analysis/__init__.py:Module]
"""
LLM Analysis Plugin for automated dashboard validation and dataset documentation.
@@ -8,4 +8,4 @@ from .plugin import DashboardValidationPlugin, DocumentationPlugin
__all__ = ['DashboardValidationPlugin', 'DocumentationPlugin']
# #endregion backend/src/plugins/llm_analysis/__init__.py
# [/DEF:backend/src/plugins/llm_analysis/__init__.py:Module]

22
backend/src/plugins/llm_analysis/__tests__/test_client_headers.py
View File

@@ -1,16 +1,18 @@
# #region TestClientHeaders [C:3] [TYPE Module] [SEMANTICS tests, llm-client, openrouter, headers]
# @BRIEF Verify OpenRouter client initialization includes provider-specific headers.
# @RELATION BELONGS_TO -> [SrcRoot]
# [DEF:TestClientHeaders:Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @COMPLEXITY: 3
# @SEMANTICS: tests, llm-client, openrouter, headers
# @PURPOSE: Verify OpenRouter client initialization includes provider-specific headers.
from src.plugins.llm_analysis.models import LLMProviderType
from src.plugins.llm_analysis.service import LLMClient
# #region test_openrouter_client_includes_referer_and_title_headers [TYPE Function]
# @BRIEF OpenRouter requests should carry site/app attribution headers for compatibility.
# @PRE Client is initialized for OPENROUTER provider.
# @POST Async client headers include Authorization, HTTP-Referer, and X-Title.
# @RELATION BINDS_TO -> [TestClientHeaders]
# [DEF:test_openrouter_client_includes_referer_and_title_headers:Function]
# @RELATION: BINDS_TO -> TestClientHeaders
# @PURPOSE: OpenRouter requests should carry site/app attribution headers for compatibility.
# @PRE: Client is initialized for OPENROUTER provider.
# @POST: Async client headers include Authorization, HTTP-Referer, and X-Title.
def test_openrouter_client_includes_referer_and_title_headers(monkeypatch):
monkeypatch.setenv("OPENROUTER_SITE_URL", "http://localhost:8000")
monkeypatch.setenv("OPENROUTER_APP_NAME", "ss-tools-test")
@@ -26,5 +28,5 @@ def test_openrouter_client_includes_referer_and_title_headers(monkeypatch):
assert headers["Authorization"] == "Bearer sk-test-provider-key-123456"
assert headers["HTTP-Referer"] == "http://localhost:8000"
assert headers["X-Title"] == "ss-tools-test"
# #endregion test_openrouter_client_includes_referer_and_title_headers
# #endregion TestClientHeaders
# [/DEF:test_openrouter_client_includes_referer_and_title_headers:Function]
# [/DEF:TestClientHeaders:Module]

94
backend/src/plugins/llm_analysis/__tests__/test_screenshot_service.py
View File

@@ -1,17 +1,19 @@
# #region TestScreenshotService [C:3] [TYPE Module] [SEMANTICS tests, screenshot-service, navigation, timeout-regression]
# @BRIEF Protect dashboard screenshot navigation from brittle networkidle waits.
# @RELATION VERIFIES -> [src.plugins.llm_analysis.service.ScreenshotService]
# [DEF:TestScreenshotService:Module]
# @RELATION: VERIFIES ->[src.plugins.llm_analysis.service.ScreenshotService]
# @COMPLEXITY: 3
# @SEMANTICS: tests, screenshot-service, navigation, timeout-regression
# @PURPOSE: Protect dashboard screenshot navigation from brittle networkidle waits.
import pytest
from src.plugins.llm_analysis.service import ScreenshotService
# #region test_iter_login_roots_includes_child_frames [TYPE Function]
# @BRIEF Login discovery must search embedded auth frames, not only the main page.
# @PRE Page exposes child frames list.
# @POST Returned roots include page plus child frames in order.
# @RELATION BINDS_TO -> [TestScreenshotService]
# [DEF:test_iter_login_roots_includes_child_frames:Function]
# @RELATION: BINDS_TO ->[TestScreenshotService]
# @PURPOSE: Login discovery must search embedded auth frames, not only the main page.
# @PRE: Page exposes child frames list.
# @POST: Returned roots include page plus child frames in order.
def test_iter_login_roots_includes_child_frames():
frame_a = object()
frame_b = object()
@@ -23,14 +25,14 @@ def test_iter_login_roots_includes_child_frames():
assert roots == [fake_page, frame_a, frame_b]
# #endregion test_iter_login_roots_includes_child_frames
# [/DEF:test_iter_login_roots_includes_child_frames:Function]
# #region test_response_looks_like_login_page_detects_login_markup [TYPE Function]
# @BRIEF Direct login fallback must reject responses that render the login screen again.
# @PRE Response body contains stable login-page markers.
# @POST Helper returns True so caller treats fallback as failed authentication.
# @RELATION BINDS_TO -> [TestScreenshotService]
# [DEF:test_response_looks_like_login_page_detects_login_markup:Function]
# @RELATION: BINDS_TO ->[TestScreenshotService]
# @PURPOSE: Direct login fallback must reject responses that render the login screen again.
# @PRE: Response body contains stable login-page markers.
# @POST: Helper returns True so caller treats fallback as failed authentication.
def test_response_looks_like_login_page_detects_login_markup():
service = ScreenshotService(env=type("Env", (), {})())
@@ -50,14 +52,14 @@ def test_response_looks_like_login_page_detects_login_markup():
assert result is True
# #endregion test_response_looks_like_login_page_detects_login_markup
# [/DEF:test_response_looks_like_login_page_detects_login_markup:Function]
# #region test_find_first_visible_locator_skips_hidden_first_match [TYPE Function]
# @BRIEF Locator helper must not reject a selector collection just because its first element is hidden.
# @PRE First matched element is hidden and second matched element is visible.
# @POST Helper returns the second visible candidate.
# @RELATION BINDS_TO -> [TestScreenshotService]
# [DEF:test_find_first_visible_locator_skips_hidden_first_match:Function]
# @RELATION: BINDS_TO ->[TestScreenshotService]
# @PURPOSE: Locator helper must not reject a selector collection just because its first element is hidden.
# @PRE: First matched element is hidden and second matched element is visible.
# @POST: Helper returns the second visible candidate.
@pytest.mark.anyio
async def test_find_first_visible_locator_skips_hidden_first_match():
class _FakeElement:
@@ -91,14 +93,14 @@ async def test_find_first_visible_locator_skips_hidden_first_match():
assert result.label == "visible"
# #endregion test_find_first_visible_locator_skips_hidden_first_match
# [/DEF:test_find_first_visible_locator_skips_hidden_first_match:Function]
# #region test_submit_login_via_form_post_uses_browser_context_request [TYPE Function]
# @BRIEF Fallback login must submit hidden fields and credentials through the context request cookie jar.
# @PRE Login DOM exposes csrf hidden field and request context returns authenticated HTML.
# @POST Helper returns True and request payload contains csrf_token plus credentials plus request options.
# @RELATION BINDS_TO -> [TestScreenshotService]
# [DEF:test_submit_login_via_form_post_uses_browser_context_request:Function]
# @RELATION: BINDS_TO ->[TestScreenshotService]
# @PURPOSE: Fallback login must submit hidden fields and credentials through the context request cookie jar.
# @PRE: Login DOM exposes csrf hidden field and request context returns authenticated HTML.
# @POST: Helper returns True and request payload contains csrf_token plus credentials plus request options.
@pytest.mark.anyio
async def test_submit_login_via_form_post_uses_browser_context_request():
class _FakeInput:
@@ -202,14 +204,14 @@ async def test_submit_login_via_form_post_uses_browser_context_request():
]
# #endregion test_submit_login_via_form_post_uses_browser_context_request
# [/DEF:test_submit_login_via_form_post_uses_browser_context_request:Function]
# #region test_submit_login_via_form_post_accepts_authenticated_redirect [TYPE Function]
# @BRIEF Fallback login must treat non-login 302 redirect as success without waiting for redirect target.
# @PRE Request response is 302 with Location outside login path.
# @POST Helper returns True.
# @RELATION BINDS_TO -> [TestScreenshotService]
# [DEF:test_submit_login_via_form_post_accepts_authenticated_redirect:Function]
# @RELATION: BINDS_TO ->[TestScreenshotService]
# @PURPOSE: Fallback login must treat non-login 302 redirect as success without waiting for redirect target.
# @PRE: Request response is 302 with Location outside login path.
# @POST: Helper returns True.
@pytest.mark.anyio
async def test_submit_login_via_form_post_accepts_authenticated_redirect():
class _FakeInput:
@@ -277,14 +279,14 @@ async def test_submit_login_via_form_post_accepts_authenticated_redirect():
assert result is True
# #endregion test_submit_login_via_form_post_accepts_authenticated_redirect
# [/DEF:test_submit_login_via_form_post_accepts_authenticated_redirect:Function]
# #region test_submit_login_via_form_post_rejects_login_markup_response [TYPE Function]
# @BRIEF Fallback login must fail when POST response still contains login form content.
# @PRE Login DOM exposes csrf hidden field and request response renders login markup.
# @POST Helper returns False.
# @RELATION BINDS_TO -> [TestScreenshotService]
# [DEF:test_submit_login_via_form_post_rejects_login_markup_response:Function]
# @RELATION: BINDS_TO ->[TestScreenshotService]
# @PURPOSE: Fallback login must fail when POST response still contains login form content.
# @PRE: Login DOM exposes csrf hidden field and request response renders login markup.
# @POST: Helper returns False.
@pytest.mark.anyio
async def test_submit_login_via_form_post_rejects_login_markup_response():
class _FakeInput:
@@ -360,14 +362,14 @@ async def test_submit_login_via_form_post_rejects_login_markup_response():
assert result is False
# #endregion test_submit_login_via_form_post_rejects_login_markup_response
# [/DEF:test_submit_login_via_form_post_rejects_login_markup_response:Function]
# #region test_goto_resilient_falls_back_from_domcontentloaded_to_load [TYPE Function]
# @BRIEF Pages with unstable primary wait must retry with fallback wait strategy.
# @PRE First page.goto call raises; second succeeds.
# @POST Helper returns second response and attempts both wait modes in order.
# @RELATION BINDS_TO -> [TestScreenshotService]
# [DEF:test_goto_resilient_falls_back_from_domcontentloaded_to_load:Function]
# @RELATION: BINDS_TO ->[TestScreenshotService]
# @PURPOSE: Pages with unstable primary wait must retry with fallback wait strategy.
# @PRE: First page.goto call raises; second succeeds.
# @POST: Helper returns second response and attempts both wait modes in order.
@pytest.mark.anyio
async def test_goto_resilient_falls_back_from_domcontentloaded_to_load():
class _FakePage:
@@ -398,5 +400,5 @@ async def test_goto_resilient_falls_back_from_domcontentloaded_to_load():
]
# #endregion test_goto_resilient_falls_back_from_domcontentloaded_to_load
# #endregion TestScreenshotService
# [/DEF:test_goto_resilient_falls_back_from_domcontentloaded_to_load:Function]
# [/DEF:TestScreenshotService:Module]

34
backend/src/plugins/llm_analysis/__tests__/test_service.py
View File

@@ -1,6 +1,8 @@
# #region TestService [C:3] [TYPE Module] [SEMANTICS tests, llm-analysis, fallback, provider-error, unknown-status]
# @BRIEF Verify LLM analysis transport/provider failures do not masquerade as dashboard FAIL results.
# @RELATION BELONGS_TO -> [SrcRoot]
# [DEF:TestService:Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @COMPLEXITY: 3
# @SEMANTICS: tests, llm-analysis, fallback, provider-error, unknown-status
# @PURPOSE: Verify LLM analysis transport/provider failures do not masquerade as dashboard FAIL results.
import pytest
@@ -8,11 +10,11 @@ from src.plugins.llm_analysis.models import LLMProviderType
from src.plugins.llm_analysis.service import LLMClient
# #region test_test_runtime_connection_uses_json_completion_transport [TYPE Function]
# @BRIEF Provider self-test must exercise the same chat completion transport as runtime analysis.
# @PRE get_json_completion is available on initialized client.
# @POST Self-test forwards a lightweight user message into get_json_completion and returns its payload.
# @RELATION BINDS_TO -> [TestService]
# [DEF:test_test_runtime_connection_uses_json_completion_transport:Function]
# @RELATION: BINDS_TO -> TestService
# @PURPOSE: Provider self-test must exercise the same chat completion transport as runtime analysis.
# @PRE: get_json_completion is available on initialized client.
# @POST: Self-test forwards a lightweight user message into get_json_completion and returns its payload.
@pytest.mark.anyio
async def test_test_runtime_connection_uses_json_completion_transport(monkeypatch):
client = LLMClient(
@@ -34,14 +36,14 @@ async def test_test_runtime_connection_uses_json_completion_transport(monkeypatc
assert result == {"ok": True}
assert recorded["messages"][0]["role"] == "user"
assert "Return exactly this JSON object" in recorded["messages"][0]["content"]
# #endregion test_test_runtime_connection_uses_json_completion_transport
# [/DEF:test_test_runtime_connection_uses_json_completion_transport:Function]
# #region test_analyze_dashboard_provider_error_maps_to_unknown [TYPE Function]
# @BRIEF Infrastructure/provider failures must produce UNKNOWN analysis status rather than FAIL.
# @PRE LLMClient.get_json_completion raises provider/auth exception.
# @POST Returned payload uses status=UNKNOWN and issue severity UNKNOWN.
# @RELATION BINDS_TO -> [TestService]
# [DEF:test_analyze_dashboard_provider_error_maps_to_unknown:Function]
# @RELATION: BINDS_TO -> TestService
# @PURPOSE: Infrastructure/provider failures must produce UNKNOWN analysis status rather than FAIL.
# @PRE: LLMClient.get_json_completion raises provider/auth exception.
# @POST: Returned payload uses status=UNKNOWN and issue severity UNKNOWN.
@pytest.mark.anyio
async def test_analyze_dashboard_provider_error_maps_to_unknown(monkeypatch, tmp_path):
screenshot_path = tmp_path / "shot.jpg"
@@ -64,5 +66,5 @@ async def test_analyze_dashboard_provider_error_maps_to_unknown(monkeypatch, tmp
assert result["status"] == "UNKNOWN"
assert "Failed to get response from LLM" in result["summary"]
assert result["issues"][0]["severity"] == "UNKNOWN"
# #endregion test_analyze_dashboard_provider_error_maps_to_unknown
# #endregion TestService
# [/DEF:test_analyze_dashboard_provider_error_maps_to_unknown:Function]
# [/DEF:TestService:Module]

24
backend/src/plugins/llm_analysis/models.py
View File

@@ -1,9 +1,9 @@
# #region LLMAnalysisModels [C:3] [TYPE Module] [SEMANTICS pydantic, models, llm]
# @BRIEF Define Pydantic models for LLM Analysis plugin.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [pydantic]
# @RELATION DEPENDS_on -> [pydantic]
# @RELATION DEPENDs_on -> [pydantic]
# @LAYER: Domain
# @RELATION DEPENDS_ON -> pydantic
# @RELATION DEPENDS_on -> pydantic
# @RELATION DEPENDs_on -> pydantic
from typing import List, Optional
from pydantic import BaseModel, Field
@@ -18,22 +18,6 @@ class LLMProviderType(str, Enum):
KILO = "kilo"
# #endregion LLMProviderType
# #region FetchModelsRequest [TYPE Class]
# @BRIEF Request model to fetch available models from a provider.
class FetchModelsRequest(BaseModel):
base_url: str = Field(description="Provider base URL (e.g. https://api.openai.com/v1)")
api_key: Optional[str] = Field(None, description="Optional API key for authenticated providers")
provider_type: LLMProviderType = Field(LLMProviderType.OPENAI, description="Provider type for auth headers")
provider_id: Optional[str] = Field(None, description="Existing provider ID — resolves api_key from DB if not provided directly")
# #endregion FetchModelsRequest
# #region FetchModelsResponse [TYPE Class]
# @BRIEF Response with available model IDs from a provider.
class FetchModelsResponse(BaseModel):
models: List[str] = Field(description="List of available model IDs")
total: int = Field(description="Total number of models found")
# #endregion FetchModelsResponse
# #region LLMProviderConfig [TYPE Class]
# @BRIEF Configuration for an LLM provider.
class LLMProviderConfig(BaseModel):

36
backend/src/plugins/llm_analysis/plugin.py
View File

@@ -1,12 +1,14 @@
# #region backend/src/plugins/llm_analysis/plugin.py [C:3] [TYPE Module] [SEMANTICS plugin, llm, analysis, documentation]
# [DEF:backend/src/plugins/llm_analysis/plugin.py:Module]
# @COMPLEXITY: 3
# @SEMANTICS: plugin, llm, analysis, documentation
# @BRIEF Implements DashboardValidationPlugin and DocumentationPlugin.
# @LAYER Domain
# @INVARIANT All LLM interactions must be executed as asynchronous tasks.
# @LAYER: Domain
# @RELATION INHERITS -> [PluginBase]
# @RELATION CALLS -> [ScreenshotService]
# @RELATION CALLS -> [LLMClient]
# @RELATION CALLS -> [LLMProviderService]
# @RELATION USES -> [TaskContext]
# @RELATION USES -> TaskContext
# @INVARIANT: All LLM interactions must be executed as asynchronous tasks.
from typing import Dict, Any, Optional
import os
@@ -31,8 +33,8 @@ from ...services.llm_prompt_templates import (
# #region _is_masked_or_invalid_api_key [TYPE Function]
# @BRIEF Guards against placeholder or malformed API keys in runtime.
# @PRE value may be None.
# @POST Returns True when value cannot be used for authenticated provider calls.
# @PRE: value may be None.
# @POST: Returns True when value cannot be used for authenticated provider calls.
def _is_masked_or_invalid_api_key(value: Optional[str]) -> bool:
key = (value or "").strip()
if not key:
@@ -45,8 +47,8 @@ def _is_masked_or_invalid_api_key(value: Optional[str]) -> bool:
# #region _json_safe_value [TYPE Function]
# @BRIEF Recursively normalize payload values for JSON serialization.
# @PRE value may be nested dict/list with datetime values.
# @POST datetime values are converted to ISO strings.
# @PRE: value may be nested dict/list with datetime values.
# @POST: datetime values are converted to ISO strings.
def _json_safe_value(value: Any):
if isinstance(value, datetime):
return value.isoformat()
@@ -59,7 +61,7 @@ def _json_safe_value(value: Any):
# #region DashboardValidationPlugin [TYPE Class]
# @BRIEF Plugin for automated dashboard health analysis using LLMs.
# @RELATION IMPLEMENTS -> [backend.src.core.plugin_base.PluginBase]
# @RELATION IMPLEMENTS -> backend.src.core.plugin_base.PluginBase
class DashboardValidationPlugin(PluginBase):
@property
def id(self) -> str:
@@ -88,8 +90,8 @@ class DashboardValidationPlugin(PluginBase):
"required": ["dashboard_id", "environment_id", "provider_id"]
}
# #region DashboardValidationPlugin.execute [TYPE Function]
# @BRIEF Executes the dashboard validation task with TaskContext support.
# [DEF:DashboardValidationPlugin.execute:Function]
# @PURPOSE: Executes the dashboard validation task with TaskContext support.
# @PARAM: params (Dict[str, Any]) - Validation parameters.
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE: params contains dashboard_id, environment_id, and provider_id.
@@ -317,12 +319,12 @@ class DashboardValidationPlugin(PluginBase):
finally:
db.close()
# #endregion DashboardValidationPlugin.execute
# [/DEF:DashboardValidationPlugin.execute:Function]
# #endregion DashboardValidationPlugin
# #region DocumentationPlugin [TYPE Class]
# @BRIEF Plugin for automated dataset documentation using LLMs.
# @RELATION IMPLEMENTS -> [backend.src.core.plugin_base.PluginBase]
# @RELATION IMPLEMENTS -> backend.src.core.plugin_base.PluginBase
class DocumentationPlugin(PluginBase):
@property
def id(self) -> str:
@@ -351,8 +353,8 @@ class DocumentationPlugin(PluginBase):
"required": ["dataset_id", "environment_id", "provider_id"]
}
# #region DocumentationPlugin.execute [TYPE Function]
# @BRIEF Executes the dataset documentation task with TaskContext support.
# [DEF:DocumentationPlugin.execute:Function]
# @PURPOSE: Executes the dataset documentation task with TaskContext support.
# @PARAM: params (Dict[str, Any]) - Documentation parameters.
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE: params contains dataset_id, environment_id, and provider_id.
@@ -473,7 +475,7 @@ class DocumentationPlugin(PluginBase):
finally:
db.close()
# #endregion DocumentationPlugin.execute
# [/DEF:DocumentationPlugin.execute:Function]
# #endregion DocumentationPlugin
# #endregion backend/src/plugins/llm_analysis/plugin.py
# [/DEF:backend/src/plugins/llm_analysis/plugin.py:Module]

13
backend/src/plugins/llm_analysis/scheduler.py
View File

@@ -1,6 +1,8 @@
# #region backend/src/plugins/llm_analysis/scheduler.py [C:3] [TYPE Module] [SEMANTICS scheduler, task, automation]
# [DEF:backend/src/plugins/llm_analysis/scheduler.py:Module]
# @COMPLEXITY: 3
# @SEMANTICS: scheduler, task, automation
# @BRIEF Provides helper functions to schedule LLM-based validation tasks.
# @LAYER Domain
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [SchedulerService]
from typing import Dict, Any
@@ -9,9 +11,6 @@ from ...core.logger import belief_scope, logger
# #region schedule_dashboard_validation [TYPE Function]
# @BRIEF Schedules a recurring dashboard validation task.
# @PARAM: dashboard_id (str) - ID of the dashboard to validate.
# @PARAM: cron_expression (str) - Standard cron expression for scheduling.
# @PARAM: params (Dict[str, Any]) - Task parameters (environment_id, provider_id).
# @SIDE_EFFECT: Adds a job to the scheduler service.
def schedule_dashboard_validation(dashboard_id: str, cron_expression: str, params: Dict[str, Any]):
with belief_scope("schedule_dashboard_validation", f"dashboard_id={dashboard_id}"):
@@ -41,8 +40,6 @@ def schedule_dashboard_validation(dashboard_id: str, cron_expression: str, param
# #region _parse_cron [TYPE Function]
# @BRIEF Basic cron parser placeholder.
# @PARAM: cron (str) - Cron expression.
# @RETURN: Dict[str, str] - Parsed cron parts.
def _parse_cron(cron: str) -> Dict[str, str]:
# Basic cron parser placeholder
parts = cron.split()
@@ -57,4 +54,4 @@ def _parse_cron(cron: str) -> Dict[str, str]:
}
# #endregion _parse_cron
# #endregion backend/src/plugins/llm_analysis/scheduler.py
# [/DEF:backend/src/plugins/llm_analysis/scheduler.py:Module]

190
backend/src/plugins/llm_analysis/service.py
View File

@@ -1,10 +1,12 @@
# #region backend/src/plugins/llm_analysis/service.py [C:3] [TYPE Module] [SEMANTICS service, llm, screenshot, playwright, openai]
# [DEF:backend/src/plugins/llm_analysis/service.py:Module]
# @COMPLEXITY: 3
# @SEMANTICS: service, llm, screenshot, playwright, openai
# @BRIEF Services for LLM interaction and dashboard screenshots.
# @LAYER Domain
# @INVARIANT Screenshots must be 1920px width and capture full page height.
# @RELATION DEPENDS_ON -> [playwright]
# @RELATION DEPENDS_ON -> [openai]
# @RELATION DEPENDS_ON -> [tenacity]
# @LAYER: Domain
# @RELATION DEPENDS_ON -> playwright
# @RELATION DEPENDS_ON -> openai
# @RELATION DEPENDS_ON -> tenacity
# @INVARIANT: Screenshots must be 1920px width and capture full page height.
import asyncio
import base64
@@ -26,17 +28,17 @@ from ...services.llm_prompt_templates import DEFAULT_LLM_PROMPTS, render_prompt
# #region ScreenshotService [TYPE Class]
# @BRIEF Handles capturing screenshots of Superset dashboards.
class ScreenshotService:
# #region ScreenshotService.__init__ [TYPE Function]
# @BRIEF Initializes the ScreenshotService with environment configuration.
# @PRE env is a valid Environment object.
# [DEF:ScreenshotService.__init__:Function]
# @PURPOSE: Initializes the ScreenshotService with environment configuration.
# @PRE: env is a valid Environment object.
def __init__(self, env: Environment):
self.env = env
# #endregion ScreenshotService.__init__
# [/DEF:ScreenshotService.__init__:Function]
# #region ScreenshotService._find_first_visible_locator [TYPE Function]
# @BRIEF Resolve the first visible locator from multiple Playwright locator strategies.
# @PRE candidates is a non-empty list of locator-like objects.
# @POST Returns a locator ready for interaction or None when nothing matches.
# [DEF:ScreenshotService._find_first_visible_locator:Function]
# @PURPOSE: Resolve the first visible locator from multiple Playwright locator strategies.
# @PRE: candidates is a non-empty list of locator-like objects.
# @POST: Returns a locator ready for interaction or None when nothing matches.
async def _find_first_visible_locator(self, candidates) -> Any:
for locator in candidates:
try:
@@ -48,12 +50,12 @@ class ScreenshotService:
except Exception:
continue
return None
# #endregion ScreenshotService._find_first_visible_locator
# [/DEF:ScreenshotService._find_first_visible_locator:Function]
# #region ScreenshotService._iter_login_roots [TYPE Function]
# @BRIEF Enumerate page and child frames where login controls may be rendered.
# @PRE page is a Playwright page-like object.
# @POST Returns ordered roots starting with main page followed by frames.
# [DEF:ScreenshotService._iter_login_roots:Function]
# @PURPOSE: Enumerate page and child frames where login controls may be rendered.
# @PRE: page is a Playwright page-like object.
# @POST: Returns ordered roots starting with main page followed by frames.
def _iter_login_roots(self, page) -> List[Any]:
roots = [page]
page_frames = getattr(page, "frames", [])
@@ -64,12 +66,12 @@ class ScreenshotService:
except Exception:
pass
return roots
# #endregion ScreenshotService._iter_login_roots
# [/DEF:ScreenshotService._iter_login_roots:Function]
# #region ScreenshotService._extract_hidden_login_fields [TYPE Function]
# @BRIEF Collect hidden form fields required for direct login POST fallback.
# @PRE Login page is loaded.
# @POST Returns hidden input name/value mapping aggregated from page and child frames.
# [DEF:ScreenshotService._extract_hidden_login_fields:Function]
# @PURPOSE: Collect hidden form fields required for direct login POST fallback.
# @PRE: Login page is loaded.
# @POST: Returns hidden input name/value mapping aggregated from page and child frames.
async def _extract_hidden_login_fields(self, page) -> Dict[str, str]:
hidden_fields: Dict[str, str] = {}
for root in self._iter_login_roots(page):
@@ -85,21 +87,21 @@ class ScreenshotService:
except Exception:
continue
return hidden_fields
# #endregion ScreenshotService._extract_hidden_login_fields
# [/DEF:ScreenshotService._extract_hidden_login_fields:Function]
# #region ScreenshotService._extract_csrf_token [TYPE Function]
# @BRIEF Resolve CSRF token value from main page or embedded login frame.
# @PRE Login page is loaded.
# @POST Returns first non-empty csrf token or empty string.
# [DEF:ScreenshotService._extract_csrf_token:Function]
# @PURPOSE: Resolve CSRF token value from main page or embedded login frame.
# @PRE: Login page is loaded.
# @POST: Returns first non-empty csrf token or empty string.
async def _extract_csrf_token(self, page) -> str:
hidden_fields = await self._extract_hidden_login_fields(page)
return str(hidden_fields.get("csrf_token") or "").strip()
# #endregion ScreenshotService._extract_csrf_token
# [/DEF:ScreenshotService._extract_csrf_token:Function]
# #region ScreenshotService._response_looks_like_login_page [TYPE Function]
# @BRIEF Detect when fallback login POST returned the login form again instead of an authenticated page.
# @PRE response_text is normalized HTML or text from login POST response.
# @POST Returns True when login-page markers dominate the response body.
# [DEF:ScreenshotService._response_looks_like_login_page:Function]
# @PURPOSE: Detect when fallback login POST returned the login form again instead of an authenticated page.
# @PRE: response_text is normalized HTML or text from login POST response.
# @POST: Returns True when login-page markers dominate the response body.
def _response_looks_like_login_page(self, response_text: str) -> bool:
normalized = str(response_text or "").strip().lower()
if not normalized:
@@ -113,23 +115,23 @@ class ScreenshotService:
'name="csrf_token"',
]
return sum(marker in normalized for marker in markers) >= 3
# #endregion ScreenshotService._response_looks_like_login_page
# [/DEF:ScreenshotService._response_looks_like_login_page:Function]
# #region ScreenshotService._redirect_looks_authenticated [TYPE Function]
# @BRIEF Treat non-login redirects after form POST as successful authentication without waiting for redirect target.
# @PRE redirect_location may be empty or relative.
# @POST Returns True when redirect target does not point back to login flow.
# [DEF:ScreenshotService._redirect_looks_authenticated:Function]
# @PURPOSE: Treat non-login redirects after form POST as successful authentication without waiting for redirect target.
# @PRE: redirect_location may be empty or relative.
# @POST: Returns True when redirect target does not point back to login flow.
def _redirect_looks_authenticated(self, redirect_location: str) -> bool:
normalized = str(redirect_location or "").strip().lower()
if not normalized:
return True
return "/login" not in normalized
# #endregion ScreenshotService._redirect_looks_authenticated
# [/DEF:ScreenshotService._redirect_looks_authenticated:Function]
# #region ScreenshotService._submit_login_via_form_post [TYPE Function]
# @BRIEF Fallback login path that submits credentials directly with csrf token.
# @PRE login_url is same-origin and csrf token can be read from DOM.
# @POST Browser context receives authenticated cookies when login succeeds.
# [DEF:ScreenshotService._submit_login_via_form_post:Function]
# @PURPOSE: Fallback login path that submits credentials directly with csrf token.
# @PRE: login_url is same-origin and csrf token can be read from DOM.
# @POST: Browser context receives authenticated cookies when login succeeds.
async def _submit_login_via_form_post(self, page, login_url: str) -> bool:
hidden_fields = await self._extract_hidden_login_fields(page)
csrf_token = str(hidden_fields.get("csrf_token") or "").strip()
@@ -186,12 +188,12 @@ class ScreenshotService:
f"[DEBUG] Direct form login fallback response: status={response_status} url={response_url} login_markup={looks_like_login_page} snippet={text_snippet!r}"
)
return not looks_like_login_page
# #endregion ScreenshotService._submit_login_via_form_post
# [/DEF:ScreenshotService._submit_login_via_form_post:Function]
# #region ScreenshotService._find_login_field_locator [TYPE Function]
# @BRIEF Resolve login form input using semantic label text plus generic visible-input fallbacks.
# @PRE field_name is `username` or `password`.
# @POST Returns a locator for the corresponding input or None.
# [DEF:ScreenshotService._find_login_field_locator:Function]
# @PURPOSE: Resolve login form input using semantic label text plus generic visible-input fallbacks.
# @PRE: field_name is `username` or `password`.
# @POST: Returns a locator for the corresponding input or None.
async def _find_login_field_locator(self, page, field_name: str) -> Any:
normalized = str(field_name or "").strip().lower()
for root in self._iter_login_roots(page):
@@ -226,12 +228,12 @@ class ScreenshotService:
return locator
return None
# #endregion ScreenshotService._find_login_field_locator
# [/DEF:ScreenshotService._find_login_field_locator:Function]
# #region ScreenshotService._find_submit_locator [TYPE Function]
# @BRIEF Resolve login submit button from main page or embedded auth frame.
# @PRE page is ready for login interaction.
# @POST Returns visible submit locator or None.
# [DEF:ScreenshotService._find_submit_locator:Function]
# @PURPOSE: Resolve login submit button from main page or embedded auth frame.
# @PRE: page is ready for login interaction.
# @POST: Returns visible submit locator or None.
async def _find_submit_locator(self, page) -> Any:
selectors = [
lambda root: root.get_by_role("button", name="Sign in", exact=False),
@@ -246,12 +248,12 @@ class ScreenshotService:
if locator:
return locator
return None
# #endregion ScreenshotService._find_submit_locator
# [/DEF:ScreenshotService._find_submit_locator:Function]
# #region ScreenshotService._goto_resilient [TYPE Function]
# @BRIEF Navigate without relying on networkidle for pages with long-polling or persistent requests.
# @PRE page is a valid Playwright page and url is non-empty.
# @POST Returns last navigation response or raises when both primary and fallback waits fail.
# [DEF:ScreenshotService._goto_resilient:Function]
# @PURPOSE: Navigate without relying on networkidle for pages with long-polling or persistent requests.
# @PRE: page is a valid Playwright page and url is non-empty.
# @POST: Returns last navigation response or raises when both primary and fallback waits fail.
async def _goto_resilient(
self,
page,
@@ -267,13 +269,13 @@ class ScreenshotService:
f"[ScreenshotService._goto_resilient] Primary navigation wait '{primary_wait_until}' failed for {url}: {primary_error}"
)
return await page.goto(url, wait_until=fallback_wait_until, timeout=timeout)
# #endregion ScreenshotService._goto_resilient
# [/DEF:ScreenshotService._goto_resilient:Function]
# #region ScreenshotService.capture_dashboard [TYPE Function]
# @BRIEF Captures a full-page screenshot of a dashboard using Playwright and CDP.
# @PRE dashboard_id is a valid string, output_path is a writable path.
# @POST Returns True if screenshot is saved successfully.
# @SIDE_EFFECT Launches a browser, performs UI login, switches tabs, and writes a PNG file.
# [DEF:ScreenshotService.capture_dashboard:Function]
# @PURPOSE: Captures a full-page screenshot of a dashboard using Playwright and CDP.
# @PRE: dashboard_id is a valid string, output_path is a writable path.
# @POST: Returns True if screenshot is saved successfully.
# @SIDE_EFFECT: Launches a browser, performs UI login, switches tabs, and writes a PNG file.
# @UX_STATE: [Navigating] -> Loading dashboard UI
# @UX_STATE: [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading
# @UX_STATE: [CalculatingHeight] -> Determining dashboard dimensions
@@ -669,15 +671,15 @@ class ScreenshotService:
await browser.close()
return True
# #endregion ScreenshotService.capture_dashboard
# [/DEF:ScreenshotService.capture_dashboard:Function]
# #endregion ScreenshotService
# #region LLMClient [TYPE Class]
# @BRIEF Wrapper for LLM provider APIs.
class LLMClient:
# #region LLMClient.__init__ [TYPE Function]
# @BRIEF Initializes the LLMClient with provider settings.
# @PRE api_key, base_url, and default_model are non-empty strings.
# [DEF:LLMClient.__init__:Function]
# @PURPOSE: Initializes the LLMClient with provider settings.
# @PRE: api_key, base_url, and default_model are non-empty strings.
def __init__(self, provider_type: LLMProviderType, api_key: str, base_url: str, default_model: str):
self.provider_type = provider_type
normalized_key = (api_key or "").strip()
@@ -715,12 +717,12 @@ class LLMClient:
default_headers=default_headers,
http_client=http_client,
)
# #endregion LLMClient.__init__
# [/DEF:LLMClient.__init__:Function]
# #region LLMClient._supports_json_response_format [TYPE Function]
# @BRIEF Detect whether provider/model is likely compatible with response_format=json_object.
# @PRE Client initialized with base_url and default_model.
# @POST Returns False for known-incompatible combinations to avoid avoidable 400 errors.
# [DEF:LLMClient._supports_json_response_format:Function]
# @PURPOSE: Detect whether provider/model is likely compatible with response_format=json_object.
# @PRE: Client initialized with base_url and default_model.
# @POST: Returns False for known-incompatible combinations to avoid avoidable 400 errors.
def _supports_json_response_format(self) -> bool:
base = (self.base_url or "").lower()
model = (self.default_model or "").lower()
@@ -735,13 +737,13 @@ class LLMClient:
if any(token in model for token in incompatible_tokens):
return False
return True
# #endregion LLMClient._supports_json_response_format
# [/DEF:LLMClient._supports_json_response_format:Function]
# #region LLMClient.get_json_completion [TYPE Function]
# @BRIEF Helper to handle LLM calls with JSON mode and fallback parsing.
# @PRE messages is a list of valid message dictionaries.
# @POST Returns a parsed JSON dictionary.
# @SIDE_EFFECT Calls external LLM API.
# [DEF:LLMClient.get_json_completion:Function]
# @PURPOSE: Helper to handle LLM calls with JSON mode and fallback parsing.
# @PRE: messages is a list of valid message dictionaries.
# @POST: Returns a parsed JSON dictionary.
# @SIDE_EFFECT: Calls external LLM API.
def _should_retry(exception: Exception) -> bool:
"""Custom retry predicate that excludes authentication errors."""
# Don't retry on authentication errors
@@ -857,13 +859,13 @@ class LLMClient:
return json.loads(json_str)
else:
raise
# #endregion LLMClient.get_json_completion
# [/DEF:LLMClient.get_json_completion:Function]
# #region LLMClient.test_runtime_connection [TYPE Function]
# @BRIEF Validate provider credentials using the same chat completions transport as runtime analysis.
# @PRE Client is initialized with provider credentials and default_model.
# @POST Returns lightweight JSON payload when runtime auth/model path is valid.
# @SIDE_EFFECT Calls external LLM API.
# [DEF:LLMClient.test_runtime_connection:Function]
# @PURPOSE: Validate provider credentials using the same chat completions transport as runtime analysis.
# @PRE: Client is initialized with provider credentials and default_model.
# @POST: Returns lightweight JSON payload when runtime auth/model path is valid.
# @SIDE_EFFECT: Calls external LLM API.
async def test_runtime_connection(self) -> Dict[str, Any]:
with belief_scope("test_runtime_connection"):
messages = [
@@ -873,13 +875,13 @@ class LLMClient:
}
]
return await self.get_json_completion(messages)
# #endregion LLMClient.test_runtime_connection
# [/DEF:LLMClient.test_runtime_connection:Function]
# #region LLMClient.analyze_dashboard [TYPE Function]
# @BRIEF Sends dashboard data (screenshot + logs) to LLM for health analysis.
# @PRE screenshot_path exists, logs is a list of strings.
# @POST Returns a structured analysis dictionary (status, summary, issues).
# @SIDE_EFFECT Reads screenshot file and calls external LLM API.
# [DEF:LLMClient.analyze_dashboard:Function]
# @PURPOSE: Sends dashboard data (screenshot + logs) to LLM for health analysis.
# @PRE: screenshot_path exists, logs is a list of strings.
# @POST: Returns a structured analysis dictionary (status, summary, issues).
# @SIDE_EFFECT: Reads screenshot file and calls external LLM API.
async def analyze_dashboard(
self,
screenshot_path: str,
@@ -945,7 +947,7 @@ class LLMClient:
"summary": f"Failed to get response from LLM: {str(e)}",
"issues": [{"severity": "UNKNOWN", "message": "LLM provider returned empty or invalid response"}]
}
# #endregion LLMClient.analyze_dashboard
# [/DEF:LLMClient.analyze_dashboard:Function]
# #endregion LLMClient
# #endregion backend/src/plugins/llm_analysis/service.py
# [/DEF:backend/src/plugins/llm_analysis/service.py:Module]

82
backend/src/plugins/mapper.py
View File

@@ -1,10 +1,9 @@
# #region MapperPluginModule [TYPE Module] [SEMANTICS plugin, mapper, datasets, postgresql, excel]
# @BRIEF Implements a plugin for mapping dataset columns using external database connections or Excel files.
# @LAYER Plugins
# @LAYER: Plugins
# @RELATION Inherits from PluginBase. Uses DatasetMapper from superset_tool.
# @RELATION USES -> [TaskContext]
# @RELATION USES -> TaskContext
# [SECTION: IMPORTS]
from typing import Dict, Any, Optional
from ..core.plugin_base import PluginBase
from ..core.superset_client import SupersetClient
@@ -13,7 +12,6 @@ from ..core.database import SessionLocal
from ..models.connection import ConnectionConfig
from ..core.utils.dataset_mapper import DatasetMapper
from ..core.task_manager.context import TaskContext
# [/SECTION]
# #region MapperPlugin [TYPE Class]
# @BRIEF Plugin for mapping dataset columns verbose names.
@@ -23,63 +21,63 @@ class MapperPlugin(PluginBase):
"""
@property
# #region id [TYPE Function]
# @BRIEF Returns the unique identifier for the mapper plugin.
# @PRE Plugin instance exists.
# @POST Returns string ID.
# @RETURN str - "dataset-mapper"
# [DEF:id:Function]
# @PURPOSE: Returns the unique identifier for the mapper plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string ID.
# @RETURN: str - "dataset-mapper"
def id(self) -> str:
with belief_scope("id"):
return "dataset-mapper"
# #endregion id
# [/DEF:id:Function]
@property
# #region name [TYPE Function]
# @BRIEF Returns the human-readable name of the mapper plugin.
# @PRE Plugin instance exists.
# @POST Returns string name.
# @RETURN str - Plugin name.
# [DEF:name:Function]
# @PURPOSE: Returns the human-readable name of the mapper plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string name.
# @RETURN: str - Plugin name.
def name(self) -> str:
with belief_scope("name"):
return "Dataset Mapper"
# #endregion name
# [/DEF:name:Function]
@property
# #region description [TYPE Function]
# @BRIEF Returns a description of the mapper plugin.
# @PRE Plugin instance exists.
# @POST Returns string description.
# @RETURN str - Plugin description.
# [DEF:description:Function]
# @PURPOSE: Returns a description of the mapper plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string description.
# @RETURN: str - Plugin description.
def description(self) -> str:
with belief_scope("description"):
return "Map dataset column verbose names using PostgreSQL comments or Excel files."
# #endregion description
# [/DEF:description:Function]
@property
# #region version [TYPE Function]
# @BRIEF Returns the version of the mapper plugin.
# @PRE Plugin instance exists.
# @POST Returns string version.
# @RETURN str - "1.0.0"
# [DEF:version:Function]
# @PURPOSE: Returns the version of the mapper plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string version.
# @RETURN: str - "1.0.0"
def version(self) -> str:
with belief_scope("version"):
return "1.0.0"
# #endregion version
# [/DEF:version:Function]
@property
# #region ui_route [TYPE Function]
# @BRIEF Returns the frontend route for the mapper plugin.
# @RETURN str - "/tools/mapper"
# [DEF:ui_route:Function]
# @PURPOSE: Returns the frontend route for the mapper plugin.
# @RETURN: str - "/tools/mapper"
def ui_route(self) -> str:
with belief_scope("ui_route"):
return "/tools/mapper"
# #endregion ui_route
# [/DEF:ui_route:Function]
# #region get_schema [TYPE Function]
# @BRIEF Returns the JSON schema for the mapper plugin parameters.
# @PRE Plugin instance exists.
# @POST Returns dictionary schema.
# @RETURN Dict[str, Any] - JSON schema.
# [DEF:get_schema:Function]
# @PURPOSE: Returns the JSON schema for the mapper plugin parameters.
# @PRE: Plugin instance exists.
# @POST: Returns dictionary schema.
# @RETURN: Dict[str, Any] - JSON schema.
def get_schema(self) -> Dict[str, Any]:
with belief_scope("get_schema"):
return {
@@ -125,10 +123,10 @@ class MapperPlugin(PluginBase):
},
"required": ["env", "dataset_id", "source"]
}
# #endregion get_schema
# [/DEF:get_schema:Function]
# #region execute [TYPE Function]
# @BRIEF Executes the dataset mapping logic with TaskContext support.
# [DEF:execute:Function]
# @PURPOSE: Executes the dataset mapping logic with TaskContext support.
# @PARAM: params (Dict[str, Any]) - Mapping parameters.
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE: Params contain valid 'env', 'dataset_id', and 'source'. params must be a dictionary.
@@ -207,7 +205,7 @@ class MapperPlugin(PluginBase):
except Exception as e:
log.error(f"Mapping failed: {e}")
raise
# #endregion execute
# [/DEF:execute:Function]
# #endregion MapperPlugin
# #endregion MapperPluginModule
# #endregion MapperPluginModule

167
backend/src/plugins/migration.py
View File

@@ -1,26 +1,23 @@
# #region MigrationPlugin [C:5] [TYPE Module] [SEMANTICS migration, superset, automation, dashboard, plugin, transformation]
# @BRIEF Orchestrates export, DB-mapping transformation, and import of Superset dashboards across environments.
# @LAYER App
# @PRE Plugin loader can resolve infrastructure dependencies and execution requests provide validated migration context.
# @POST Plugin metadata remains stable and migration execution preserves mapped-environment import guarantees.
# @SIDE_EFFECT Reads config, opens database sessions, creates temporary artifacts, and triggers Superset export/import workflows.
# @DATA_CONTRACT Input[TaskContext{from_env,to_env,dashboard_regex,replace_db_config,from_db_id,to_db_id,passwords?}] -> Output[MigrationResult|artifact set]
# @INVARIANT Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution.
# @RELATION IMPLEMENTS -> [PluginBase]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [MigrationEngine]
# @RELATION DEPENDS_ON -> [IdMappingService]
# @RELATION USES -> [TaskContext]
# @LAYER: App
# @RELATION IMPLEMENTS -> PluginBase
# @RELATION DEPENDS_ON -> SupersetClient
# @RELATION DEPENDS_ON -> MigrationEngine
# @RELATION DEPENDS_ON -> IdMappingService
# @RELATION USES -> TaskContext
# @PRE: Plugin loader can resolve infrastructure dependencies and execution requests provide validated migration context.
# @POST: Plugin metadata remains stable and migration execution preserves mapped-environment import guarantees.
# @SIDE_EFFECT: Reads config, opens database sessions, creates temporary artifacts, and triggers Superset export/import workflows.
# @DATA_CONTRACT: Input[TaskContext{from_env,to_env,dashboard_regex,replace_db_config,from_db_id,to_db_id,passwords?}] -> Output[MigrationResult|artifact set]
# @INVARIANT: Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution.
from typing import Dict, Any, Optional
import re
from ..core.plugin_base import PluginBase
from ..core.logger import belief_scope
from ..core.cot_logger import MarkerLogger
from ..core.logger import belief_scope, logger as app_logger
from ..core.superset_client import SupersetClient
log = MarkerLogger("MigrationPlugin")
from ..core.utils.fileio import create_temp_file
from ..dependencies import get_config_manager
from ..core.migration_engine import MigrationEngine
@@ -31,8 +28,8 @@ from ..core.task_manager.context import TaskContext
# #region MigrationPlugin [TYPE Class]
# @BRIEF Implementation of the migration plugin workflow and transformation orchestration.
# @PRE SupersetClient authenticated, database session active
# @POST Returns MigrationResult with success/failure status and artifact list
# @PRE: SupersetClient authenticated, database session active
# @POST: Returns MigrationResult with success/failure status and artifact list
# @TEST_FIXTURE: superset_export_zip -> file:backend/tests/fixtures/migration/dashboard_export.zip
# @TEST_FIXTURE: db_mapping_payload -> INLINE_JSON: {"db_mappings": {"source_uuid_1": "target_uuid_2"}}
# @TEST_FIXTURE: password_inject_payload -> INLINE_JSON: {"passwords": {"PostgreSQL": "secret123"}}
@@ -45,68 +42,68 @@ class MigrationPlugin(PluginBase):
"""
@property
# #region id [TYPE Function]
# @BRIEF Returns the unique identifier for the migration plugin.
# @PRE None.
# @POST Returns stable string "superset-migration".
# @RETURN str
# [DEF:id:Function]
# @PURPOSE: Returns the unique identifier for the migration plugin.
# @PRE: None.
# @POST: Returns stable string "superset-migration".
# @RETURN: str
def id(self) -> str:
with belief_scope("MigrationPlugin.id"):
return "superset-migration"
# #endregion id
# [/DEF:id:Function]
@property
# #region name [TYPE Function]
# @BRIEF Returns the human-readable name of the plugin.
# @PRE None.
# @POST Returns "Superset Dashboard Migration".
# @RETURN str
# [DEF:name:Function]
# @PURPOSE: Returns the human-readable name of the plugin.
# @PRE: None.
# @POST: Returns "Superset Dashboard Migration".
# @RETURN: str
def name(self) -> str:
with belief_scope("MigrationPlugin.name"):
return "Superset Dashboard Migration"
# #endregion name
# [/DEF:name:Function]
@property
# #region description [TYPE Function]
# @BRIEF Returns the semantic description of the plugin.
# @PRE None.
# @POST Returns description string.
# @RETURN str
# [DEF:description:Function]
# @PURPOSE: Returns the semantic description of the plugin.
# @PRE: None.
# @POST: Returns description string.
# @RETURN: str
def description(self) -> str:
with belief_scope("MigrationPlugin.description"):
return "Migrates dashboards between Superset environments."
# #endregion description
# [/DEF:description:Function]
@property
# #region version [TYPE Function]
# @BRIEF Returns the semantic version of the migration plugin.
# @PRE None.
# @POST Returns "1.0.0".
# @RETURN str
# [DEF:version:Function]
# @PURPOSE: Returns the semantic version of the migration plugin.
# @PRE: None.
# @POST: Returns "1.0.0".
# @RETURN: str
def version(self) -> str:
with belief_scope("MigrationPlugin.version"):
return "1.0.0"
# #endregion version
# [/DEF:version:Function]
@property
# #region ui_route [TYPE Function]
# @BRIEF Returns the frontend routing anchor for the plugin.
# @PRE None.
# @POST Returns "/migration".
# @RETURN str
# [DEF:ui_route:Function]
# @PURPOSE: Returns the frontend routing anchor for the plugin.
# @PRE: None.
# @POST: Returns "/migration".
# @RETURN: str
def ui_route(self) -> str:
with belief_scope("MigrationPlugin.ui_route"):
return "/migration"
# #endregion ui_route
# [/DEF:ui_route:Function]
# #region get_schema [TYPE Function]
# @BRIEF Generates the JSON Schema for the plugin execution form dynamically.
# @PRE ConfigManager is accessible and environments are defined.
# @POST Returns a JSON Schema dict matching current system environments.
# @RETURN Dict[str, Any]
# [DEF:get_schema:Function]
# @PURPOSE: Generates the JSON Schema for the plugin execution form dynamically.
# @PRE: ConfigManager is accessible and environments are defined.
# @POST: Returns a JSON Schema dict matching current system environments.
# @RETURN: Dict[str, Any]
def get_schema(self) -> Dict[str, Any]:
with belief_scope("MigrationPlugin.get_schema"):
log.reason("Generating migration UI schema")
app_logger.reason("Generating migration UI schema")
config_manager = get_config_manager()
envs = [e.name for e in config_manager.get_environments()]
@@ -149,12 +146,12 @@ class MigrationPlugin(PluginBase):
},
"required": ["from_env", "to_env", "dashboard_regex"],
}
log.reflect("Schema generated successfully", payload={"environments_count": len(envs)})
app_logger.reflect("Schema generated successfully", extra={"environments_count": len(envs)})
return schema
# #endregion get_schema
# [/DEF:get_schema:Function]
# #region execute [TYPE Function]
# @BRIEF Orchestrates the dashboard migration pipeline including extraction, AST mutation, and ingestion.
# [DEF:execute:Function]
# @PURPOSE: Orchestrates the dashboard migration pipeline including extraction, AST mutation, and ingestion.
# @PARAM: params (Dict[str, Any]) - Extracted parameters from UI/API execution request.
# @PARAM: context (Optional[TaskContext]) - Dependency injected TaskContext for IO tracing.
# @PRE: Source and target environments must resolve. Matching dashboards must exist.
@@ -169,7 +166,7 @@ class MigrationPlugin(PluginBase):
# @TEST_EDGE: target_api_timeout -> [Dashboard added to failed_dashboards, task concludes with PARTIAL_SUCCESS]
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
with belief_scope("MigrationPlugin.execute"):
log.reason("Evaluating migration task parameters", payload={"params": params})
app_logger.reason("Evaluating migration task parameters", extra={"params": params})
source_env_id = params.get("source_env_id")
target_env_id = params.get("target_env_id")
@@ -185,11 +182,11 @@ class MigrationPlugin(PluginBase):
from ..dependencies import get_task_manager
tm = get_task_manager()
exec_log = context.logger if context else log
superset_log = exec_log.with_source("superset_api") if context else exec_log
migration_log = exec_log.with_source("migration") if context else exec_log
log = context.logger if context else app_logger
superset_log = log.with_source("superset_api") if context else log
migration_log = log.with_source("migration") if context else log
exec_log.info("Starting migration task.")
log.info("Starting migration task.")
try:
config_manager = get_config_manager()
@@ -200,13 +197,13 @@ class MigrationPlugin(PluginBase):
tgt_env = next((e for e in environments if e.id == target_env_id), None) if target_env_id else next((e for e in environments if e.name == to_env_name), None)
if not src_env or not tgt_env:
log.explore("Environment resolution failed", error="Could not resolve source or target environment", payload={"src": source_env_id or from_env_name, "tgt": target_env_id or to_env_name})
app_logger.explore("Environment resolution failed", extra={"src": source_env_id or from_env_name, "tgt": target_env_id or to_env_name})
raise ValueError(f"Could not resolve source or target environment. Source: {source_env_id or from_env_name}, Target: {target_env_id or to_env_name}")
from_env_name = src_env.name
to_env_name = tgt_env.name
log.reason("Environments resolved successfully", payload={"from": from_env_name, "to": to_env_name})
app_logger.reason("Environments resolved successfully", extra={"from": from_env_name, "to": to_env_name})
migration_result = {
"status": "SUCCESS",
@@ -233,12 +230,12 @@ class MigrationPlugin(PluginBase):
regex_pattern = re.compile(str(dashboard_regex), re.IGNORECASE)
dashboards_to_migrate = [d for d in all_dashboards if regex_pattern.search(d.get("dashboard_title", ""))]
else:
log.explore("No deterministic selection criteria provided", error="No dashboard selection criteria (selected_ids or dashboard_regex)")
app_logger.explore("No deterministic selection criteria provided")
migration_result["status"] = "NO_SELECTION"
return migration_result
if not dashboards_to_migrate:
log.explore("Zero dashboards match selection criteria", error="No dashboards matched the given selection criteria")
app_logger.explore("Zero dashboards match selection criteria")
migration_result["status"] = "NO_MATCHES"
return migration_result
@@ -250,7 +247,7 @@ class MigrationPlugin(PluginBase):
db_mapping = {}
if replace_db_config:
log.reason("Fetching environment DB mappings from catalog")
app_logger.reason("Fetching environment DB mappings from catalog")
db = SessionLocal()
try:
src_env_db = db.query(Environment).filter(Environment.name == from_env_name).first()
@@ -264,7 +261,7 @@ class MigrationPlugin(PluginBase):
stored_map_dict = {m.source_db_uuid: m.target_db_uuid for m in stored_mappings}
stored_map_dict.update(db_mapping)
db_mapping = stored_map_dict
exec_log.info(f"Loaded {len(stored_mappings)} database mappings from database.")
log.info(f"Loaded {len(stored_mappings)} database mappings from database.")
finally:
db.close()
@@ -274,7 +271,7 @@ class MigrationPlugin(PluginBase):
# Migration Loop
for dash in dashboards_to_migrate:
dash_id, dash_slug, title = dash["id"], dash.get("slug"), dash["dashboard_title"]
log.reason(f"Starting pipeline for dashboard '{title}'", payload={"dash_id": dash_id})
app_logger.reason(f"Starting pipeline for dashboard '{title}'", extra={"dash_id": dash_id})
try:
exported_content, _ = from_c.export_dashboard(dash_id)
@@ -292,10 +289,10 @@ class MigrationPlugin(PluginBase):
if not success and replace_db_config:
if task_id:
log.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", error="AST transform blocked by missing mappings", payload={"task_id": task_id})
app_logger.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", extra={"task_id": task_id})
await tm.wait_for_resolution(task_id)
log.reason("Task resumed, re-evaluating mapping states")
app_logger.reason("Task resumed, re-evaluating mapping states")
db = SessionLocal()
try:
src_env_rt = db.query(Environment).filter(Environment.name == from_env_name).first()
@@ -318,12 +315,12 @@ class MigrationPlugin(PluginBase):
)
if success:
log.reason("Pushing transformed ZIP to target Superset")
app_logger.reason("Pushing transformed ZIP to target Superset")
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug)
migration_result["migrated_dashboards"].append({"id": dash_id, "title": title})
log.reflect("Import successful", payload={"title": title})
app_logger.reflect("Import successful", extra={"title": title})
else:
log.explore("Transformation strictly failed, bypassing ingestion", error="Dashboard ZIP transformation failed")
app_logger.explore("Transformation strictly failed, bypassing ingestion")
migration_log.error(f"Failed to transform ZIP for dashboard {title}")
migration_result["failed_dashboards"].append({
"id": dash_id, "title": title, "error": "Failed to transform ZIP"
@@ -341,7 +338,7 @@ class MigrationPlugin(PluginBase):
if match_alt:
db_name = match_alt.group(1)
log.explore("Missing DB password detected during ingestion. Escalating to UI.", error="Missing DB password for import", payload={"db_name": db_name})
app_logger.explore(f"Missing DB password detected during ingestion. Escalating to UI.", extra={"db_name": db_name})
if task_id:
tm.await_input(task_id, {
@@ -355,15 +352,15 @@ class MigrationPlugin(PluginBase):
passwords = task.params.get("passwords", {})
if passwords:
log.reason(f"Retrying import for {title} with injected credentials")
app_logger.reason(f"Retrying import for {title} with injected credentials")
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug, passwords=passwords)
migration_result["migrated_dashboards"].append({"id": dash_id, "title": title})
log.reflect("Password injection unblocked import")
app_logger.reflect("Password injection unblocked import")
if "passwords" in task.params:
del task.params["passwords"]
continue
log.explore("Catastrophic dashboard ingestion failure", error=f"Dashboard ingestion failed: {exc}")
app_logger.explore(f"Catastrophic dashboard ingestion failure: {exc}")
migration_result["failed_dashboards"].append({"id": dash_id, "title": title, "error": str(exc)})
if migration_result["failed_dashboards"]:
@@ -371,21 +368,21 @@ class MigrationPlugin(PluginBase):
# Post-Migration ID Mapping Synchronization
try:
log.reason("Executing incremental ID catalog sync on target")
app_logger.reason("Executing incremental ID catalog sync on target")
db_session = SessionLocal()
mapping_service = IdMappingService(db_session)
mapping_service.sync_environment(tgt_env.id, to_c, incremental=True)
db_session.close()
log.reflect("Incremental catalog sync closed out cleanly")
app_logger.reflect("Incremental catalog sync closed out cleanly")
except Exception as sync_exc:
log.explore("ID Mapping sync failed, mapping state might be degraded", error=f"ID Mapping sync error: {sync_exc}")
app_logger.explore(f"ID Mapping sync failed, mapping state might be degraded: {sync_exc}")
log.reflect("Migration cycle fully resolved", payload={"result": migration_result})
app_logger.reflect("Migration cycle fully resolved", extra={"result": migration_result})
return migration_result
except Exception as e:
log.explore("Fatal plugin failure", error=f"Plugin execution failed: {e}", exc_info=True)
app_logger.explore(f"Fatal plugin failure: {e}", exc_info=True)
raise e
# #endregion execute
# #endregion MigrationPlugin
# [/DEF:execute:Function]
# #endregion MigrationPlugin
# #endregion MigrationPlugin

88
backend/src/plugins/search.py
View File

@@ -1,17 +1,15 @@
# #region SearchPluginModule [TYPE Module] [SEMANTICS plugin, search, datasets, regex, superset]
# @BRIEF Implements a plugin for searching text patterns across all datasets in a specific Superset environment.
# @LAYER Plugins
# @LAYER: Plugins
# @RELATION Inherits from PluginBase. Uses SupersetClient from core.
# @RELATION USES -> [TaskContext]
# @RELATION USES -> TaskContext
# [SECTION: IMPORTS]
import re
from typing import Dict, Any, Optional
from ..core.plugin_base import PluginBase
from ..core.superset_client import SupersetClient
from ..core.logger import logger, belief_scope
from ..core.task_manager.context import TaskContext
# [/SECTION]
# #region SearchPlugin [TYPE Class]
# @BRIEF Plugin for searching text patterns in Superset datasets.
@@ -21,63 +19,63 @@ class SearchPlugin(PluginBase):
"""
@property
# #region id [TYPE Function]
# @BRIEF Returns the unique identifier for the search plugin.
# @PRE Plugin instance exists.
# @POST Returns string ID.
# @RETURN str - "search-datasets"
# [DEF:id:Function]
# @PURPOSE: Returns the unique identifier for the search plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string ID.
# @RETURN: str - "search-datasets"
def id(self) -> str:
with belief_scope("id"):
return "search-datasets"
# #endregion id
# [/DEF:id:Function]
@property
# #region name [TYPE Function]
# @BRIEF Returns the human-readable name of the search plugin.
# @PRE Plugin instance exists.
# @POST Returns string name.
# @RETURN str - Plugin name.
# [DEF:name:Function]
# @PURPOSE: Returns the human-readable name of the search plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string name.
# @RETURN: str - Plugin name.
def name(self) -> str:
with belief_scope("name"):
return "Search Datasets"
# #endregion name
# [/DEF:name:Function]
@property
# #region description [TYPE Function]
# @BRIEF Returns a description of the search plugin.
# @PRE Plugin instance exists.
# @POST Returns string description.
# @RETURN str - Plugin description.
# [DEF:description:Function]
# @PURPOSE: Returns a description of the search plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string description.
# @RETURN: str - Plugin description.
def description(self) -> str:
with belief_scope("description"):
return "Search for text patterns across all datasets in a specific environment."
# #endregion description
# [/DEF:description:Function]
@property
# #region version [TYPE Function]
# @BRIEF Returns the version of the search plugin.
# @PRE Plugin instance exists.
# @POST Returns string version.
# @RETURN str - "1.0.0"
# [DEF:version:Function]
# @PURPOSE: Returns the version of the search plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string version.
# @RETURN: str - "1.0.0"
def version(self) -> str:
with belief_scope("version"):
return "1.0.0"
# #endregion version
# [/DEF:version:Function]
@property
# #region ui_route [TYPE Function]
# @BRIEF Returns the frontend route for the search plugin.
# @RETURN str - "/tools/search"
# [DEF:ui_route:Function]
# @PURPOSE: Returns the frontend route for the search plugin.
# @RETURN: str - "/tools/search"
def ui_route(self) -> str:
with belief_scope("ui_route"):
return "/tools/search"
# #endregion ui_route
# [/DEF:ui_route:Function]
# #region get_schema [TYPE Function]
# @BRIEF Returns the JSON schema for the search plugin parameters.
# @PRE Plugin instance exists.
# @POST Returns dictionary schema.
# @RETURN Dict[str, Any] - JSON schema.
# [DEF:get_schema:Function]
# @PURPOSE: Returns the JSON schema for the search plugin parameters.
# @PRE: Plugin instance exists.
# @POST: Returns dictionary schema.
# @RETURN: Dict[str, Any] - JSON schema.
def get_schema(self) -> Dict[str, Any]:
with belief_scope("get_schema"):
return {
@@ -96,10 +94,10 @@ class SearchPlugin(PluginBase):
},
"required": ["env", "query"]
}
# #endregion get_schema
# [/DEF:get_schema:Function]
# #region execute [TYPE Function]
# @BRIEF Executes the dataset search logic with TaskContext support.
# [DEF:execute:Function]
# @PURPOSE: Executes the dataset search logic with TaskContext support.
# @PARAM: params (Dict[str, Any]) - Search parameters.
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE: Params contain valid 'env' and 'query'.
@@ -175,10 +173,10 @@ class SearchPlugin(PluginBase):
except Exception as e:
log.error(f"Error during search: {e}")
raise
# #endregion execute
# [/DEF:execute:Function]
# #region _get_context [TYPE Function]
# @BRIEF Extracts a small context around the match for display.
# [DEF:_get_context:Function]
# @PURPOSE: Extracts a small context around the match for display.
# @PARAM: text (str) - The full text to extract context from.
# @PARAM: match_text (str) - The matched text pattern.
# @PARAM: context_lines (int) - Number of lines of context to include.
@@ -213,7 +211,7 @@ class SearchPlugin(PluginBase):
return "\n".join(context)
return text[:100] + "..." if len(text) > 100 else text
# #endregion _get_context
# [/DEF:_get_context:Function]
# #endregion SearchPlugin
# #endregion SearchPluginModule
# #endregion SearchPluginModule

188
backend/src/plugins/storage/plugin.py
View File

@@ -1,14 +1,13 @@
# #region StoragePlugin [TYPE Module] [SEMANTICS storage, files, filesystem, plugin]
#
# @BRIEF Provides core filesystem operations for managing backups and repositories.
# @LAYER App
# @INVARIANT All file operations must be restricted to the configured storage root.
# @RELATION IMPLEMENTS -> [PluginBase]
# @RELATION DEPENDS_ON -> [backend.src.models.storage]
# @RELATION USES -> [TaskContext]
#
# @LAYER: App
# @RELATION IMPLEMENTS -> PluginBase
# @RELATION DEPENDS_ON -> backend.src.models.storage
# @RELATION USES -> TaskContext
#
# @INVARIANT: All file operations must be restricted to the configured storage root.
# [SECTION: IMPORTS]
import os
import shutil
from pathlib import Path
@@ -17,14 +16,10 @@ from typing import Dict, Any, List, Optional
from fastapi import UploadFile
from ...core.plugin_base import PluginBase
from ...core.logger import belief_scope
from ...core.cot_logger import MarkerLogger
from ...core.logger import belief_scope, logger
from ...models.storage import StoredFile, FileCategory
log = MarkerLogger("StoragePlugin")
from ...dependencies import get_config_manager
from ...core.task_manager.context import TaskContext
# [/SECTION]
# #region StoragePlugin [TYPE Class]
# @BRIEF Implementation of the storage management plugin.
@@ -33,73 +28,73 @@ class StoragePlugin(PluginBase):
Plugin for managing local file storage for backups and repositories.
"""
# #region __init__ [TYPE Function]
# @BRIEF Initializes the StoragePlugin and ensures required directories exist.
# @PRE Configuration manager must be accessible.
# @POST Storage root and category directories are created on disk.
# [DEF:__init__:Function]
# @PURPOSE: Initializes the StoragePlugin and ensures required directories exist.
# @PRE: Configuration manager must be accessible.
# @POST: Storage root and category directories are created on disk.
def __init__(self):
with belief_scope("StoragePlugin:init"):
self.ensure_directories()
# #endregion __init__
# [/DEF:__init__:Function]
@property
# #region id [TYPE Function]
# @BRIEF Returns the unique identifier for the storage plugin.
# @PRE None.
# @POST Returns the plugin ID string.
# @RETURN str - "storage-manager"
# [DEF:id:Function]
# @PURPOSE: Returns the unique identifier for the storage plugin.
# @PRE: None.
# @POST: Returns the plugin ID string.
# @RETURN: str - "storage-manager"
def id(self) -> str:
with belief_scope("StoragePlugin:id"):
return "storage-manager"
# #endregion id
# [/DEF:id:Function]
@property
# #region name [TYPE Function]
# @BRIEF Returns the human-readable name of the storage plugin.
# @PRE None.
# @POST Returns the plugin name string.
# @RETURN str - "Storage Manager"
# [DEF:name:Function]
# @PURPOSE: Returns the human-readable name of the storage plugin.
# @PRE: None.
# @POST: Returns the plugin name string.
# @RETURN: str - "Storage Manager"
def name(self) -> str:
with belief_scope("StoragePlugin:name"):
return "Storage Manager"
# #endregion name
# [/DEF:name:Function]
@property
# #region description [TYPE Function]
# @BRIEF Returns a description of the storage plugin.
# @PRE None.
# @POST Returns the plugin description string.
# @RETURN str - Plugin description.
# [DEF:description:Function]
# @PURPOSE: Returns a description of the storage plugin.
# @PRE: None.
# @POST: Returns the plugin description string.
# @RETURN: str - Plugin description.
def description(self) -> str:
with belief_scope("StoragePlugin:description"):
return "Manages local file storage for backups and repositories."
# #endregion description
# [/DEF:description:Function]
@property
# #region version [TYPE Function]
# @BRIEF Returns the version of the storage plugin.
# @PRE None.
# @POST Returns the version string.
# @RETURN str - "1.0.0"
# [DEF:version:Function]
# @PURPOSE: Returns the version of the storage plugin.
# @PRE: None.
# @POST: Returns the version string.
# @RETURN: str - "1.0.0"
def version(self) -> str:
with belief_scope("StoragePlugin:version"):
return "1.0.0"
# #endregion version
# [/DEF:version:Function]
@property
# #region ui_route [TYPE Function]
# @BRIEF Returns the frontend route for the storage plugin.
# @RETURN str - "/tools/storage"
# [DEF:ui_route:Function]
# @PURPOSE: Returns the frontend route for the storage plugin.
# @RETURN: str - "/tools/storage"
def ui_route(self) -> str:
with belief_scope("StoragePlugin:ui_route"):
return "/tools/storage"
# #endregion ui_route
# [/DEF:ui_route:Function]
# #region get_schema [TYPE Function]
# @BRIEF Returns the JSON schema for storage plugin parameters.
# @PRE None.
# @POST Returns a dictionary representing the JSON schema.
# @RETURN Dict[str, Any] - JSON schema.
# [DEF:get_schema:Function]
# @PURPOSE: Returns the JSON schema for storage plugin parameters.
# @PRE: None.
# @POST: Returns a dictionary representing the JSON schema.
# @RETURN: Dict[str, Any] - JSON schema.
def get_schema(self) -> Dict[str, Any]:
with belief_scope("StoragePlugin:get_schema"):
return {
@@ -113,23 +108,30 @@ class StoragePlugin(PluginBase):
},
"required": ["category"]
}
# #endregion get_schema
# [/DEF:get_schema:Function]
# #region execute [TYPE Function]
# @BRIEF Executes storage-related tasks with TaskContext support.
# [DEF:execute:Function]
# @PURPOSE: Executes storage-related tasks with TaskContext support.
# @PARAM: params (Dict[str, Any]) - Storage parameters.
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE: params must match the plugin schema.
# @POST: Task is executed and logged.
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
with belief_scope("StoragePlugin:execute"):
log.reason(f"Executing with params: {params}")
# #endregion execute
# Use TaskContext logger if available, otherwise fall back to app logger
log = context.logger if context else logger
# Create sub-loggers for different components
storage_log = log.with_source("storage") if context else log
log.with_source("filesystem") if context else log
storage_log.info(f"Executing with params: {params}")
# [/DEF:execute:Function]
# #region get_storage_root [TYPE Function]
# @BRIEF Resolves the absolute path to the storage root.
# @PRE Settings must define a storage root path.
# @POST Returns a Path object representing the storage root.
# [DEF:get_storage_root:Function]
# @PURPOSE: Resolves the absolute path to the storage root.
# @PRE: Settings must define a storage root path.
# @POST: Returns a Path object representing the storage root.
def get_storage_root(self) -> Path:
with belief_scope("StoragePlugin:get_storage_root"):
config_manager = get_config_manager()
@@ -146,10 +148,10 @@ class StoragePlugin(PluginBase):
project_root = Path(__file__).parents[3]
root = (project_root / root).resolve()
return root
# #endregion get_storage_root
# [/DEF:get_storage_root:Function]
# #region resolve_path [TYPE Function]
# @BRIEF Resolves a dynamic path pattern using provided variables.
# [DEF:resolve_path:Function]
# @PURPOSE: Resolves a dynamic path pattern using provided variables.
# @PARAM: pattern (str) - The path pattern to resolve.
# @PARAM: variables (Dict[str, str]) - Variables to substitute in the pattern.
# @PRE: pattern must be a valid format string.
@@ -167,16 +169,16 @@ class StoragePlugin(PluginBase):
# Clean up any double slashes or leading/trailing slashes for relative path
return os.path.normpath(resolved).strip("/")
except KeyError as e:
log.explore("Missing variable for path resolution", error=str(e))
logger.warning(f"[StoragePlugin][Coherence:Failed] Missing variable for path resolution: {e}")
# Fallback to literal pattern if formatting fails partially (or handle as needed)
return pattern.replace("{", "").replace("}", "")
# #endregion resolve_path
# [/DEF:resolve_path:Function]
# #region ensure_directories [TYPE Function]
# @BRIEF Creates the storage root and category subdirectories if they don't exist.
# @PRE Storage root must be resolvable.
# @POST Directories are created on the filesystem.
# @SIDE_EFFECT Creates directories on the filesystem.
# [DEF:ensure_directories:Function]
# @PURPOSE: Creates the storage root and category subdirectories if they don't exist.
# @PRE: Storage root must be resolvable.
# @POST: Directories are created on the filesystem.
# @SIDE_EFFECT: Creates directories on the filesystem.
def ensure_directories(self):
with belief_scope("StoragePlugin:ensure_directories"):
root = self.get_storage_root()
@@ -184,13 +186,13 @@ class StoragePlugin(PluginBase):
# Use singular name for consistency with BackupPlugin and GitService
path = root / category.value
path.mkdir(parents=True, exist_ok=True)
log.reason(f"Ensured directory: {path}")
# #endregion ensure_directories
logger.debug(f"[StoragePlugin][Action] Ensured directory: {path}")
# [/DEF:ensure_directories:Function]
# #region validate_path [TYPE Function]
# @BRIEF Prevents path traversal attacks by ensuring the path is within the storage root.
# @PRE path must be a Path object.
# @POST Returns the resolved absolute path if valid, otherwise raises ValueError.
# [DEF:validate_path:Function]
# @PURPOSE: Prevents path traversal attacks by ensuring the path is within the storage root.
# @PRE: path must be a Path object.
# @POST: Returns the resolved absolute path if valid, otherwise raises ValueError.
def validate_path(self, path: Path) -> Path:
with belief_scope("StoragePlugin:validate_path"):
root = self.get_storage_root().resolve()
@@ -198,13 +200,13 @@ class StoragePlugin(PluginBase):
try:
resolved.relative_to(root)
except ValueError:
log.explore(f"Path traversal detected: {resolved} is not under {root}", error="Path outside storage root")
logger.error(f"[StoragePlugin][Coherence:Failed] Path traversal detected: {resolved} is not under {root}")
raise ValueError("Access denied: Path is outside of storage root.")
return resolved
# #endregion validate_path
# [/DEF:validate_path:Function]
# #region list_files [TYPE Function]
# @BRIEF Lists all files and directories in a specific category and subpath.
# [DEF:list_files:Function]
# @PURPOSE: Lists all files and directories in a specific category and subpath.
# @PARAM: category (Optional[FileCategory]) - The category to list.
# @PARAM: subpath (Optional[str]) - Nested path within the category.
# @PARAM: recursive (bool) - Whether to scan nested subdirectories recursively.
@@ -219,8 +221,8 @@ class StoragePlugin(PluginBase):
) -> List[StoredFile]:
with belief_scope("StoragePlugin:list_files"):
root = self.get_storage_root()
log.reason(
f"Listing files in root: {root}, category: {category}, subpath: {subpath}, recursive: {recursive}"
logger.info(
f"[StoragePlugin][Action] Listing files in root: {root}, category: {category}, subpath: {subpath}, recursive: {recursive}"
)
files = []
@@ -256,7 +258,7 @@ class StoragePlugin(PluginBase):
if not target_dir.exists():
continue
log.reason(f"Scanning directory: {target_dir}")
logger.debug(f"[StoragePlugin][Action] Scanning directory: {target_dir}")
if recursive:
for current_root, dirs, filenames in os.walk(target_dir):
@@ -299,10 +301,10 @@ class StoragePlugin(PluginBase):
# Sort: directories first, then by name
return sorted(files, key=lambda x: (x.mime_type != "directory", x.name))
# #endregion list_files
# [/DEF:list_files:Function]
# #region save_file [TYPE Function]
# @BRIEF Saves an uploaded file to the specified category and optional subpath.
# [DEF:save_file:Function]
# @PURPOSE: Saves an uploaded file to the specified category and optional subpath.
# @PARAM: file (UploadFile) - The uploaded file.
# @PARAM: category (FileCategory) - The target category.
# @PARAM: subpath (Optional[str]) - The target subpath.
@@ -333,10 +335,10 @@ class StoragePlugin(PluginBase):
category=category,
mime_type=file.content_type
)
# #endregion save_file
# [/DEF:save_file:Function]
# #region delete_file [TYPE Function]
# @BRIEF Deletes a file or directory from the specified category and path.
# [DEF:delete_file:Function]
# @PURPOSE: Deletes a file or directory from the specified category and path.
# @PARAM: category (FileCategory) - The category.
# @PARAM: path (str) - The relative path of the file or directory.
# @PRE: path must belong to the specified category and exist on disk.
@@ -356,13 +358,13 @@ class StoragePlugin(PluginBase):
shutil.rmtree(full_path)
else:
full_path.unlink()
log.reflect(f"Deleted: {full_path}")
logger.info(f"[StoragePlugin][Action] Deleted: {full_path}")
else:
raise FileNotFoundError(f"Item {path} not found")
# #endregion delete_file
# [/DEF:delete_file:Function]
# #region get_file_path [TYPE Function]
# @BRIEF Returns the absolute path of a file for download.
# [DEF:get_file_path:Function]
# @PURPOSE: Returns the absolute path of a file for download.
# @PARAM: category (FileCategory) - The category.
# @PARAM: path (str) - The relative path of the file.
# @PRE: path must belong to the specified category and be a file.
@@ -380,7 +382,7 @@ class StoragePlugin(PluginBase):
raise FileNotFoundError(f"File {path} not found")
return file_path
# #endregion get_file_path
# [/DEF:get_file_path:Function]
# #endregion StoragePlugin
# #endregion StoragePlugin

3
backend/src/plugins/translate/__init__.py
View File

@@ -1,2 +1,3 @@
# #region TranslatePluginPackage [C:1] [TYPE Package]
# #region TranslatePluginPackage [TYPE Package]
# @BRIEF Translation plugin package for LLM-based cross-Superset SQL/dashboard translation.
# #endregion TranslatePluginPackage

6
backend/src/plugins/translate/__tests__/__init__.py
View File

@@ -1,3 +1,3 @@
# #region TranslatePluginTestsPackage [TYPE Package]
# @BRIEF Tests for the translate plugin package.
# #endregion TranslatePluginTestsPackage
# [DEF:TranslatePluginTestsPackage:Package]
# @PURPOSE: Tests for the translate plugin package.
# [/DEF:TranslatePluginTestsPackage:Package]

33
backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py
View File

@@ -12,9 +12,7 @@
# @TEST_EDGE: unix_seconds_timestamp -> converted to YYYY-MM-DD
# @TEST_EDGE: non_timestamp_string -> passed through unchanged
import pytest
from unittest.mock import MagicMock, patch, call
from typing import Any, Dict, List
from unittest.mock import MagicMock, patch
from src.plugins.translate.sql_generator import (
SQLGenerator,
@@ -37,11 +35,15 @@ def _make_mock_job(**overrides):
job.upsert_strategy = "INSERT"
job.target_database_id = "1"
job.environment_id = "test-env"
job.context_columns = []
for k, v in overrides.items():
setattr(job, k, v)
return job
# [/DEF:_make_mock_job:Function]
# [DEF:_make_mock_record:Function]
# @PURPOSE: Create a mock TranslationRecord with source_data containing a Unix timestamp.
def _make_mock_record(
@@ -68,6 +70,9 @@ def _make_mock_record(
return rec
# [/DEF:_make_mock_record:Function]
# #region TestClickHouseTimestampNormalization [TYPE Class]
# @BRIEF Verify Unix timestamp detection and conversion for ClickHouse Date columns.
class TestClickHouseTimestampNormalization:
@@ -119,6 +124,15 @@ class TestClickHouseTimestampNormalization:
def test_postgresql_encode_not_affected(self) -> None:
result = _encode_sql_value("1726358400000.0", dialect="postgresql")
assert result == "'1726358400000.0'"
# [/DEF:test_financial_arrears_date_millis:Function]
# [/DEF:test_financial_arrears_date_seconds:Function]
# [/DEF:test_document_number_not_affected:Function]
# [/DEF:test_empty_string:Function]
# [/DEF:test_none_value:Function]
# [/DEF:test_clickhouse_encode_with_timestamp:Function]
# [/DEF:test_clickhouse_encode_with_document_number:Function]
# [/DEF:test_postgresql_encode_not_affected:Function]
# #endregion TestClickHouseTimestampNormalization
@@ -227,6 +241,10 @@ class TestClickHouseInsertSqlGeneration:
assert "'2024-09-15'" in sql
assert "1726358400" not in sql or "'2024-09-15'" in sql
# [/DEF:test_insert_with_timestamp_key_cols:Function]
# [/DEF:test_insert_multiple_rows_mixed_timestamps:Function]
# [/DEF:test_insert_with_seconds_timestamp:Function]
# #endregion TestClickHouseInsertSqlGeneration
@@ -295,6 +313,9 @@ class TestClickHouseJoinVerification:
assert "`report_date`" in insert_sql
assert "`document_number`" in insert_sql
assert "`comment_text_en`" in insert_sql
# [/DEF:test_join_query_structure:Function]
# [/DEF:test_insert_then_join_consistency:Function]
# #endregion TestClickHouseJoinVerification
@@ -383,6 +404,8 @@ class TestOrchestratorInsertFlow:
# Verify result
assert result["status"] == "success"
# [/DEF:test_generate_and_insert_sql_with_timestamps:Function]
# #endregion TestOrchestratorInsertFlow
@@ -509,4 +532,8 @@ LIMIT 10
assert "f_a.document_number = f_c.document_number" in join_sql
assert "dm_view.financial_arrears" in join_sql
assert "dm_view.financial_comments_translated" in join_sql
# [/DEF:test_full_insert_sql_valid_for_clickhouse:Function]
# [/DEF:test_verification_join_query:Function]
# #endregion TestClickHouseEndToEnd
# [/DEF:ClickHouseInsertIntegration:Module]

173
backend/src/plugins/translate/__tests__/test_dictionary.py
View File

@@ -1,6 +1,8 @@
# #region DictionaryTests [C:3] [TYPE Module] [SEMANTICS tests, dictionary, crud, import, filter]
# @BRIEF Validate DictionaryManager CRUD, import, deletion guards, and batch filtering.
# @RELATION BINDS_TO -> [DictionaryManager:Class]
# [DEF:DictionaryTests:Module]
# @COMPLEXITY: 3
# @SEMANTICS: tests, dictionary, crud, import, filter
# @PURPOSE: Validate DictionaryManager CRUD, import, deletion guards, and batch filtering.
# @RELATION: BINDS_TO -> [DictionaryManager:Class]
#
# @TEST_CONTRACT: [DictionaryManager] -> {
# invariants: [
@@ -35,15 +37,16 @@ from src.plugins.translate.dictionary import DictionaryManager
from src.plugins.translate._utils import _normalize_term, _detect_delimiter
# #region _FakeJob [C:1] [TYPE Class]
# @BRIEF Helper to create inline TranslationJob records.
# [DEF:_FakeJob:Class]
# @COMPLEXITY: 1
# @PURPOSE: Helper to create inline TranslationJob records.
class _FakeJob:
pass
# #endregion _FakeJob
# [/DEF:_FakeJob:Class]
# #region db_session [TYPE Fixture]
# @BRIEF Provide an in-memory SQLite session for each test, with tables created and torn down.
# [DEF:db_session:Fixture]
# @PURPOSE: Provide an in-memory SQLite session for each test, with tables created and torn down.
@pytest.fixture
def db_session():
engine = create_engine("sqlite:///:memory:", echo=False)
@@ -62,11 +65,11 @@ def db_session():
finally:
session.close()
Base.metadata.drop_all(bind=engine)
# #endregion db_session
# [/DEF:db_session:Fixture]
# #region test_create_dictionary [TYPE Function]
# @BRIEF Verify dictionary creation and read-back.
# [DEF:test_create_dictionary:Function]
# @PURPOSE: Verify dictionary creation and read-back.
def test_create_dictionary(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session,
@@ -87,11 +90,11 @@ def test_create_dictionary(db_session: Session):
fetched = DictionaryManager.get_dictionary(db_session, d.id)
assert fetched.id == d.id
assert fetched.name == "Finance Terms"
# #endregion test_create_dictionary
# [/DEF:test_create_dictionary:Function]
# #region test_update_dictionary [TYPE Function]
# @BRIEF Verify dictionary metadata update.
# [DEF:test_update_dictionary:Function]
# @PURPOSE: Verify dictionary metadata update.
def test_update_dictionary(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Old Name",
@@ -106,11 +109,11 @@ def test_update_dictionary(db_session: Session):
assert updated.name == "New Name"
assert updated.description == "Updated desc"
assert updated.is_active is False
# #endregion test_update_dictionary
# [/DEF:test_update_dictionary:Function]
# #region test_delete_dictionary [TYPE Function]
# @BRIEF Verify dictionary deletion.
# [DEF:test_delete_dictionary:Function]
# @PURPOSE: Verify dictionary deletion.
def test_delete_dictionary(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="To Delete",
@@ -123,11 +126,11 @@ def test_delete_dictionary(db_session: Session):
DictionaryManager.delete_dictionary(db_session, d.id)
with pytest.raises(ValueError, match="Dictionary not found"):
DictionaryManager.get_dictionary(db_session, d.id)
# #endregion test_delete_dictionary
# [/DEF:test_delete_dictionary:Function]
# #region test_list_dictionaries [TYPE Function]
# @BRIEF Verify paginated dictionary listing.
# [DEF:test_list_dictionaries:Function]
# @PURPOSE: Verify paginated dictionary listing.
def test_list_dictionaries(db_session: Session):
for i in range(5):
DictionaryManager.create_dictionary(
@@ -138,11 +141,11 @@ def test_list_dictionaries(db_session: Session):
dicts, total = DictionaryManager.list_dictionaries(db_session, page=1, page_size=2)
assert total == 5
assert len(dicts) == 2
# #endregion test_list_dictionaries
# [/DEF:test_list_dictionaries:Function]
# #region test_add_entry_duplicate [TYPE Function]
# @BRIEF Verify duplicate entry raises ValueError and unique constraint is enforced.
# [DEF:test_add_entry_duplicate:Function]
# @PURPOSE: Verify duplicate entry raises ValueError and unique constraint is enforced.
def test_add_entry_duplicate(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test",
@@ -157,11 +160,11 @@ def test_add_entry_duplicate(db_session: Session):
# Different case, same normalized should also raise
with pytest.raises(ValueError, match="already exists"):
DictionaryManager.add_entry(db_session, d.id, "HELLO", "Ciao")
# #endregion test_add_entry_duplicate
# [/DEF:test_add_entry_duplicate:Function]
# #region test_add_entry_duplicate_per_dictionary [TYPE Function]
# @BRIEF Verify duplicate is per-dictionary (same term in different dictionaries is OK).
# [DEF:test_add_entry_duplicate_per_dictionary:Function]
# @PURPOSE: Verify duplicate is per-dictionary (same term in different dictionaries is OK).
def test_add_entry_duplicate_per_dictionary(db_session: Session):
d1 = DictionaryManager.create_dictionary(
db_session, name="Dict1",
@@ -175,11 +178,11 @@ def test_add_entry_duplicate_per_dictionary(db_session: Session):
# Same term in different dictionary should work
entry = DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour")
assert entry.id is not None
# #endregion test_add_entry_duplicate_per_dictionary
# [/DEF:test_add_entry_duplicate_per_dictionary:Function]
# #region test_edit_entry [TYPE Function]
# @BRIEF Verify entry edit updates fields and enforces uniqueness.
# [DEF:test_edit_entry:Function]
# @PURPOSE: Verify entry edit updates fields and enforces uniqueness.
def test_edit_entry(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test",
@@ -194,11 +197,11 @@ def test_edit_entry(db_session: Session):
DictionaryManager.add_entry(db_session, d.id, "world", "mundo")
with pytest.raises(ValueError, match="already exists"):
DictionaryManager.edit_entry(db_session, entry.id, source_term="WORLD")
# #endregion test_edit_entry
# [/DEF:test_edit_entry:Function]
# #region test_delete_entry [TYPE Function]
# @BRIEF Verify entry deletion.
# [DEF:test_delete_entry:Function]
# @PURPOSE: Verify entry deletion.
def test_delete_entry(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test",
@@ -209,11 +212,11 @@ def test_delete_entry(db_session: Session):
entries, total = DictionaryManager.list_entries(db_session, d.id)
assert total == 0
# #endregion test_delete_entry
# [/DEF:test_delete_entry:Function]
# #region test_import_csv_overwrite [TYPE Function]
# @BRIEF Verify CSV import with overwrite conflict mode.
# [DEF:test_import_csv_overwrite:Function]
# @PURPOSE: Verify CSV import with overwrite conflict mode.
def test_import_csv_overwrite(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test",
@@ -236,11 +239,11 @@ def test_import_csv_overwrite(db_session: Session):
entries, _ = DictionaryManager.list_entries(db_session, d.id)
hello_entry = [e for e in entries if e.source_term_normalized == "hello"][0]
assert hello_entry.target_term == "HELLO"
# #endregion test_import_csv_overwrite
# [/DEF:test_import_csv_overwrite:Function]
# #region test_import_csv_keep_existing [TYPE Function]
# @BRIEF Verify CSV import with keep_existing conflict mode.
# [DEF:test_import_csv_keep_existing:Function]
# @PURPOSE: Verify CSV import with keep_existing conflict mode.
def test_import_csv_keep_existing(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test",
@@ -261,11 +264,11 @@ def test_import_csv_keep_existing(db_session: Session):
entries, _ = DictionaryManager.list_entries(db_session, d.id)
hello_entry = [e for e in entries if e.source_term_normalized == "hello"][0]
assert hello_entry.target_term == "hola"
# #endregion test_import_csv_keep_existing
# [/DEF:test_import_csv_keep_existing:Function]
# #region test_import_csv_cancel_on_conflict [TYPE Function]
# @BRIEF Verify CSV import with cancel conflict mode raises errors.
# [DEF:test_import_csv_cancel_on_conflict:Function]
# @PURPOSE: Verify CSV import with cancel conflict mode raises errors.
def test_import_csv_cancel_on_conflict(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test",
@@ -283,11 +286,11 @@ def test_import_csv_cancel_on_conflict(db_session: Session):
assert result["updated"] == 0
assert len(result["errors"]) == 1 # hello conflict error
assert "Conflict" in result["errors"][0]["error"]
# #endregion test_import_csv_cancel_on_conflict
# [/DEF:test_import_csv_cancel_on_conflict:Function]
# #region test_import_tsv [TYPE Function]
# @BRIEF Verify TSV import.
# [DEF:test_import_tsv:Function]
# @PURPOSE: Verify TSV import.
def test_import_tsv(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test",
@@ -300,11 +303,11 @@ def test_import_tsv(db_session: Session):
)
assert result["created"] == 2
assert result["total"] == 2
# #endregion test_import_tsv
# [/DEF:test_import_tsv:Function]
# #region test_import_invalid_format [TYPE Function]
# @BRIEF Verify import raises ValueError for missing required columns.
# [DEF:test_import_invalid_format:Function]
# @PURPOSE: Verify import raises ValueError for missing required columns.
def test_import_invalid_format(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test",
@@ -316,11 +319,11 @@ def test_import_invalid_format(db_session: Session):
db_session, d.id, bad_content,
delimiter=",", on_conflict="overwrite",
)
# #endregion test_import_invalid_format
# [/DEF:test_import_invalid_format:Function]
# #region test_import_empty_rows [TYPE Function]
# @BRIEF Verify import handles rows with missing fields gracefully.
# [DEF:test_import_empty_rows:Function]
# @PURPOSE: Verify import handles rows with missing fields gracefully.
def test_import_empty_rows(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test",
@@ -333,11 +336,11 @@ def test_import_empty_rows(db_session: Session):
)
assert result["created"] == 1 # hello
assert len(result["errors"]) == 2 # empty source or target
# #endregion test_import_empty_rows
# [/DEF:test_import_empty_rows:Function]
# #region test_import_preview [TYPE Function]
# @BRIEF Verify import preview returns conflicts without mutating.
# [DEF:test_import_preview:Function]
# @PURPOSE: Verify import preview returns conflicts without mutating.
def test_import_preview(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test",
@@ -361,11 +364,11 @@ def test_import_preview(db_session: Session):
# Verify no mutation happened
entries, total = DictionaryManager.list_entries(db_session, d.id)
assert total == 1 # still only the original entry
# #endregion test_import_preview
# [/DEF:test_import_preview:Function]
# #region test_delete_dictionary_blocked_by_active_job [TYPE Function]
# @BRIEF Verify deletion is blocked when dictionary is attached to active/scheduled jobs.
# [DEF:test_delete_dictionary_blocked_by_active_job:Function]
# @PURPOSE: Verify deletion is blocked when dictionary is attached to active/scheduled jobs.
def test_delete_dictionary_blocked_by_active_job(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test",
@@ -392,11 +395,11 @@ def test_delete_dictionary_blocked_by_active_job(db_session: Session):
with pytest.raises(ValueError, match="active/scheduled"):
DictionaryManager.delete_dictionary(db_session, d.id)
# #endregion test_delete_dictionary_blocked_by_active_job
# [/DEF:test_delete_dictionary_blocked_by_active_job:Function]
# #region test_delete_dictionary_allowed_with_completed_job [TYPE Function]
# @BRIEF Verify deletion is allowed when only completed/failed jobs reference the dictionary.
# [DEF:test_delete_dictionary_allowed_with_completed_job:Function]
# @PURPOSE: Verify deletion is allowed when only completed/failed jobs reference the dictionary.
def test_delete_dictionary_allowed_with_completed_job(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test",
@@ -424,11 +427,11 @@ def test_delete_dictionary_allowed_with_completed_job(db_session: Session):
DictionaryManager.delete_dictionary(db_session, d.id)
with pytest.raises(ValueError, match="Dictionary not found"):
DictionaryManager.get_dictionary(db_session, d.id)
# #endregion test_delete_dictionary_allowed_with_completed_job
# [/DEF:test_delete_dictionary_allowed_with_completed_job:Function]
# #region test_filter_for_batch_no_dictionaries [TYPE Function]
# @BRIEF Verify filter_for_batch returns empty when job has no dictionaries.
# [DEF:test_filter_for_batch_no_dictionaries:Function]
# @PURPOSE: Verify filter_for_batch returns empty when job has no dictionaries.
def test_filter_for_batch_no_dictionaries(db_session: Session):
job = TranslationJob(
name="No Dict Job",
@@ -440,11 +443,11 @@ def test_filter_for_batch_no_dictionaries(db_session: Session):
result = DictionaryManager.filter_for_batch(db_session, ["hello world"], job.id)
assert result == []
# #endregion test_filter_for_batch_no_dictionaries
# [/DEF:test_filter_for_batch_no_dictionaries:Function]
# #region test_filter_for_batch_matches [TYPE Function]
# @BRIEF Verify filter_for_batch returns correct matched entries with word-boundary awareness.
# [DEF:test_filter_for_batch_matches:Function]
# @PURPOSE: Verify filter_for_batch returns correct matched entries with word-boundary awareness.
def test_filter_for_batch_matches(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test Dict",
@@ -486,11 +489,11 @@ def test_filter_for_batch_matches(db_session: Session):
foo_matches = [m for m in result if m["source_term"] == "foo"]
assert len(foo_matches) == 1
assert foo_matches[0]["text_index"] == 2
# #endregion test_filter_for_batch_matches
# [/DEF:test_filter_for_batch_matches:Function]
# #region test_filter_for_batch_case_insensitive [TYPE Function]
# @BRIEF Verify filter_for_batch matching is case-insensitive.
# [DEF:test_filter_for_batch_case_insensitive:Function]
# @PURPOSE: Verify filter_for_batch matching is case-insensitive.
def test_filter_for_batch_case_insensitive(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test Dict",
@@ -517,11 +520,11 @@ def test_filter_for_batch_case_insensitive(db_session: Session):
assert len(result) == 1
assert result[0]["source_term"] == "Hello World"
assert result[0]["target_term"] == "Hola Mundo"
# #endregion test_filter_for_batch_case_insensitive
# [/DEF:test_filter_for_batch_case_insensitive:Function]
# #region test_filter_for_batch_word_boundary [TYPE Function]
# @BRIEF Verify filter_for_batch respects word boundaries (no substring matching within words).
# [DEF:test_filter_for_batch_word_boundary:Function]
# @PURPOSE: Verify filter_for_batch respects word boundaries (no substring matching within words).
def test_filter_for_batch_word_boundary(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test Dict",
@@ -554,11 +557,11 @@ def test_filter_for_batch_word_boundary(db_session: Session):
job.id,
)
assert len(result) == 1
# #endregion test_filter_for_batch_word_boundary
# [/DEF:test_filter_for_batch_word_boundary:Function]
# #region test_filter_for_batch_multi_dictionary_priority [TYPE Function]
# @BRIEF Verify filter_for_batch respects dictionary link order priority.
# [DEF:test_filter_for_batch_multi_dictionary_priority:Function]
# @PURPOSE: Verify filter_for_batch respects dictionary link order priority.
def test_filter_for_batch_multi_dictionary_priority(db_session: Session):
d1 = DictionaryManager.create_dictionary(
db_session, name="Priority1", source_dialect="a", target_dialect="b",
@@ -593,11 +596,11 @@ def test_filter_for_batch_multi_dictionary_priority(db_session: Session):
assert result[0]["target_term"] == "hola"
assert result[1]["dictionary_id"] == d2.id
assert result[1]["target_term"] == "bonjour"
# #endregion test_filter_for_batch_multi_dictionary_priority
# [/DEF:test_filter_for_batch_multi_dictionary_priority:Function]
# #region test_normalize_term [TYPE Function]
# @BRIEF Verify _normalize_term produces lowercase NFC-normalized strings.
# [DEF:test_normalize_term:Function]
# @PURPOSE: Verify _normalize_term produces lowercase NFC-normalized strings.
def test_normalize_term():
assert _normalize_term("Hello") == "hello"
assert _normalize_term("HELLO") == "hello"
@@ -606,11 +609,11 @@ def test_normalize_term():
composed = "\u00C9" # É precomposed
decomposed = "\u0045\u0301" # E + combining acute
assert _normalize_term(composed) == _normalize_term(decomposed)
# #endregion test_normalize_term
# [/DEF:test_normalize_term:Function]
# #region test_detect_delimiter [TYPE Function]
# @BRIEF Verify _detect_delimiter correctly identifies CSV vs TSV.
# [DEF:test_detect_delimiter:Function]
# @PURPOSE: Verify _detect_delimiter correctly identifies CSV vs TSV.
def test_detect_delimiter():
csv_content = "source_term,target_term,context_notes\nhello,hola,"
assert _detect_delimiter(csv_content) == ","
@@ -620,11 +623,11 @@ def test_detect_delimiter():
empty_content = ""
assert _detect_delimiter(empty_content) == "," # default fallback
# #endregion test_detect_delimiter
# [/DEF:test_detect_delimiter:Function]
# #region test_clear_entries [TYPE Function]
# @BRIEF Verify clearing all entries for a dictionary.
# [DEF:test_clear_entries:Function]
# @PURPOSE: Verify clearing all entries for a dictionary.
def test_clear_entries(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test", source_dialect="a", target_dialect="b",
@@ -637,5 +640,5 @@ def test_clear_entries(db_session: Session):
entries, total = DictionaryManager.list_entries(db_session, d.id)
assert total == 0
# #endregion test_clear_entries
# #endregion DictionaryTests
# [/DEF:test_clear_entries:Function]
# [/DEF:DictionaryTests:Module]

58
backend/src/plugins/translate/__tests__/test_orchestrator.py
View File

@@ -1,8 +1,10 @@
# #region OrchestratorTests [C:3] [TYPE Module] [SEMANTICS test, translate, orchestrator, events]
# @BRIEF Tests for TranslationOrchestrator: run lifecycle, partial failure, batch retry, event invariants, NULL handling.
# @LAYER Test
# @RELATION BINDS_TO -> [TranslationOrchestrator:Module]
# @RELATION BINDS_TO -> [TranslationEventLog:Module]
# [DEF:OrchestratorTests:Module]
# @COMPLEXITY: 3
# @SEMANTICS: test, translate, orchestrator, events
# @PURPOSE: Tests for TranslationOrchestrator: run lifecycle, partial failure, batch retry, event invariants, NULL handling.
# @LAYER: Test
# @RELATION: BINDS_TO -> [TranslationOrchestrator:Module]
# @RELATION: BINDS_TO -> [TranslationEventLog:Module]
# @TEST_CONTRACT: TranslationOrchestrator -> start_run, execute_run, cancel_run, retry_failed_batches
# @TEST_FIXTURE: mock_db -> MagicMock SQLAlchemy session
# @TEST_FIXTURE: mock_config_manager -> MagicMock ConfigManager
@@ -10,16 +12,14 @@
# @TEST_EDGE: invalid_run_status -> raises ValueError
import pytest
from unittest.mock import MagicMock, patch, PropertyMock
from unittest.mock import MagicMock
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from src.plugins.translate.orchestrator import TranslationOrchestrator
from src.plugins.translate.events import TranslationEventLog
from src.models.translate import (
TranslationJob,
TranslationRun,
TranslationBatch,
TranslationRecord,
TranslationPreviewSession,
)
@@ -39,11 +39,11 @@ def mock_job() -> MagicMock:
job.source_datasource_id = "42"
job.translation_column = "name"
job.context_columns = ["category"]
job.source_table = "source_table_name"
job.target_schema = "public"
job.target_table = "translated_data"
job.target_key_cols = ["id"]
job.source_key_cols = ["id"]
job.source_table = "source_data"
job.target_language = "en"
job.provider_id = "provider-1"
job.batch_size = 50
@@ -51,6 +51,9 @@ def mock_job() -> MagicMock:
return job
# [/DEF:mock_job:Function]
# [DEF:mock_preview_session:Function]
# @PURPOSE: Create a mock accepted preview session.
@pytest.fixture
@@ -63,8 +66,11 @@ def mock_preview_session() -> MagicMock:
return session
# #region TestTranslationEventLog [TYPE Class]
# @BRIEF Tests for TranslationEventLog terminal event invariant.
# [/DEF:mock_preview_session:Function]
# [DEF:TestTranslationEventLog:Class]
# @PURPOSE: Tests for TranslationEventLog terminal event invariant.
class TestTranslationEventLog:
# [DEF:test_log_event_creates_record:Function]
@@ -140,11 +146,17 @@ class TestTranslationEventLog:
log = TranslationEventLog(db)
result = log.prune_expired(retention_days=90)
assert result["pruned"] >= 0
# #endregion TestTranslationEventLog
# [/DEF:test_log_event_creates_record:Function]
# [/DEF:test_invalid_event_type_raises:Function]
# [/DEF:test_terminal_event_invariant:Function]
# [/DEF:test_run_started_must_precede_other_events:Function]
# [/DEF:test_prune_expired_creates_snapshot:Function]
# [/DEF:TestTranslationEventLog:Class]
# #region TestTranslationOrchestrator [TYPE Class]
# @BRIEF Tests for TranslationOrchestrator run lifecycle.
# [DEF:TestTranslationOrchestrator:Class]
# @PURPOSE: Tests for TranslationOrchestrator run lifecycle.
class TestTranslationOrchestrator:
# [DEF:test_start_run_success:Function]
@@ -233,7 +245,8 @@ class TestTranslationOrchestrator:
run.job_id = "job-123"
run.status = "RUNNING"
db.query.return_value.filter.return_value.first.return_value = run
# Chain: first query returns run, event-log queries return None (no existing terminal events)
db.query.return_value.filter.return_value.first.side_effect = [run, None]
orch = TranslationOrchestrator(db, config_manager, "test-user")
result = orch.cancel_run("run-1")
@@ -370,5 +383,16 @@ class TestTranslationOrchestrator:
assert total == 1
assert len(runs) == 1
assert runs[0]["id"] == "run-1"
# #endregion TestTranslationOrchestrator
# #endregion OrchestratorTests
# [/DEF:test_start_run_success:Function]
# [/DEF:test_start_run_missing_preview_raises:Function]
# [/DEF:test_start_run_draft_job_raises:Function]
# [/DEF:test_execute_run_invalid_status:Function]
# [/DEF:test_cancel_run:Function]
# [/DEF:test_cancel_run_invalid_status:Function]
# [/DEF:test_retry_failed_batches_no_failures:Function]
# [/DEF:test_get_run_status:Function]
# [/DEF:test_get_run_records:Function]
# [/DEF:test_get_run_history:Function]
# [/DEF:TestTranslationOrchestrator:Class]
# [/DEF:OrchestratorTests:Module]

114
backend/src/plugins/translate/__tests__/test_preview.py
View File

@@ -1,7 +1,9 @@
# #region TranslationPreviewTests [C:3] [TYPE Module] [SEMANTICS test, translate, preview, session]
# @BRIEF Test TranslationPreview service: preview, approve/edit/reject state transitions, cost estimation, acceptance gate.
# @LAYER Test
# @RELATION BINDS_TO -> [TranslationPreview:Module]
# [DEF:TranslationPreviewTests:Module]
# @COMPLEXITY: 3
# @SEMANTICS: test, translate, preview, session
# @PURPOSE: Test TranslationPreview service: preview, approve/edit/reject state transitions, cost estimation, acceptance gate.
# @LAYER: Test
# @RELATION: BINDS_TO -> [TranslationPreview:Module]
from unittest.mock import MagicMock, patch, PropertyMock
from datetime import datetime, timezone, timedelta
@@ -17,8 +19,8 @@ from src.models.translate import (
from src.plugins.translate.preview import TranslationPreview, TokenEstimator
# #region _make_mock_job [TYPE Function]
# @BRIEF Create a mock TranslationJob with test config.
# [DEF:_make_mock_job:Function]
# @PURPOSE: Create a mock TranslationJob with test config.
def _make_mock_job(**overrides):
job = MagicMock(spec=TranslationJob)
job.id = overrides.get("id", "job-123")
@@ -35,11 +37,11 @@ def _make_mock_job(**overrides):
job.upsert_strategy = overrides.get("upsert_strategy", "MERGE")
job.status = overrides.get("status", "DRAFT")
return job
# #endregion _make_mock_job
# [/DEF:_make_mock_job:Function]
# #region _make_mock_provider [TYPE Function]
# @BRIEF Create a mock LLMProvider.
# [DEF:_make_mock_provider:Function]
# @PURPOSE: Create a mock LLMProvider.
def _make_mock_provider(**overrides):
provider = MagicMock()
provider.id = overrides.get("id", "provider-1")
@@ -48,15 +50,15 @@ def _make_mock_provider(**overrides):
provider.default_model = overrides.get("default_model", "gpt-4o-mini")
provider.api_key = "encrypted-key-123"
return provider
# #endregion _make_mock_provider
# [/DEF:_make_mock_provider:Function]
# #region TestTranslationPreview [TYPE Class]
# @BRIEF Test suite for TranslationPreview service.
# [DEF:TestTranslationPreview:Class]
# @PURPOSE: Test suite for TranslationPreview service.
class TestTranslationPreview:
# #region test_preview_valid_job [TYPE Function]
# @BRIEF Verify preview creates session and records successfully.
# [DEF:test_preview_valid_job:Function]
# @PURPOSE: Verify preview creates session and records successfully.
def test_preview_valid_job(self):
"""Preview with valid job should create session and return records."""
db = MagicMock()
@@ -159,10 +161,10 @@ class TestTranslationPreview:
# Verify session was added to DB
assert db.add.called
assert db.commit.called
# #endregion test_preview_valid_job
# [/DEF:test_preview_valid_job:Function]
# #region test_preview_with_dictionary [TYPE Function]
# @BRIEF Verify glossary entries from dictionary are included in LLM prompt.
# [DEF:test_preview_with_dictionary:Function]
# @PURPOSE: Verify glossary entries from dictionary are included in LLM prompt.
def test_preview_with_dictionary(self):
"""Preview should include dictionary glossary in the prompt sent to LLM."""
db = MagicMock()
@@ -218,10 +220,10 @@ class TestTranslationPreview:
# Verify LLM was called
mock_llm.assert_called_once()
assert len(result["records"]) == 1
# #endregion test_preview_with_dictionary
# [/DEF:test_preview_with_dictionary:Function]
# #region test_preview_row_approve [TYPE Function]
# @BRIEF Verify approving a preview row changes its status.
# [DEF:test_preview_row_approve:Function]
# @PURPOSE: Verify approving a preview row changes its status.
def test_preview_row_approve(self):
"""Approve action should set record status to APPROVED."""
db = MagicMock()
@@ -257,10 +259,10 @@ class TestTranslationPreview:
assert record.status == "APPROVED"
assert result["status"] == "APPROVED"
assert db.commit.called
# #endregion test_preview_row_approve
# [/DEF:test_preview_row_approve:Function]
# #region test_preview_row_reject [TYPE Function]
# @BRIEF Verify rejecting a preview row changes its status.
# [DEF:test_preview_row_reject:Function]
# @PURPOSE: Verify rejecting a preview row changes its status.
def test_preview_row_reject(self):
"""Reject action should set record status to REJECTED."""
db = MagicMock()
@@ -290,10 +292,10 @@ class TestTranslationPreview:
assert record.status == "REJECTED"
assert result["status"] == "REJECTED"
# #endregion test_preview_row_reject
# [/DEF:test_preview_row_reject:Function]
# #region test_preview_row_edit [TYPE Function]
# @BRIEF Verify editing a preview row updates its translation and status.
# [DEF:test_preview_row_edit:Function]
# @PURPOSE: Verify editing a preview row updates its translation and status.
def test_preview_row_edit(self):
"""Edit action should update translation and set status to APPROVED."""
db = MagicMock()
@@ -326,10 +328,10 @@ class TestTranslationPreview:
assert record.status == "APPROVED"
assert result["status"] == "APPROVED"
assert result["target_sql"] == "edited translation"
# #endregion test_preview_row_edit
# [/DEF:test_preview_row_edit:Function]
# #region test_preview_row_invalid_action [TYPE Function]
# @BRIEF Verify invalid action raises ValueError.
# [DEF:test_preview_row_invalid_action:Function]
# @PURPOSE: Verify invalid action raises ValueError.
def test_preview_row_invalid_action(self):
"""Invalid action should raise ValueError."""
db = MagicMock()
@@ -355,10 +357,10 @@ class TestTranslationPreview:
row_id="record-1",
action="invalid_action",
)
# #endregion test_preview_row_invalid_action
# [/DEF:test_preview_row_invalid_action:Function]
# #region test_preview_accept_session [TYPE Function]
# @BRIEF Verify accepting a preview session gates execution.
# [DEF:test_preview_accept_session:Function]
# @PURPOSE: Verify accepting a preview session gates execution.
def test_preview_accept_session(self):
"""Accept should set session status to APPLIED and return records."""
db = MagicMock()
@@ -393,10 +395,10 @@ class TestTranslationPreview:
assert len(result["records"]) == 1
assert result["records"][0]["status"] == "APPROVED"
assert db.commit.called
# #endregion test_preview_accept_session
# [/DEF:test_preview_accept_session:Function]
# #region test_preview_no_active_session [TYPE Function]
# @BRIEF Verify accept raises error when no active session.
# [DEF:test_preview_no_active_session:Function]
# @PURPOSE: Verify accept raises error when no active session.
def test_preview_no_active_session(self):
"""Accept without active session should raise ValueError."""
db = MagicMock()
@@ -407,10 +409,10 @@ class TestTranslationPreview:
preview_service = TranslationPreview(db, config_manager, "test-user")
with pytest.raises(ValueError, match="No active preview session"):
preview_service.accept_preview_session(job_id="job-123")
# #endregion test_preview_no_active_session
# [/DEF:test_preview_no_active_session:Function]
# #region test_cost_estimation [TYPE Function]
# @BRIEF Verify token and cost estimation methods.
# [DEF:test_cost_estimation:Function]
# @PURPOSE: Verify token and cost estimation methods.
def test_cost_estimation(self):
"""Token and cost estimation should return reasonable values."""
prompt = "Translate this text from English to Russian. " * 100
@@ -426,10 +428,10 @@ class TestTranslationPreview:
cost_default = TokenEstimator.estimate_cost(1000)
assert cost_default == 0.002
# #endregion test_cost_estimation
# [/DEF:test_cost_estimation:Function]
# #region test_preview_parse_llm_response [TYPE Function]
# @BRIEF Verify LLM JSON response parsing.
# [DEF:test_preview_parse_llm_response:Function]
# @PURPOSE: Verify LLM JSON response parsing.
def test_preview_parse_llm_response(self):
"""Parse LLM response should extract translations keyed by row_id."""
response = json.dumps({
@@ -440,19 +442,19 @@ class TestTranslationPreview:
})
result = TranslationPreview._parse_llm_response(response, 2)
assert result == {"0": "Привет", "1": "Мир"}
# #endregion test_preview_parse_llm_response
# [/DEF:test_preview_parse_llm_response:Function]
# #region test_preview_parse_llm_response_with_code_block [TYPE Function]
# @BRIEF Verify LLM response parsing handles markdown code blocks.
# [DEF:test_preview_parse_llm_response_with_code_block:Function]
# @PURPOSE: Verify LLM response parsing handles markdown code blocks.
def test_preview_parse_llm_response_with_code_block(self):
"""Parse LLM response should handle markdown code block wrapping."""
response = "```json\n{\n \"rows\": [\n {\"row_id\": \"0\", \"translation\": \"Test\"}\n ]\n}\n```"
result = TranslationPreview._parse_llm_response(response, 1)
assert result == {"0": "Test"}
# #endregion test_preview_parse_llm_response_with_code_block
# [/DEF:test_preview_parse_llm_response_with_code_block:Function]
# #region test_preview_compute_config_hash [TYPE Function]
# @BRIEF Verify config hash computation is deterministic.
# [DEF:test_preview_compute_config_hash:Function]
# @PURPOSE: Verify config hash computation is deterministic.
def test_preview_compute_config_hash(self):
"""Config hash should be deterministic and non-empty."""
job = _make_mock_job()
@@ -460,10 +462,10 @@ class TestTranslationPreview:
hash2 = TranslationPreview._compute_config_hash(job)
assert hash1 == hash2
assert len(hash1) == 16
# #endregion test_preview_compute_config_hash
# [/DEF:test_preview_compute_config_hash:Function]
# #region test_preview_missing_datasource [TYPE Function]
# @BRIEF Verify error when job has no datasource configured.
# [DEF:test_preview_missing_datasource:Function]
# @PURPOSE: Verify error when job has no datasource configured.
def test_preview_missing_datasource(self):
"""Preview should fail if job has no datasource."""
db = MagicMock()
@@ -475,10 +477,10 @@ class TestTranslationPreview:
preview_service = TranslationPreview(db, config_manager, "test-user")
with pytest.raises(ValueError, match="source datasource"):
preview_service.preview_rows(job_id="job-123")
# #endregion test_preview_missing_datasource
# [/DEF:test_preview_missing_datasource:Function]
# #region test_preview_missing_translation_column [TYPE Function]
# @BRIEF Verify error when job has no translation column.
# [DEF:test_preview_missing_translation_column:Function]
# @PURPOSE: Verify error when job has no translation column.
def test_preview_missing_translation_column(self):
"""Preview should fail if job has no translation column."""
db = MagicMock()
@@ -490,9 +492,9 @@ class TestTranslationPreview:
preview_service = TranslationPreview(db, config_manager, "test-user")
with pytest.raises(ValueError, match="translation column"):
preview_service.preview_rows(job_id="job-123")
# #endregion test_preview_missing_translation_column
# [/DEF:test_preview_missing_translation_column:Function]
# #endregion TestTranslationPreview
# [/DEF:TestTranslationPreview:Class]
# #endregion TranslationPreviewTests
# [/DEF:TranslationPreviewTests:Module]

140
backend/src/plugins/translate/__tests__/test_sql_generator.py
View File

@@ -1,7 +1,9 @@
# #region SQLGeneratorTests [C:3] [TYPE Module] [SEMANTICS test, translate, sql_generator]
# @BRIEF Tests for SQLGenerator: PostgreSQL INSERT + UPSERT, ClickHouse INSERT, dialect quoting, NULL handling, injection safety.
# @LAYER Test
# @RELATION BINDS_TO -> [SQLGenerator:Module]
# [DEF:SQLGeneratorTests:Module]
# @COMPLEXITY: 3
# @SEMANTICS: test, translate, sql_generator
# @PURPOSE: Tests for SQLGenerator: PostgreSQL INSERT + UPSERT, ClickHouse INSERT, dialect quoting, NULL handling, injection safety.
# @LAYER: Test
# @RELATION: BINDS_TO -> [SQLGenerator:Module]
# @TEST_CONTRACT: SQLGenerator.generate -> SQL string, row_count
# @TEST_FIXTURE: sample_rows -> [{"col1": "val1", "col2": 42}, {"col1": "val2", "col2": 99}]
# @TEST_EDGE: empty_rows -> raises ValueError
@@ -9,14 +11,12 @@
# @TEST_EDGE: sql_injection -> values with single quotes are escaped
import pytest
from datetime import datetime, timezone
from typing import Any, Dict, List
from src.plugins.translate.sql_generator import (
SQLGenerator,
_quote_identifier,
_encode_sql_value,
_normalize_timestamp_value,
generate_insert_sql,
generate_upsert_sql,
)
@@ -28,6 +28,9 @@ def _normalize_sql(sql: str) -> str:
return " ".join(sql.split())
# [/DEF:_normalize_sql:Function]
# [DEF:sample_rows:Function]
# @PURPOSE: Sample rows fixture for SQL generation tests.
@pytest.fixture
@@ -38,8 +41,11 @@ def sample_rows() -> List[Dict[str, Any]]:
]
# #region TestQuoteIdentifier [TYPE Class]
# @BRIEF Tests for identifier quoting per dialect.
# [/DEF:sample_rows:Function]
# [DEF:TestQuoteIdentifier:Class]
# @PURPOSE: Tests for identifier quoting per dialect.
class TestQuoteIdentifier:
# [DEF:test_quote_postgresql:Function]
# @PURPOSE: PostgreSQL uses double quotes.
@@ -62,11 +68,16 @@ class TestQuoteIdentifier:
def test_quote_schema_table(self) -> None:
assert _quote_identifier("my_schema", "postgresql") == '"my_schema"'
assert _quote_identifier("my_table", "postgresql") == '"my_table"'
# #endregion TestQuoteIdentifier
# [/DEF:test_quote_postgresql:Function]
# [/DEF:test_quote_clickhouse:Function]
# [/DEF:test_quote_with_existing_quotes:Function]
# [/DEF:test_quote_schema_table:Function]
# [/DEF:TestQuoteIdentifier:Class]
# #region TestEncodeSqlValue [TYPE Class]
# @BRIEF Tests for SQL value encoding.
# [DEF:TestEncodeSqlValue:Class]
# @PURPOSE: Tests for SQL value encoding.
class TestEncodeSqlValue:
# [DEF:test_null:Function]
# @PURPOSE: None encodes as NULL.
@@ -104,71 +115,18 @@ class TestEncodeSqlValue:
def test_special_chars(self) -> None:
assert _encode_sql_value("line1\nline2") == "'line1\nline2'"
# [DEF:test_clickhouse_timestamp_millis:Function]
# @PURPOSE: ClickHouse dialect converts Unix timestamp millis to 'YYYY-MM-DD'.
def test_clickhouse_timestamp_millis(self) -> None:
# 1726358400000 millis = 2024-09-15
result = _encode_sql_value("1726358400000.0", dialect="clickhouse")
assert result == "'2024-09-15'"
# [DEF:test_clickhouse_timestamp_seconds:Function]
# @PURPOSE: ClickHouse dialect converts Unix timestamp seconds to 'YYYY-MM-DD'.
def test_clickhouse_timestamp_seconds(self) -> None:
# 1726358400 seconds = 2024-09-15
result = _encode_sql_value("1726358400", dialect="clickhouse")
assert result == "'2024-09-15'"
# [DEF:test_clickhouse_non_timestamp_string:Function]
# @PURPOSE: Non-timestamp strings are not affected by ClickHouse normalization.
def test_clickhouse_non_timestamp_string(self) -> None:
result = _encode_sql_value("hello world", dialect="clickhouse")
assert result == "'hello world'"
# [DEF:test_postgresql_timestamp_not_normalized:Function]
# @PURPOSE: PostgreSQL dialect does NOT normalize timestamps (different handling).
def test_postgresql_timestamp_not_normalized(self) -> None:
result = _encode_sql_value("1726358400000.0", dialect="postgresql")
assert result == "'1726358400000.0'"
# #endregion TestEncodeSqlValue
# [/DEF:test_null:Function]
# [/DEF:test_string:Function]
# [/DEF:test_integer:Function]
# [/DEF:test_float:Function]
# [/DEF:test_boolean:Function]
# [/DEF:test_sql_injection:Function]
# [/DEF:test_special_chars:Function]
# [/DEF:TestEncodeSqlValue:Class]
# #region TestNormalizeTimestampValue [TYPE Class]
# @BRIEF Tests for Unix timestamp detection and conversion.
class TestNormalizeTimestampValue:
# [DEF:test_millis_timestamp:Function]
# @PURPOSE: Millisecond Unix timestamps convert to YYYY-MM-DD.
def test_millis_timestamp(self) -> None:
# 1726358400000 = 2024-09-15 00:00:00 UTC
assert _normalize_timestamp_value("1726358400000.0") == "2024-09-15"
assert _normalize_timestamp_value("1726358400000") == "2024-09-15"
# [DEF:test_seconds_timestamp:Function]
# @PURPOSE: Second-precision Unix timestamps convert to YYYY-MM-DD.
def test_seconds_timestamp(self) -> None:
assert _normalize_timestamp_value("1726358400") == "2024-09-15"
# [DEF:test_non_timestamp_values:Function]
# @PURPOSE: Regular strings and small numbers return None.
def test_non_timestamp_values(self) -> None:
assert _normalize_timestamp_value("hello") is None
assert _normalize_timestamp_value("12345") is None
assert _normalize_timestamp_value("") is None
assert _normalize_timestamp_value(None) is None
# [DEF:test_float_millis:Function]
# @PURPOSE: Float millis timestamp converts correctly.
def test_float_millis(self) -> None:
assert _normalize_timestamp_value(1726358400000.0) == "2024-09-15"
# [DEF:test_int_seconds:Function]
# @PURPOSE: Integer seconds timestamp converts correctly.
def test_int_seconds(self) -> None:
assert _normalize_timestamp_value(1726358400) == "2024-09-15"
# #endregion TestNormalizeTimestampValue
# #region TestGenerateInsertSql [TYPE Class]
# @BRIEF Tests for plain INSERT SQL generation.
# [DEF:TestGenerateInsertSql:Class]
# @PURPOSE: Tests for plain INSERT SQL generation.
class TestGenerateInsertSql:
# [DEF:test_basic_insert:Function]
# @PURPOSE: Generates basic INSERT with VALUES.
@@ -246,11 +204,18 @@ class TestGenerateInsertSql:
# The single quote should be doubled, preventing injection
assert "'';" in sql or "''" in sql
assert "DROP TABLE" in sql # It's literal data, not a command
# #endregion TestGenerateInsertSql
# [/DEF:test_basic_insert:Function]
# [/DEF:test_insert_with_schema:Function]
# [/DEF:test_empty_columns_raises:Function]
# [/DEF:test_empty_rows_raises:Function]
# [/DEF:test_null_handling:Function]
# [/DEF:test_injection_safety:Function]
# [/DEF:TestGenerateInsertSql:Class]
# #region TestGenerateUpsertSql [TYPE Class]
# @BRIEF Tests for PostgreSQL UPSERT SQL generation.
# [DEF:TestGenerateUpsertSql:Class]
# @PURPOSE: Tests for PostgreSQL UPSERT SQL generation.
class TestGenerateUpsertSql:
# [DEF:test_basic_upsert:Function]
# @PURPOSE: Generates INSERT ... ON CONFLICT DO UPDATE.
@@ -292,11 +257,15 @@ class TestGenerateUpsertSql:
key_columns=[],
rows=sample_rows,
)
# #endregion TestGenerateUpsertSql
# [/DEF:test_basic_upsert:Function]
# [/DEF:test_upsert_all_keys:Function]
# [/DEF:test_upsert_empty_keys:Function]
# [/DEF:TestGenerateUpsertSql:Class]
# #region TestSQLGenerator [TYPE Class]
# @BRIEF Tests for the full SQLGenerator class.
# [DEF:TestSQLGenerator:Class]
# @PURPOSE: Tests for the full SQLGenerator class.
class TestSQLGenerator:
# [DEF:test_postgresql_insert:Function]
# @PURPOSE: PostgreSQL dialect generates proper INSERT.
@@ -388,5 +357,12 @@ class TestSQLGenerator:
assert len(statements) > 1 # Should be split into multiple statements
total_count = sum(count for _, count in statements)
assert total_count == 10
# #endregion TestSQLGenerator
# #endregion SQLGeneratorTests
# [/DEF:test_postgresql_insert:Function]
# [/DEF:test_postgresql_upsert:Function]
# [/DEF:test_clickhouse_insert:Function]
# [/DEF:test_clickhouse_upsert_fallback:Function]
# [/DEF:test_empty_rows_via_generator:Function]
# [/DEF:test_generate_batch:Function]
# [/DEF:TestSQLGenerator:Class]
# [/DEF:SQLGeneratorTests:Module]

17
backend/src/plugins/translate/_utils.py
View File

@@ -1,12 +1,17 @@
# #region DictionaryUtils [C:1] [TYPE Module]
# #endregion DictionaryUtils
# #region DictionaryUtils [C:1] [TYPE Module] [SEMANTICS dictionary, utils, normalize, delimiter]
# @BRIEF Utility helpers for dictionary management (term normalization and delimiter detection).
# @LAYER: Domain
import re
import unicodedata
# #region _normalize_term [C:2] [TYPE Function] [SEMANTICS dictionary,normalize]
# @BRIEF Normalize a term for case-insensitive unique constraint lookup using NFC normalization.
# #region _normalize_term [TYPE Function]
# @BRIEF Normalize a term for case-insensitive unique constraint lookup.
# @RATIONALE: NFC normalization is applied before lowercasing to ensure consistent
# comparison of Unicode characters (e.g. precomposed vs decomposed forms).
# @REJECTED: Lowercasing without NFC normalization — would cause duplicate entries
# for semantically identical Unicode strings in different normalization forms.
def _normalize_term(term: str) -> str:
"""Normalize a term by NFC, lowercasing, and removing extra whitespace."""
if not term:
@@ -16,7 +21,8 @@ def _normalize_term(term: str) -> str:
# #endregion _normalize_term
# #region _detect_delimiter [C:1] [TYPE Function] [SEMANTICS dictionary,delimiter]
# #region _detect_delimiter [TYPE Function]
# @BRIEF Detect the delimiter used in a CSV/TSV header line.
def _detect_delimiter(header_line: str) -> str:
"""Detect delimiter by counting tabs vs commas in the first line."""
if not header_line:
@@ -25,3 +31,4 @@ def _detect_delimiter(header_line: str) -> str:
comma_count = header_line.count(",")
return "\t" if tab_count > comma_count else ","
# #endregion _detect_delimiter
# #endregion DictionaryUtils

227
backend/src/plugins/translate/dictionary.py
View File

@@ -1,15 +1,13 @@
# #region DictionaryManagerModule [C:5] [TYPE Module] [SEMANTICS dictionary,manager,terminology,crud,import,filter]
# #region DictionaryManagerModule [C:4] [TYPE Module] [SEMANTICS dictionary, manager, terminology, crud, import, filter]
# @BRIEF Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TerminologyDictionary]
# @RELATION DEPENDS_ON -> [DictionaryEntry]
# @RELATION DEPENDS_ON -> [TranslationJobDictionary]
# @RELATION DEPENDS_ON -> [TranslationJob]
# @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary]
# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) prohibits duplicate entries within a dictionary.
# @RATIONALE C5 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term.
# @REJECTED Pure C3 CRUD without state guards would allow orphaned job-dictionary links.
# @REJECTED "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TerminologyDictionary:Class]
# @RELATION DEPENDS_ON -> [DictionaryEntry:Class]
# @RELATION DEPENDS_ON -> [TranslationJobDictionary:Class]
# @RELATION DEPENDS_ON -> [TranslationJob:Class]
# @RATIONALE: C4 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term.
# @REJECTED: Pure C3 CRUD without state guards would allow orphaned job-dictionary links.
# @REJECTED: "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing.
import csv
import io
@@ -17,8 +15,7 @@ import re
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.orm import Session
from sqlalchemy import func
from ...core.cot_logger import MarkerLogger
from ...core.logger import belief_scope
from ...core.logger import logger, belief_scope
from ...models.translate import (
TerminologyDictionary,
DictionaryEntry,
@@ -27,20 +24,18 @@ from ...models.translate import (
)
from ._utils import _normalize_term, _detect_delimiter
log = MarkerLogger("DictionaryManager")
# #region DictionaryManager [C:5] [TYPE Class] [SEMANTICS dictionary,manager,crud,import]
# @BRIEF Manages terminology dictionaries and their entries with referential integrity, import, and batch filtering.
# @PRE Database session is open and valid.
# @POST Dictionary and entry mutations are persisted with conflict detection.
# @SIDE_EFFECT Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
# @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary]
# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) enforces no duplicates within a dictionary.
# #region DictionaryManager [C:4] [TYPE Class]
# @BRIEF Manages terminology dictionaries and their entries with referential integrity.
# @PRE: Database session is open and valid.
# @POST: Dictionary and entry mutations are persisted with conflict detection.
# @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
class DictionaryManager:
# #region DictionaryManager.create_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,create]
# @BRIEF Create a new terminology dictionary with name, source_dialect, and target_dialect.
# @RELATION DEPENDS_ON -> [TerminologyDictionary]
# [DEF:DictionaryManager.create_dictionary:Function]
# @COMPLEXITY: 3
# @PURPOSE: Create a new terminology dictionary.
# @PRE: payload contains name, source_dialect, target_dialect.
# @POST: New TerminologyDictionary row is created and returned.
@staticmethod
def create_dictionary(
db: Session, name: str, source_dialect: str, target_dialect: str,
@@ -48,7 +43,7 @@ class DictionaryManager:
is_active: bool = True,
) -> TerminologyDictionary:
with belief_scope("DictionaryManager.create_dictionary"):
log.reason("Creating dictionary", payload={"name": name, "source": source_dialect, "target": target_dialect})
logger.reason("Creating dictionary", {"name": name, "source": source_dialect, "target": target_dialect})
dictionary = TerminologyDictionary(
name=name,
description=description,
@@ -60,13 +55,15 @@ class DictionaryManager:
db.add(dictionary)
db.commit()
db.refresh(dictionary)
log.reflect("Dictionary created", payload={"id": dictionary.id})
logger.reflect("Dictionary created", {"id": dictionary.id})
return dictionary
# #endregion DictionaryManager.create_dictionary
# [/DEF:DictionaryManager.create_dictionary:Function]
# #region DictionaryManager.update_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,update]
# @BRIEF Update an existing terminology dictionary's metadata fields.
# @RELATION DEPENDS_ON -> [TerminologyDictionary]
# [DEF:DictionaryManager.update_dictionary:Function]
# @COMPLEXITY: 3
# @PURPOSE: Update an existing terminology dictionary.
# @PRE: dict_id exists in terminology_dictionaries table.
# @POST: Dictionary metadata is updated and returned.
@staticmethod
def update_dictionary(
db: Session, dict_id: str, name: Optional[str] = None,
@@ -77,7 +74,7 @@ class DictionaryManager:
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
if not dictionary:
raise ValueError(f"Dictionary not found: {dict_id}")
log.reason("Updating dictionary", payload={"id": dict_id})
logger.reason("Updating dictionary", {"id": dict_id})
if name is not None:
dictionary.name = name
if description is not None:
@@ -90,15 +87,16 @@ class DictionaryManager:
dictionary.is_active = is_active
db.commit()
db.refresh(dictionary)
log.reflect("Dictionary updated", payload={"id": dictionary.id})
logger.reflect("Dictionary updated", {"id": dictionary.id})
return dictionary
# #endregion DictionaryManager.update_dictionary
# [/DEF:DictionaryManager.update_dictionary:Function]
# #region DictionaryManager.delete_dictionary [C:4] [TYPE Function] [SEMANTICS dictionary,delete]
# @BRIEF Delete a dictionary, blocked if attached to active/scheduled jobs (referential integrity guard).
# @PRE dict_id exists.
# @POST Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.
# @SIDE_EFFECT Deletes TerminologyDictionary row and all DictionaryEntry rows; checks job attachments first.
# [DEF:DictionaryManager.delete_dictionary:Function]
# @COMPLEXITY: 4
# @PURPOSE: Delete a dictionary, blocked if attached to active/scheduled jobs.
# @PRE: dict_id exists.
# @POST: Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.
# @SIDE_EFFECT: Deletes TerminologyDictionary row and all DictionaryEntry rows.
@staticmethod
def delete_dictionary(db: Session, dict_id: str) -> None:
with belief_scope("DictionaryManager.delete_dictionary"):
@@ -122,33 +120,39 @@ class DictionaryManager:
)
if attached_jobs > 0:
log.explore("Delete blocked: dictionary attached to active jobs", payload={"dict_id": dict_id, "jobs": attached_jobs}, error=f"Dictionary attached to {attached_jobs} active job(s)")
logger.explore("Delete blocked: dictionary attached to active jobs", {"dict_id": dict_id, "jobs": attached_jobs})
raise ValueError(
f"Cannot delete dictionary attached to {attached_jobs} active/scheduled job(s). "
"Remove dictionary associations from jobs first."
)
log.reason("Deleting dictionary", payload={"id": dict_id})
logger.reason("Deleting dictionary", {"id": dict_id})
# Delete entries and job-dictionary links first
db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()
db.query(TranslationJobDictionary).filter(TranslationJobDictionary.dictionary_id == dict_id).delete()
db.delete(dictionary)
db.commit()
log.reflect("Dictionary deleted", payload={"id": dict_id})
# #endregion DictionaryManager.delete_dictionary
logger.reflect("Dictionary deleted", {"id": dict_id})
# [/DEF:DictionaryManager.delete_dictionary:Function]
# #region DictionaryManager.get_dictionary [C:2] [TYPE Function] [SEMANTICS dictionary,get]
# @BRIEF Get a single dictionary by ID with entry count.
# [DEF:DictionaryManager.get_dictionary:Function]
# @COMPLEXITY: 2
# @PURPOSE: Get a single dictionary by ID with entry count.
# @PRE: dict_id exists.
# @POST: Returns dict with dictionary + entry_count or raises ValueError.
@staticmethod
def get_dictionary(db: Session, dict_id: str) -> TerminologyDictionary:
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
if not dictionary:
raise ValueError(f"Dictionary not found: {dict_id}")
return dictionary
# #endregion DictionaryManager.get_dictionary
# [/DEF:DictionaryManager.get_dictionary:Function]
# #region DictionaryManager.list_dictionaries [C:2] [TYPE Function] [SEMANTICS dictionary,list]
# @BRIEF List dictionaries with pagination and total count.
# [DEF:DictionaryManager.list_dictionaries:Function]
# @COMPLEXITY: 2
# @PURPOSE: List dictionaries with pagination and entry counts.
# @PRE: page >= 1, page_size between 1 and 100.
# @POST: Returns (list of dicts, total_count).
@staticmethod
def list_dictionaries(
db: Session, page: int = 1, page_size: int = 20,
@@ -162,11 +166,13 @@ class DictionaryManager:
.all()
)
return dictionaries, total
# #endregion DictionaryManager.list_dictionaries
# [/DEF:DictionaryManager.list_dictionaries:Function]
# #region DictionaryManager.add_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,add]
# @BRIEF Add an entry to a dictionary, enforcing unique source_term_normalized per dictionary.
# @RELATION DEPENDS_ON -> [DictionaryEntry]
# [DEF:DictionaryManager.add_entry:Function]
# @COMPLEXITY: 3
# @PURPOSE: Add an entry to a dictionary, enforcing unique source_term_normalized.
# @PRE: dict_id exists. source_term and target_term are non-empty.
# @POST: New DictionaryEntry row is created or raises on duplicate.
@staticmethod
def add_entry(
db: Session, dict_id: str, source_term: str, target_term: str,
@@ -188,7 +194,7 @@ class DictionaryManager:
f"(id={existing.id}). Use overwrite or keep_existing conflict mode."
)
log.reason("Adding dictionary entry", payload={"dict_id": dict_id, "term": source_term})
logger.reason("Adding dictionary entry", {"dict_id": dict_id, "term": source_term})
entry = DictionaryEntry(
dictionary_id=dict_id,
source_term=source_term.strip(),
@@ -199,13 +205,15 @@ class DictionaryManager:
db.add(entry)
db.commit()
db.refresh(entry)
log.reflect("Entry added", payload={"entry_id": entry.id})
logger.reflect("Entry added", {"entry_id": entry.id})
return entry
# #endregion DictionaryManager.add_entry
# [/DEF:DictionaryManager.add_entry:Function]
# #region DictionaryManager.edit_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,edit]
# @BRIEF Edit an existing dictionary entry with duplicate-aware normalization check.
# @RELATION DEPENDS_ON -> [DictionaryEntry]
# [DEF:DictionaryManager.edit_entry:Function]
# @COMPLEXITY: 3
# @PURPOSE: Edit an existing dictionary entry with duplicate-aware normalization.
# @PRE: entry_id exists.
# @POST: Entry fields are updated.
@staticmethod
def edit_entry(
db: Session, entry_id: str, source_term: Optional[str] = None,
@@ -216,7 +224,7 @@ class DictionaryManager:
if not entry:
raise ValueError(f"Entry not found: {entry_id}")
log.reason("Editing dictionary entry", payload={"entry_id": entry_id})
logger.reason("Editing dictionary entry", {"entry_id": entry_id})
if source_term is not None:
normalized = _normalize_term(source_term)
# Check uniqueness within the same dictionary
@@ -242,41 +250,50 @@ class DictionaryManager:
entry.context_notes = context_notes.strip() if context_notes else None
db.commit()
db.refresh(entry)
log.reflect("Entry updated", payload={"entry_id": entry.id})
logger.reflect("Entry updated", {"entry_id": entry.id})
return entry
# #endregion DictionaryManager.edit_entry
# [/DEF:DictionaryManager.edit_entry:Function]
# #region DictionaryManager.delete_entry [C:2] [TYPE Function] [SEMANTICS dictionary,entry,delete]
# @BRIEF Delete a single dictionary entry.
# [DEF:DictionaryManager.delete_entry:Function]
# @COMPLEXITY: 2
# @PURPOSE: Delete a single dictionary entry.
# @PRE: entry_id exists.
# @POST: Entry is deleted.
@staticmethod
def delete_entry(db: Session, entry_id: str) -> None:
with belief_scope("DictionaryManager.delete_entry"):
entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()
if not entry:
raise ValueError(f"Entry not found: {entry_id}")
log.reason("Deleting dictionary entry", payload={"entry_id": entry_id})
logger.reason("Deleting dictionary entry", {"entry_id": entry_id})
db.delete(entry)
db.commit()
log.reflect("Entry deleted", payload={"entry_id": entry_id})
# #endregion DictionaryManager.delete_entry
logger.reflect("Entry deleted", {"entry_id": entry_id})
# [/DEF:DictionaryManager.delete_entry:Function]
# #region DictionaryManager.clear_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,clear]
# @BRIEF Delete all entries for a dictionary.
# [DEF:DictionaryManager.clear_entries:Function]
# @COMPLEXITY: 2
# @PURPOSE: Delete all entries for a dictionary.
# @PRE: dict_id exists.
# @POST: All entries for the dictionary are deleted.
@staticmethod
def clear_entries(db: Session, dict_id: str) -> int:
with belief_scope("DictionaryManager.clear_entries"):
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
if not dictionary:
raise ValueError(f"Dictionary not found: {dict_id}")
log.reason("Clearing all entries", payload={"dict_id": dict_id})
logger.reason("Clearing all entries", {"dict_id": dict_id})
deleted = db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()
db.commit()
log.reflect("Entries cleared", payload={"dict_id": dict_id, "count": deleted})
logger.reflect("Entries cleared", {"dict_id": dict_id, "count": deleted})
return deleted
# #endregion DictionaryManager.clear_entries
# [/DEF:DictionaryManager.clear_entries:Function]
# #region DictionaryManager.list_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,list]
# @BRIEF List entries for a dictionary with pagination.
# [DEF:DictionaryManager.list_entries:Function]
# @COMPLEXITY: 2
# @PURPOSE: List entries for a dictionary with pagination.
# @PRE: dict_id exists.
# @POST: Returns (list of entries, total_count).
@staticmethod
def list_entries(
db: Session, dict_id: str, page: int = 1, page_size: int = 50,
@@ -296,13 +313,14 @@ class DictionaryManager:
.all()
)
return entries, total
# #endregion DictionaryManager.list_entries
# [/DEF:DictionaryManager.list_entries:Function]
# #region DictionaryManager.import_entries [C:4] [TYPE Function] [SEMANTICS dictionary,import,csv]
# @BRIEF Import entries from CSV/TSV content with duplicate detection and conflict resolution (overwrite/keep_existing/cancel).
# @PRE content is valid CSV or TSV. dict_id exists.
# @POST Entries are created/updated/skipped per conflict mode. Returns result summary with preview support.
# @SIDE_EFFECT Batch-inserts or updates DictionaryEntry rows.
# [DEF:DictionaryManager.import_entries:Function]
# @COMPLEXITY: 4
# @PURPOSE: Import entries from CSV/TSV content with duplicate detection and conflict resolution.
# @PRE: content is valid CSV or TSV. dict_id exists.
# @POST: Entries are created/updated/skipped per conflict mode. Returns result summary.
# @SIDE_EFFECT: Batch-inserts or updates DictionaryEntry rows.
@staticmethod
def import_entries(
db: Session, dict_id: str, content: str,
@@ -319,7 +337,7 @@ class DictionaryManager:
# Detect delimiter if not specified
if not delimiter:
delimiter = _detect_delimiter(content)
log.reason("Detected delimiter", payload={"delimiter": repr(delimiter)})
logger.reason("Detected delimiter", {"delimiter": repr(delimiter)})
if delimiter not in (",", "\t"):
raise ValueError(f"Unsupported delimiter: {repr(delimiter)}. Use ',' or '\\t'.")
@@ -426,7 +444,7 @@ class DictionaryManager:
if not preview_only:
db.commit()
log.reflect("Import complete", payload={
logger.reflect("Import complete", {
"dict_id": dict_id,
"total": result["total"],
"created": result["created"],
@@ -435,13 +453,14 @@ class DictionaryManager:
"errors": len(result["errors"]),
})
return result
# #endregion DictionaryManager.import_entries
# [/DEF:DictionaryManager.import_entries:Function]
# #region DictionaryManager.filter_for_batch [C:4] [TYPE Function] [SEMANTICS dictionary,filter,batch]
# @BRIEF Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.
# @PRE job_id exists and source_texts is a list of strings.
# @POST Returns list of matched entries with match info, sorted by dictionary link priority.
# @SIDE_EFFECT Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.
# [DEF:DictionaryManager.filter_for_batch:Function]
# @COMPLEXITY: 4
# @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.
# @PRE: job_id exists and source_texts is a list of strings.
# @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries.
# @SIDE_EFFECT: Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.
@staticmethod
def filter_for_batch(
db: Session, source_texts: List[str], job_id: str,
@@ -454,7 +473,7 @@ class DictionaryManager:
.all()
)
if not job_dict_links:
log.reason("No dictionaries attached to job", payload={"job_id": job_id})
logger.reason("No dictionaries attached to job", {"job_id": job_id})
return []
dict_ids = [jd.dictionary_id for jd in job_dict_links]
@@ -525,21 +544,22 @@ class DictionaryManager:
dict_priority = {d_id: idx for idx, d_id in enumerate(dict_ids)}
matched.sort(key=lambda m: (dict_priority.get(m["dictionary_id"], 999), m["text_index"]))
log.reflect("Batch filter match complete", payload={
logger.reflect("Batch filter match complete", {
"job_id": job_id,
"source_texts": len(source_texts),
"matches": len(matched),
"dictionaries_used": len(dictionaries),
})
return matched
# #endregion DictionaryManager.filter_for_batch
# [/DEF:DictionaryManager.filter_for_batch:Function]
# #region DictionaryManager.submit_correction [C:4] [TYPE Function] [SEMANTICS dictionary,correction,submit]
# @BRIEF Submit a term correction from a run result with conflict detection and origin tracking.
# @PRE source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.
# @POST Entry created or updated; origin tracking populated. Returns action + conflict info.
# @SIDE_EFFECT Creates/updates DictionaryEntry row.
# [DEF:DictionaryManager.submit_correction:Function]
# @COMPLEXITY: 4
# @PURPOSE: Submit a term correction from a run result. Validates language match, detects conflicts.
# @PRE: source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.
# @POST: Entry created or updated; origin tracking populated. Returns action + conflict info.
# @SIDE_EFFECT: Creates/updates DictionaryEntry row.
@staticmethod
def submit_correction(
db: Session,
@@ -628,15 +648,16 @@ class DictionaryManager:
result["message"] = f"Entry created for '{source_term}' -> '{corrected_target_term}'"
db.commit()
log.reflect("Correction processed", payload=result)
logger.reflect("Correction processed", result)
return result
# #endregion DictionaryManager.submit_correction
# [/DEF:DictionaryManager.submit_correction:Function]
# #region DictionaryManager.submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS dictionary,correction,bulk]
# @BRIEF Submit multiple term corrections atomically — all succeed or all fail on conflict.
# @PRE corrections list is non-empty. dict_id exists.
# @POST All corrections applied or none applied with conflict list (atomic).
# @SIDE_EFFECT Creates/updates DictionaryEntry rows; commits once.
# [DEF:DictionaryManager.submit_bulk_corrections:Function]
# @COMPLEXITY: 4
# @PURPOSE: Submit multiple term corrections atomically — all succeed or all fail.
# @PRE: corrections list is non-empty. dict_id exists.
# @POST: All corrections applied or none applied with conflict list.
# @SIDE_EFFECT: Creates/updates DictionaryEntry rows; commits once.
@staticmethod
def submit_bulk_corrections(
db: Session,
@@ -750,7 +771,7 @@ class DictionaryManager:
"total": len(corrections),
"applied": sum(1 for r in results if r["action"] in ("created", "updated")),
}
# #endregion DictionaryManager.submit_bulk_corrections
# [/DEF:DictionaryManager.submit_bulk_corrections:Function]
# #endregion DictionaryManager

156
backend/src/plugins/translate/events.py
View File

@@ -1,27 +1,24 @@
# #region TranslationEventLog [C:5] [TYPE Module] [SEMANTICS translate,events,audit,logging]
# #region TranslationEventLog [C:5] [TYPE Module] [SEMANTICS translate, events, audit, logging]
# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement.
# @LAYER Domain
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationEvent]
# @RELATION DEPENDS_ON -> [MetricSnapshot]
# @PRE Database session is open and valid.
# @POST Events are persisted immutably; terminal events enforce exactly-one invariant per run.
# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion.
# @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]
# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id.
# @RATIONALE Immutable event log with nullable run_id allows job-level events (no run context) and run-level events.
# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# @REJECTED Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit.
# @PRE: Database session is open and valid.
# @POST: Events are persisted immutably; terminal events enforce exactly-one invariant per run.
# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion.
# @DATA_CONTRACT: Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]
# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id.
# @RATIONALE: Immutable event log with nullable run_id allows job-level events (no run context) and run-level events.
# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# @REJECTED: Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit.
from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from datetime import datetime, timezone, timedelta
import uuid
from ...core.cot_logger import MarkerLogger
from ...core.logger import belief_scope
from ...core.logger import logger, belief_scope
from ...models.translate import TranslationEvent, MetricSnapshot
log = MarkerLogger("EventLog")
# Terminal events per run — exactly one of these must exist for a completed run.
TERMINAL_EVENT_TYPES = {"RUN_COMPLETED", "RUN_FAILED", "RUN_CANCELLED"}
# All valid event types for validation.
@@ -39,48 +36,23 @@ VALID_EVENT_TYPES = TERMINAL_EVENT_TYPES | {
DEFAULT_RETENTION_DAYS = 90
# #region TranslationEventLog [C:5] [TYPE Class] [SEMANTICS translate,events,log,invariants]
# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement and pruning.
# @PRE Database session is available.
# @POST Events are written immutably; terminal events enforced per run; expired events pruned with MetricSnapshot.
# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events.
# @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]
# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id.
# #region TranslationEventLog [C:5] [TYPE Class]
# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement.
# @PRE: Database session is available.
# @POST: Events are written immutably; terminal events enforced per run.
# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events.
# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id.
class TranslationEventLog:
def __init__(self, db: Session):
self.db = db
# #region _create_event_raw [C:3] [TYPE Function] [SEMANTICS translate,events,create]
# @BRIEF Create a TranslationEvent row without invariant checks. Internal helper for callers that manage invariants themselves.
# @RELATION DEPENDS_ON -> [TranslationEvent]
def _create_event_raw(
self,
job_id: str,
event_type: str,
payload: Optional[Dict[str, Any]] = None,
run_id: Optional[str] = None,
created_by: Optional[str] = None,
) -> TranslationEvent:
event = TranslationEvent(
id=str(uuid.uuid4()),
job_id=job_id,
run_id=run_id,
event_type=event_type,
event_data=payload or {},
created_by=created_by,
created_at=datetime.now(timezone.utc),
)
self.db.add(event)
self.db.flush()
return event
# #endregion _create_event_raw
# #region log_event [C:4] [TYPE Function] [SEMANTICS translate,events,log]
# @BRIEF Write an immutable event. Enforces terminal event invariant and run_started ordering for non-null run_id.
# @PRE event_type must be a known type. If run_id is not None, enforce terminal + start invariants.
# @POST TranslationEvent row is created; invariants checked before write.
# @SIDE_EFFECT DB write.
# [DEF:log_event:Function]
# @COMPLEXITY: 4
# @PURPOSE: Write an immutable event. Enforces terminal event invariant for non-null run_id.
# @PRE: event_type must be a known type. If run_id is not None, enforce terminal invariant.
# @POST: TranslationEvent row is created.
# @SIDE_EFFECT: DB write.
def log_event(
self,
job_id: str,
@@ -107,10 +79,9 @@ class TranslationEventLog:
.first()
)
if existing_terminal:
existing_id = existing_terminal[0] if isinstance(existing_terminal, (tuple, list)) else str(existing_terminal.id)
raise ValueError(
f"Run '{run_id}' already has a terminal event "
f"({existing_id}). Cannot add another terminal event."
f"({existing_terminal[0]}). Cannot add another terminal event."
)
# Enforce run_started invariant — must exist before other run events
@@ -129,24 +100,31 @@ class TranslationEventLog:
f"RUN_STARTED event must precede other run events."
)
event = self._create_event_raw(
event = TranslationEvent(
id=str(uuid.uuid4()),
job_id=job_id,
event_type=event_type,
payload=payload,
run_id=run_id,
event_type=event_type,
event_data=payload or {},
created_by=created_by,
created_at=datetime.now(timezone.utc),
)
log.reason("Event logged", payload={
self.db.add(event)
self.db.flush()
logger.reason(f"Event logged: {event_type}", {
"event_id": event.id,
"job_id": job_id,
"run_id": run_id,
"event_type": event_type,
})
return event
# #endregion log_event
# [/DEF:log_event:Function]
# #region query_events [C:2] [TYPE Function] [SEMANTICS translate,events,query]
# @BRIEF Query events with optional filters (job_id, run_id, event_type) and pagination.
# [DEF:query_events:Function]
# @PURPOSE: Query events with optional filters.
# @PRE: None.
# @POST: Returns list of TranslationEvent dicts matching filters.
def query_events(
self,
job_id: Optional[str] = None,
@@ -184,13 +162,13 @@ class TranslationEventLog:
}
for e in events
]
# #endregion query_events
# [/DEF:query_events:Function]
# #region prune_expired [C:4] [TYPE Function] [SEMANTICS translate,events,prune]
# @BRIEF Delete events older than retention_days. Creates MetricSnapshot before pruning for audit trail.
# @PRE None.
# @POST Expired events are deleted; MetricSnapshot is created before deletion in batch.
# @SIDE_EFFECT Creates MetricSnapshot row; deletes TranslationEvent rows in batches.
# [DEF:prune_expired:Function]
# @PURPOSE: Delete events older than retention_days. Persists MetricSnapshot before pruning.
# @PRE: None.
# @POST: Expired events are deleted; MetricSnapshot is created before deletion.
# @SIDE_EFFECT: Creates MetricSnapshot row; deletes TranslationEvent rows.
def prune_expired(
self,
retention_days: int = DEFAULT_RETENTION_DAYS,
@@ -198,7 +176,7 @@ class TranslationEventLog:
) -> Dict[str, Any]:
with belief_scope("TranslationEventLog.prune_expired"):
cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
log.reason("Pruning expired events", payload={
logger.reason("Pruning expired events", {
"cutoff": cutoff.isoformat(),
"retention_days": retention_days,
})
@@ -210,7 +188,7 @@ class TranslationEventLog:
total_expired = expired_query.count()
if total_expired == 0:
log.reflect("No expired events to prune", payload={})
logger.reflect("No expired events to prune", {})
return {"pruned": 0, "snapshot_id": None}
# Create MetricSnapshot before pruning
@@ -243,20 +221,21 @@ class TranslationEventLog:
).delete(synchronize_session=False)
self.db.flush()
pruned += len(ids)
log.reason("Pruned batch", payload={"batch_size": len(ids), "total_pruned": pruned})
logger.reason("Pruned batch", {"batch_size": len(ids), "total_pruned": pruned})
self.db.commit()
log.reflect("Pruning complete", payload={
logger.reflect("Pruning complete", {
"pruned": pruned,
"snapshot_id": snapshot_id,
})
return {"pruned": pruned, "snapshot_id": snapshot_id}
# #endregion prune_expired
# [/DEF:prune_expired:Function]
# #region get_run_event_summary [C:3] [TYPE Function] [SEMANTICS translate,events,summary]
# @BRIEF Get a summary of events for a run, including invariant validity check.
# @RELATION DEPENDS_ON -> [TranslationEvent]
# [DEF:get_run_event_summary:Function]
# @PURPOSE: Get a summary of events for a run, including invariant check.
# @PRE: run_id is not None.
# @POST: Returns dict with event list and invariant validity.
def get_run_event_summary(self, run_id: str) -> Dict[str, Any]:
with belief_scope("TranslationEventLog.get_run_event_summary"):
events = self.query_events(run_id=run_id)
@@ -273,34 +252,7 @@ class TranslationEventLog:
"invariant_valid": has_started and len(terminal_events) <= 1,
"events": events,
}
# #endregion get_run_event_summary
# #region get_run_event_invariants_lightweight [C:3] [TYPE Function] [SEMANTICS translate,events,invariants]
# @BRIEF Lightweight invariant check for a run — fetches only event_type column, avoids event_data for performance.
# @RELATION DEPENDS_ON -> [TranslationEvent]
def get_run_event_invariants_lightweight(self, run_id: str) -> Dict[str, Any]:
with belief_scope("TranslationEventLog.get_run_event_invariants_lightweight"):
# Lightweight query: only fetch event_type column, not event_data
event_types = [
row[0]
for row in (
self.db.query(TranslationEvent.event_type)
.filter(TranslationEvent.run_id == run_id)
.order_by(TranslationEvent.created_at.desc())
.limit(100)
.all()
)
]
has_run_started = "RUN_STARTED" in event_types
terminal_count = sum(1 for et in event_types if et in TERMINAL_EVENT_TYPES)
return {
"has_run_started": has_run_started,
"terminal_event_count": terminal_count,
"invariant_valid": has_run_started and terminal_count <= 1,
}
# #endregion get_run_event_invariants_lightweight
# [/DEF:get_run_event_summary:Function]
# #endregion TranslationEventLog

285
backend/src/plugins/translate/executor.py
View File

@@ -1,19 +1,17 @@
# #region TranslationExecutor [C:5] [TYPE Module] [SEMANTICS translate,executor,batch,llm]
# #region TranslationExecutor [C:4] [TYPE Module] [SEMANTICS translate, executor, batch, llm]
# @BRIEF Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows.
# @LAYER Domain
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationBatch]
# @RELATION DEPENDS_ON -> [TranslationRecord]
# @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [DictionaryManager]
# @RELATION DEPENDS_ON -> [TranslationPreview]
# @PRE Valid TranslationRun with job configuration. DB session is available.
# @POST TranslationBatch and TranslationRecord rows are created. Run status is updated.
# @SIDE_EFFECT Calls LLM provider; creates DB rows; updates run statistics.
# @DATA_CONTRACT Input[TranslationRun, job] -> Output[TranslationRun with batches and records]
# @INVARIANT Batch processing with retry — independent batches allow partial recovery; MAX_RETRIES_PER_BATCH = 3.
# @RATIONALE Batch processing with retry — independent batches allow partial recovery.
# @REJECTED Single monolithic LLM call — would lose all progress on any failure.
# @PRE: Valid TranslationRun with job configuration. DB session is available.
# @POST: TranslationBatch and TranslationRecord rows are created. Run status is updated.
# @SIDE_EFFECT: Calls LLM provider; creates DB rows; updates run statistics.
# @RATIONALE: Batch processing with retry — independent batches allow partial recovery.
# @REJECTED: Single monolithic LLM call — would lose all progress on any failure.
import json
import time
@@ -24,7 +22,6 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Callable
from sqlalchemy.orm import Session
from ...core.logger import logger, belief_scope
from ...core.cot_logger import MarkerLogger
from ...core.config_manager import ConfigManager
from ...models.translate import (
TranslationJob,
@@ -36,24 +33,20 @@ from ...models.translate import (
)
from ...services.llm_provider import LLMProviderService
from ...services.llm_prompt_templates import render_prompt
from ...core.superset_client import SupersetClient
from .dictionary import DictionaryManager
from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE
log = MarkerLogger("TranslationExecutor")
# #region MAX_RETRIES_PER_BATCH [C:1] [TYPE Constant] [SEMANTICS translate,executor,retry]
# #region MAX_RETRIES_PER_BATCH [TYPE Constant]
# @BRIEF Maximum number of retries for a single batch before marking it failed.
MAX_RETRIES_PER_BATCH = 3
# #endregion MAX_RETRIES_PER_BATCH
# #region TranslationExecutor [C:5] [TYPE Class] [SEMANTICS translate,executor,batch]
# #region TranslationExecutor [C:4] [TYPE Class]
# @BRIEF Process translation batches: fetch source rows, filter dict, call LLM, persist results.
# @PRE DB session and config manager available.
# @POST Batches and records created with status tracking; run statistics updated.
# @SIDE_EFFECT LLM API calls; DB writes.
# @DATA_CONTRACT Input[TranslationRun] -> Output[TranslationRun with batches, records, stats]
# @INVARIANT MAX_RETRIES_PER_BATCH limit enforced; partial batch success tracked via COMPLETED_WITH_ERRORS.
# @PRE: DB session and config manager available.
# @POST: Batches and records created with status tracking.
# @SIDE_EFFECT: LLM API calls; DB writes.
class TranslationExecutor:
def __init__(
@@ -69,27 +62,25 @@ class TranslationExecutor:
self.on_batch_progress = on_batch_progress
self._current_run_id: Optional[str] = None
# #region execute_run [C:4] [TYPE Function] [SEMANTICS translate,run,execute]
# @BRIEF Run full translation execution for a TranslationRun: fetch rows, batch, call LLM, persist.
# @PRE run is in PENDING or RUNNING status with valid job config.
# @POST Run is populated with batches and records; run status updated to COMPLETED/FAILED.
# @SIDE_EFFECT LLM API calls; DB batch writes.
# [DEF:execute_run:Function]
# @PURPOSE: Run full translation execution for a TranslationRun.
# @PRE: run is in PENDING or RUNNING status with valid job config.
# @POST: Run is populated with batches and records.
# @SIDE_EFFECT: LLM API calls; DB batch writes.
def execute_run(
self,
run: TranslationRun,
llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None,
full_translation: bool = False,
) -> TranslationRun:
with belief_scope("TranslationExecutor.execute_run"):
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
if not job:
raise ValueError(f"Job '{run.job_id}' not found for run '{run.id}'")
log.reason("Starting translation execution", payload={
logger.reason("Starting translation execution", {
"run_id": run.id,
"job_id": job.id,
"batch_size": job.batch_size,
"full_translation": full_translation,
})
# Mark run as RUNNING
@@ -97,15 +88,10 @@ class TranslationExecutor:
run.started_at = datetime.now(timezone.utc)
self.db.flush()
# Fetch source rows
if full_translation:
# Full translation: fetch ALL rows from Superset dataset
source_rows = self._fetch_all_rows_from_superset(job)
else:
# Preview-based: fetch rows from the accepted preview session
source_rows = self._fetch_source_rows(job.id, run.id)
# Fetch source rows from the accepted preview session
source_rows = self._fetch_source_rows(job.id, run.id)
if not source_rows:
log.explore("No source rows to translate", payload={"run_id": run.id}, error="Preview produced 0 source rows for translation")
logger.explore("No source rows to translate", {"run_id": run.id})
run.status = "COMPLETED"
run.completed_at = datetime.now(timezone.utc)
self.db.flush()
@@ -121,7 +107,7 @@ class TranslationExecutor:
for i in range(0, total_rows, batch_size)
]
log.reason(f"Processing {len(batches)} batches", payload={
logger.reason(f"Processing {len(batches)} batches", {
"run_id": run.id,
"total_rows": total_rows,
"batch_size": batch_size,
@@ -165,7 +151,7 @@ class TranslationExecutor:
run.completed_at = datetime.now(timezone.utc)
self.db.flush()
log.reflect("Translation execution complete", payload={
logger.reflect("Translation execution complete", {
"run_id": run.id,
"status": run.status,
"total": total_rows,
@@ -175,13 +161,12 @@ class TranslationExecutor:
})
return run
# #endregion execute_run
# [/DEF:execute_run:Function]
# #region _fetch_source_rows [C:4] [TYPE Function] [SEMANTICS translate,source,rows]
# @BRIEF Fetch source rows from the accepted preview session for this job.
# @PRE job_id exists.
# @POST Returns list of dicts with source data (row_index, source_text, approved_translation, etc).
# @SIDE_EFFECT Queries preview session and records from DB.
# [DEF:_fetch_source_rows:Function]
# @PURPOSE: Fetch source rows from the accepted preview session for this job.
# @PRE: job_id exists.
# @POST: Returns list of dicts with source data.
def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]:
with belief_scope("TranslationExecutor._fetch_source_rows"):
# Get the latest APPLIED preview session
@@ -195,7 +180,7 @@ class TranslationExecutor:
.first()
)
if not session:
log.explore("No accepted preview session found", error="Preview session has no accepted rows", payload={"job_id": job_id})
logger.explore("No accepted preview session found", {"job_id": job_id})
return []
# Fetch APPROVED or all records from the session
@@ -215,124 +200,20 @@ class TranslationExecutor:
"source_text": rec.source_sql or "",
"approved_translation": rec.target_sql if rec.status == "APPROVED" else None,
"source_object_name": rec.source_object_name or "",
"source_data": rec.source_data or {},
})
log.reason(f"Fetched {len(source_rows)} source rows from preview", payload={
logger.reason(f"Fetched {len(source_rows)} source rows from preview", {
"run_id": run_id,
"session_id": session.id,
})
return source_rows
# #endregion _fetch_source_rows
# [/DEF:_fetch_source_rows:Function]
# #region _fetch_all_rows_from_superset [C:4] [TYPE Function] [SEMANTICS translate,superset,rows]
# @BRIEF Fetch ALL rows from the Superset dataset for full translation (paginated).
# @PRE job has source_datasource_id and environment_id configured.
# @POST Returns list of row dicts with row_index, source_text, source_data.
# @SIDE_EFFECT Calls Superset chart data endpoint (paginated).
def _fetch_all_rows_from_superset(self, job: TranslationJob) -> List[Dict[str, Any]]:
with belief_scope("TranslationExecutor._fetch_all_rows_from_superset"):
env_id = job.environment_id or job.source_dialect or ""
environments = self.config_manager.get_environments()
env_config = next(
(e for e in environments if e.id == env_id),
None,
)
if not env_config:
raise ValueError(f"Superset environment '{env_id}' not found")
client = SupersetClient(env_config)
dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))
# Build query context (same as preview, but with large row_limit)
query_context = client.build_dataset_preview_query_context(
dataset_id=int(job.source_datasource_id),
dataset_record=dataset_detail,
template_params={},
effective_filters=[],
)
queries = query_context.get("queries", [])
if queries:
queries[0]["row_limit"] = 100000 # Superset max default
queries[0].pop("result_type", None)
queries[0].pop("columns", None)
queries[0]["metrics"] = []
queries[0]["row_offset"] = 0
query_context["result_type"] = "samples"
form_data = query_context.get("form_data", {})
form_data.pop("query_mode", None)
all_rows: List[Dict[str, Any]] = []
batch_size_fetch = 10000 # Fetch 10K rows per pagination request
while True:
# Set pagination for this batch
if queries:
queries[0]["row_limit"] = min(batch_size_fetch, 100000)
queries[0]["row_offset"] = len(all_rows)
try:
response = client.network.request(
method="POST",
endpoint="/api/v1/chart/data",
data=json.dumps(query_context),
headers={"Content-Type": "application/json"},
)
except Exception as e:
log.explore("Chart data API failed during full fetch", error="Chart data API error",
payload={
"offset": len(all_rows),
"error": str(e),
})
if not all_rows:
raise ValueError(f"Failed to fetch data from Superset: {e}")
break # Return what we have
from .preview import TranslationPreview
rows = TranslationPreview._extract_data_rows(response)
if not rows:
break # No more data
all_rows.extend(rows)
log.reason(f"Fetched {len(rows)} rows (total: {len(all_rows)})", payload={
"offset": len(all_rows) - len(rows),
})
if len(rows) < batch_size_fetch:
break # Last page
# Convert to the format expected by _process_batch
source_rows = []
for idx, row in enumerate(all_rows):
translation_value = str(row.get(job.translation_column, "") or "")
context_values = {}
if job.context_columns:
for col in job.context_columns:
context_values[col] = str(row.get(col, "") or "")
# Also include target_key_cols values in context_data
if job.target_key_cols:
for col in job.target_key_cols:
context_values[col] = str(row.get(col, "") or "")
source_rows.append({
"row_index": str(idx),
"source_text": translation_value,
"source_data": context_values,
"source_object_name": f"Row {idx + 1}",
"approved_translation": None,
})
log.reason(f"Prepared {len(source_rows)} source rows for full translation", payload={
"job_id": job.id,
})
return source_rows
# #endregion _fetch_all_rows_from_superset
# #region _process_batch [C:4] [TYPE Function] [SEMANTICS translate,batch,process]
# @BRIEF Process a single batch: filter dictionary, build prompt, call LLM, persist records.
# @PRE job and batch_rows are valid.
# @POST TranslationBatch and TranslationRecord rows are created.
# @SIDE_EFFECT LLM API call; DB writes.
# [DEF:_process_batch:Function]
# @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records.
# @PRE: job and batch_rows are valid.
# @POST: TranslationBatch and TranslationRecord rows are created.
# @SIDE_EFFECT: LLM API call.
def _process_batch(
self,
job: TranslationJob,
@@ -391,7 +272,6 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SUCCESS",
)
self.db.add(record)
@@ -419,20 +299,20 @@ class TranslationExecutor:
self.db.flush()
batch_latency = int((time.monotonic() - batch_start) * 1000)
log.reason(f"Batch {batch_index} complete", payload={
logger.reason(f"Batch {batch_index} complete", {
"batch_id": batch_id,
"latency_ms": batch_latency,
**result,
})
return result
# #endregion _process_batch
# [/DEF:_process_batch:Function]
# #region _call_llm_for_batch [C:4] [TYPE Function] [SEMANTICS translate,llm,batch]
# @BRIEF Call LLM for a batch of rows requiring translation. Parse structured JSON response.
# @PRE job has valid provider_id. batch_rows is non-empty.
# @POST Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.
# @SIDE_EFFECT HTTP call to LLM provider.
# [DEF:_call_llm_for_batch:Function]
# @PURPOSE: Call LLM for a batch of rows requiring translation. Parse structured JSON response.
# @PRE: job has valid provider_id. batch_rows is non-empty.
# @POST: Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.
# @SIDE_EFFECT: HTTP call to LLM provider.
def _call_llm_for_batch(
self,
job: TranslationJob,
@@ -493,8 +373,7 @@ class TranslationExecutor:
except Exception as e:
last_error = str(e)
retries += 1
log.explore("LLM call failed", error="LLM call failed, retrying",
payload={
logger.explore(f"LLM call failed (attempt {attempt})", {
"batch_id": batch_id,
"error": last_error,
"attempt": attempt,
@@ -502,8 +381,7 @@ class TranslationExecutor:
if attempt < MAX_RETRIES_PER_BATCH:
time.sleep(2 ** attempt) # Exponential backoff
else:
log.explore("LLM call exhausted retries", error="LLM retries exhausted",
payload={
logger.explore("LLM call exhausted retries", {
"batch_id": batch_id,
"last_error": last_error,
})
@@ -520,7 +398,6 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="FAILED",
error_message=f"LLM call failed after {retries} retries: {last_error}",
)
@@ -532,8 +409,7 @@ class TranslationExecutor:
translations = self._parse_llm_response(llm_response, len(batch_rows))
except ValueError as e:
# Parse failure — mark all rows as SKIPPED
log.explore("LLM response parse failed", error="Failed to parse LLM JSON response",
payload={
logger.explore("LLM response parse failed", {
"batch_id": batch_id,
"error": str(e),
"response_preview": llm_response[:500] if llm_response else "",
@@ -549,7 +425,6 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED",
error_message=f"LLM parse failure: {e}",
)
@@ -581,7 +456,6 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED",
error_message="NULL translation returned by LLM",
)
@@ -600,7 +474,6 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SKIPPED",
error_message="Empty translation returned by LLM",
)
@@ -617,7 +490,6 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data", {}),
status="SUCCESS",
)
self.db.add(record)
@@ -628,13 +500,13 @@ class TranslationExecutor:
"skipped": skipped,
"retries": retries,
}
# #endregion _call_llm_for_batch
# [/DEF:_call_llm_for_batch:Function]
# #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call]
# @BRIEF Call the configured LLM provider with the batch prompt, routing by provider type.
# @PRE job has valid provider_id.
# @POST Returns raw LLM response string.
# @SIDE_EFFECT HTTP call to LLM provider.
# [DEF:_call_llm:Function]
# @PURPOSE: Call the configured LLM provider with the batch prompt.
# @PRE: job has valid provider_id.
# @POST: Returns raw LLM response string.
# @SIDE_EFFECT: HTTP call to LLM provider.
def _call_llm(self, job: TranslationJob, prompt: str) -> str:
with belief_scope("TranslationExecutor._call_llm"):
if not job.provider_id:
@@ -662,13 +534,13 @@ class TranslationExecutor:
)
else:
raise ValueError(f"Unsupported provider type '{provider_type}'")
# #endregion _call_llm
# [/DEF:_call_llm:Function]
# #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai]
# @BRIEF Call OpenAI-compatible API for batch translation with retry and structured output support.
# @PRE Valid API endpoint, key, model, and prompt.
# @POST Returns response text.
# @SIDE_EFFECT HTTP POST to LLM API.
# [DEF:_call_openai_compatible:Function]
# @PURPOSE: Call OpenAI-compatible API for batch translation.
# @PRE: Valid API endpoint, key, model, and prompt.
# @POST: Returns response text.
# @SIDE_EFFECT: HTTP POST to LLM API.
@staticmethod
def _call_openai_compatible(
base_url: str,
@@ -699,7 +571,7 @@ class TranslationExecutor:
if provider_type in ("openai", "openai_compatible"):
payload["response_format"] = {"type": "json_object"}
log.reason(
logger.reason(
f"LLM request model={payload.get('model')} "
f"provider_type={provider_type} "
f"response_format={'yes' if 'response_format' in payload else 'no'} "
@@ -707,11 +579,11 @@ class TranslationExecutor:
)
response = http_requests.post(url, headers=headers, json=payload, timeout=180)
if not response.ok:
log.explore("LLM API error", error=f"LLM API returned status {response.status_code}", payload={
"status_code": response.status_code,
"model": payload.get('model'),
"body": response.text[:2000],
})
logger.explore(
f"LLM API error status={response.status_code} "
f"model={payload.get('model')} "
f"body={response.text[:2000]}"
)
response.raise_for_status()
data = response.json()
@@ -721,27 +593,15 @@ class TranslationExecutor:
content = choices[0].get("message", {}).get("content", "")
if not content:
# Log full response for diagnostics
finish_reason = choices[0].get("finish_reason", "unknown")
log.explore("LLM returned empty content", error="Empty response from LLM",
payload={
"finish_reason": finish_reason,
"model": payload.get("model"),
"prompt_len": len(prompt),
"response_keys": list(data.keys()),
"choices_count": len(choices),
})
raise ValueError(
f"LLM returned empty content (finish_reason={finish_reason}, "
f"model={payload.get('model')}, prompt_len={len(prompt)})"
)
raise ValueError("LLM returned empty content")
return content
# #endregion _call_openai_compatible
# [/DEF:_call_openai_compatible:Function]
# #region _parse_llm_response [C:3] [TYPE Function] [SEMANTICS translate,llm,parse]
# @BRIEF Parse LLM JSON response into dict of row_id -> translation text.
# @RELATION DEPENDS_ON -> [json]
# [DEF:_parse_llm_response:Function]
# @PURPOSE: Parse LLM JSON response into dict of row_id -> translation.
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
# @POST: Returns dict mapping row_id to translation text.
@staticmethod
def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:
with belief_scope("TranslationExecutor._parse_llm_response"):
@@ -774,8 +634,9 @@ class TranslationExecutor:
translations[row_id] = str(translation)
return translations
# #endregion _parse_llm_response
# [/DEF:_parse_llm_response:Function]
# #endregion TranslationExecutor
# #endregion TranslationExecutor
# #endregion TranslationExecutor

36
backend/src/plugins/translate/metrics.py
View File

@@ -1,10 +1,14 @@
# #region TranslationMetrics [C:3] [TYPE Module] [SEMANTICS translate,metrics,aggregation]
# #region TranslationMetrics [C:3] [TYPE Module] [SEMANTICS translate, metrics, aggregation]
# @BRIEF Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationEvent]
# @RELATION DEPENDS_ON -> [MetricSnapshot]
# @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [TranslationSchedule]
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationEvent:Class]
# @RELATION DEPENDS_ON -> [MetricSnapshot:Class]
# @RELATION DEPENDS_ON -> [TranslationRun:Class]
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @PRE: Database session is open.
# @POST: Metrics are aggregated and returned; no side effects.
# @RATIONALE: Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture.
# @REJECTED: Querying all events for all time — would be slow; MetricSnapshot provides historical summary.
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Tuple
@@ -20,18 +24,17 @@ from ...models.translate import (
)
# #region TranslationMetrics [C:3] [TYPE Class] [SEMANTICS translate,metrics]
# #region TranslationMetrics [C:3] [TYPE Class]
# @BRIEF Aggregate translation metrics from live events and MetricSnapshot.
# @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [MetricSnapshot]
# @RELATION DEPENDS_ON -> [TranslationEvent]
class TranslationMetrics:
def __init__(self, db: Session):
self.db = db
# #region get_job_metrics [C:2] [TYPE Function] [SEMANTICS translate,metrics,job]
# @BRIEF Get aggregated metrics for a specific job including run counts, record stats, and duration.
# [DEF:get_job_metrics:Function]
# @PURPOSE: Get aggregated metrics for a specific job.
# @PRE: job_id exists.
# @POST: Returns dict with metrics from events + latest snapshot.
def get_job_metrics(self, job_id: str) -> Dict[str, Any]:
with belief_scope("TranslationMetrics.get_job_metrics"):
# Run counts from TranslationRun
@@ -145,10 +148,11 @@ class TranslationMetrics:
"last_run_at": last_run.created_at.isoformat() if last_run else None,
"next_scheduled_run": next_schedule.last_run_at.isoformat() if next_schedule and next_schedule.last_run_at else None,
}
# #endregion get_job_metrics
# [/DEF:get_job_metrics:Function]
# #region get_all_metrics [C:2] [TYPE Function] [SEMANTICS translate,metrics,all]
# @BRIEF Get aggregated metrics for all jobs.
# [DEF:get_all_metrics:Function]
# @PURPOSE: Get aggregated metrics for all jobs.
# @POST: Returns list of per-job metrics.
def get_all_metrics(self) -> List[Dict[str, Any]]:
with belief_scope("TranslationMetrics.get_all_metrics"):
job_ids = (
@@ -157,7 +161,7 @@ class TranslationMetrics:
.all()
)
return [self.get_job_metrics(jid[0]) for jid in job_ids if jid[0]]
# #endregion get_all_metrics
# [/DEF:get_all_metrics:Function]
# #endregion TranslationMetrics

241
backend/src/plugins/translate/orchestrator.py
View File

@@ -1,6 +1,6 @@
# #region TranslationOrchestrator [C:5] [TYPE Module] [SEMANTICS translate,orchestrator,lifecycle,run]
# #region TranslationOrchestrator [C:5] [TYPE Module] [SEMANTICS translate, orchestrator, lifecycle, run]
# @BRIEF Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events.
# @LAYER Domain
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationRun]
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION DEPENDS_ON -> [TranslationExecutor]
@@ -8,14 +8,14 @@
# @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor]
# @RELATION DEPENDS_ON -> [TranslationEventLog]
# @RELATION DEPENDS_ON -> [TranslationPreviewSession]
# @PRE Valid job and accepted preview (for manual runs). Superset and LLM are reachable.
# @POST Translation run is executed, SQL generated and submitted, events recorded.
# @SIDE_EFFECT Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events.
# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun]
# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run.
# @RATIONALE C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary.
# @REJECTED Distributed actor model (Celery) — eventual-consistency challenges at current scale.
# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# @PRE: Valid job and accepted preview (for manual runs). Superset and LLM are reachable.
# @POST: Translation run is executed, SQL generated and submitted, events recorded.
# @SIDE_EFFECT: Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events.
# @DATA_CONTRACT: Input[db, config_manager, current_user] -> Output[TranslationRun]
# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run.
# @RATIONALE: C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary.
# @REJECTED: Distributed actor model (Celery) — eventual-consistency challenges at current scale.
# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
import json
import time
@@ -26,7 +26,6 @@ from typing import Any, Dict, List, Optional, Callable, Tuple
from sqlalchemy.orm import Session
from ...core.logger import logger, belief_scope
from ...core.cot_logger import MarkerLogger
from ...core.config_manager import ConfigManager
from ...models.translate import (
TranslationJob,
@@ -42,15 +41,13 @@ from .superset_executor import SupersetSqlLabExecutor
from .events import TranslationEventLog
from ..translate.service import TranslateJobService
log = MarkerLogger("TranslationOrchestrator")
# #region TranslationOrchestrator [C:5] [TYPE Class] [SEMANTICS translate,orchestrator]
# #region TranslationOrchestrator [C:5] [TYPE Class]
# @BRIEF Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging.
# @PRE DB session and config manager are available.
# @POST Runs are created, executed, and finalized with event records.
# @SIDE_EFFECT Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows.
# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED.
# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun]
# @PRE: DB session and config manager are available.
# @POST: Runs are created, executed, and finalized with event records.
# @SIDE_EFFECT: Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows.
# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED.
class TranslationOrchestrator:
def __init__(
@@ -65,11 +62,12 @@ class TranslationOrchestrator:
self.event_log = TranslationEventLog(db)
self._job: Optional[TranslationJob] = None
# #region start_run [C:5] [TYPE Function] [SEMANTICS translate,run,create]
# @BRIEF Start a new translation run for a job with config snapshot and hash computation.
# @PRE job_id exists. For manual runs, there must be an accepted preview session.
# @POST TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.
# @SIDE_EFFECT DB writes; event logging.
# [DEF:start_run:Function]
# @COMPLEXITY: 5
# @PURPOSE: Start a new translation run for a job.
# @PRE: job_id exists. For manual runs, there must be an accepted preview session.
# @POST: TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.
# @SIDE_EFFECT: DB writes.
def start_run(
self,
job_id: str,
@@ -83,7 +81,7 @@ class TranslationOrchestrator:
raise ValueError(f"Translation job '{job_id}' not found")
self._job = job
log.reason("Starting translation run", payload={
logger.reason("Starting translation run", {
"job_id": job_id,
"is_scheduled": is_scheduled,
})
@@ -158,26 +156,25 @@ class TranslationOrchestrator:
self.db.commit()
self.db.refresh(run)
log.reflect("Run created", payload={
logger.reflect("Run created", {
"run_id": run.id,
"job_id": job_id,
"status": run.status,
"trigger_type": run.trigger_type,
})
return run
# #endregion start_run
# [/DEF:start_run:Function]
# #region execute_run [C:5] [TYPE Function] [SEMANTICS translate,run,execute]
# @BRIEF Execute a translation run: dispatch executor, generate SQL, submit to Superset.
# @PRE run is in PENDING status.
# @POST Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
# [DEF:execute_run:Function]
# @PURPOSE: Execute a translation run: dispatch executor, generate SQL, submit to Superset.
# @PRE: run is in PENDING status.
# @POST: Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT: LLM calls, DB writes, Superset API calls.
def execute_run(
self,
run: TranslationRun,
on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None,
skip_insert: bool = False,
full_translation: bool = False,
) -> TranslationRun:
with belief_scope("TranslationOrchestrator.execute_run"):
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
@@ -191,7 +188,7 @@ class TranslationOrchestrator:
f"Run must be in PENDING status."
)
log.reason("Executing run", payload={
logger.reason("Executing run", {
"run_id": run.id,
"job_id": job.id,
"skip_insert": skip_insert,
@@ -212,10 +209,11 @@ class TranslationOrchestrator:
on_batch_progress=on_batch_progress,
)
try:
run = executor.execute_run(run, llm_progress_callback=None, full_translation=full_translation)
run = executor.execute_run(run, llm_progress_callback=None)
except Exception as e:
log.explore("Translation execution failed", error=str(e), payload={
logger.explore("Translation execution failed", {
"run_id": run.id,
"error": str(e),
})
run.status = "FAILED"
run.error_message = f"Translation execution failed: {e}"
@@ -292,19 +290,19 @@ class TranslationOrchestrator:
self.db.commit()
self.db.refresh(run)
log.reflect("Run execution complete", payload={
logger.reflect("Run execution complete", {
"run_id": run.id,
"status": run.status,
"insert_status": run.insert_status,
})
return run
# #endregion execute_run
# [/DEF:execute_run:Function]
# #region _generate_and_insert_sql [C:5] [TYPE Function] [SEMANTICS translate,sql,insert]
# @BRIEF Generate INSERT SQL from successful records and submit to Superset SQL Lab.
# @PRE job has target table configured. run has successful records.
# @POST SQL is generated and submitted. Returns execution result.
# @SIDE_EFFECT Superset API call; event logging.
# [DEF:_generate_and_insert_sql:Function]
# @PURPOSE: Generate INSERT SQL from successful records and submit to Superset SQL Lab.
# @PRE: job has target table configured. run has successful records.
# @POST: SQL is generated and submitted. Returns execution result.
# @SIDE_EFFECT: Superset API call.
def _generate_and_insert_sql(
self,
job: TranslationJob,
@@ -323,21 +321,20 @@ class TranslationOrchestrator:
)
if not records:
log.reason("No successful records to insert", payload={"run_id": run.id})
logger.reason("No successful records to insert", {"run_id": run.id})
return {"status": "skipped", "reason": "no_records", "query_id": None}
log.reason(f"Generating SQL for {len(records)} records", payload={
logger.reason(f"Generating SQL for {len(records)} records", {
"run_id": run.id,
"dialect": job.database_dialect or job.target_dialect,
})
# Build rows for SQL generation
# Only include the translation column — context columns are for LLM only
columns = []
if job.translation_column:
columns = job.context_columns or []
if job.translation_column and job.translation_column not in columns:
columns.append(job.translation_column)
# Include key columns if present (for UPSERT matching)
# Also include key columns if used for upsert
if job.target_key_cols:
for k in job.target_key_cols:
if k not in columns:
@@ -346,6 +343,9 @@ class TranslationOrchestrator:
rows_for_sql = []
for rec in records:
row_data = {}
if job.context_columns:
for col in job.context_columns:
row_data[col] = ""
if job.translation_column:
row_data[job.translation_column] = rec.target_sql or ""
if job.target_key_cols:
@@ -372,20 +372,10 @@ class TranslationOrchestrator:
upsert_strategy=job.upsert_strategy or "MERGE",
)
except ValueError as e:
log.explore("SQL generation failed", error=str(e))
logger.explore("SQL generation failed", {"error": str(e)})
return {"status": "failed", "error_message": str(e), "query_id": None}
# Log insert phase start
log.reason("Insert SQL generated", payload={
"run_id": run.id,
"sql_preview": sql[:500],
"sql_length": len(sql),
"row_count": row_count,
"dialect": job.database_dialect or job.target_dialect or "postgresql",
"columns": columns,
"has_key_cols": bool(job.target_key_cols),
})
self.event_log.log_event(
job_id=job.id,
run_id=run.id,
@@ -397,26 +387,16 @@ class TranslationOrchestrator:
# Submit to Superset
try:
env_id = job.environment_id or job.source_dialect or ""
# Resolve database_id: explicit target_database_id from job (int or UUID), or resolve from environment
target_db_id = None
if job.target_database_id:
try:
target_db_id = int(job.target_database_id)
except (ValueError, TypeError):
# Could be a UUID — pass as-is, executor will resolve it
target_db_id = job.target_database_id
log.reason("target_database_id is not an integer, will resolve as UUID", payload={
"target_database_id": job.target_database_id,
})
executor = SupersetSqlLabExecutor(self.config_manager, env_id, database_id=target_db_id)
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
result = executor.execute_and_poll(
sql=sql,
max_polls=30,
poll_interval_seconds=2.0,
)
except Exception as e:
log.explore("Superset SQL submission failed", error=str(e), payload={
logger.explore("Superset SQL submission failed", {
"run_id": run.id,
"error": str(e),
})
result = {"status": "failed", "error_message": str(e), "query_id": None}
@@ -430,13 +410,13 @@ class TranslationOrchestrator:
)
return result
# #endregion _generate_and_insert_sql
# [/DEF:_generate_and_insert_sql:Function]
# #region _validate_preconditions [C:5] [TYPE Function] [SEMANTICS translate,validation]
# @BRIEF Validate preconditions before starting a translation run.
# @PRE None.
# @POST Raises ValueError if preconditions are not met.
# @SIDE_EFFECT None.
# [DEF:_validate_preconditions:Function]
# @PURPOSE: Validate preconditions before starting a run.
# @PRE: None.
# @POST: Raises ValueError if preconditions are not met.
# @SIDE_EFFECT: None.
def _validate_preconditions(
self,
job: TranslationJob,
@@ -488,17 +468,17 @@ class TranslationOrchestrator:
"Run and accept a preview before executing a manual translation run."
)
log.reason("Preconditions validated", payload={
logger.reason("Preconditions validated", {
"job_id": job.id,
"is_scheduled": is_scheduled,
})
# #endregion _validate_preconditions
# [/DEF:_validate_preconditions:Function]
# #region retry_failed_batches [C:5] [TYPE Function] [SEMANTICS translate,retry,batch]
# @BRIEF Retry failed batches in a run by re-processing failed records.
# @PRE run exists and has failed batches.
# @POST Failed batches are re-processed; run status and stats updated.
# @SIDE_EFFECT LLM calls; DB writes; event logging.
# [DEF:retry_failed_batches:Function]
# @PURPOSE: Retry failed batches in a run.
# @PRE: run exists and has failed batches.
# @POST: Failed batches are re-processed.
# @SIDE_EFFECT: LLM calls, DB writes.
def retry_failed_batches(
self,
run_id: str,
@@ -526,7 +506,7 @@ class TranslationOrchestrator:
if not failed_batches:
raise ValueError(f"No failed batches found for run '{run_id}'")
log.reason("Retrying failed batches", payload={
logger.reason("Retrying failed batches", {
"run_id": run_id,
"batch_count": len(failed_batches),
})
@@ -602,18 +582,18 @@ class TranslationOrchestrator:
)
self.db.commit()
log.reflect("Retry complete", payload={
logger.reflect("Retry complete", {
"run_id": run_id,
"status": run.status,
})
return run
# #endregion retry_failed_batches
# [/DEF:retry_failed_batches:Function]
# #region retry_insert [C:5] [TYPE Function] [SEMANTICS translate,retry,insert]
# @BRIEF Retry the SQL insert phase for a completed run.
# @PRE run exists and has successful records.
# @POST SQL is regenerated and re-submitted to Superset.
# @SIDE_EFFECT Superset API call; event logging.
# [DEF:retry_insert:Function]
# @PURPOSE: Retry the SQL insert phase for a completed run.
# @PRE: run exists and has successful records.
# @POST: SQL is regenerated and re-submitted to Superset.
# @SIDE_EFFECT: Superset API call.
def retry_insert(
self,
run_id: str,
@@ -628,7 +608,7 @@ class TranslationOrchestrator:
raise ValueError(f"Job '{run.job_id}' not found")
self._job = job
log.reason("Retrying insert phase", payload={
logger.reason("Retrying insert phase", {
"run_id": run_id,
})
@@ -652,18 +632,18 @@ class TranslationOrchestrator:
self.db.commit()
self.db.refresh(run)
log.reflect("Insert retry complete", payload={
logger.reflect("Insert retry complete", {
"run_id": run_id,
"insert_status": run.insert_status,
})
return run
# #endregion retry_insert
# [/DEF:retry_insert:Function]
# #region cancel_run [C:5] [TYPE Function] [SEMANTICS translate,run,cancel]
# @BRIEF Cancel a running translation run.
# @PRE run is in PENDING or RUNNING status.
# @POST Run status is set to CANCELLED; event recorded.
# @SIDE_EFFECT DB write; records event.
# [DEF:cancel_run:Function]
# @PURPOSE: Cancel a running translation.
# @PRE: run is in PENDING or RUNNING status.
# @POST: Run status is set to CANCELLED.
# @SIDE_EFFECT: DB write; records event.
def cancel_run(self, run_id: str) -> TranslationRun:
with belief_scope("TranslationOrchestrator.cancel_run"):
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
@@ -680,8 +660,7 @@ class TranslationOrchestrator:
run.completed_at = datetime.now(timezone.utc)
self.db.flush()
# Record terminal event directly — we are the source of the terminal event
self.event_log._create_event_raw(
self.event_log.log_event(
job_id=run.job_id,
run_id=run.id,
event_type="RUN_CANCELLED",
@@ -692,16 +671,16 @@ class TranslationOrchestrator:
self.db.commit()
self.db.refresh(run)
log.reflect("Run cancelled", payload={
logger.reflect("Run cancelled", {
"run_id": run_id,
})
return run
# #endregion cancel_run
# [/DEF:cancel_run:Function]
# #region get_run_status [C:5] [TYPE Function] [SEMANTICS translate,run,status]
# @BRIEF Get run status with statistics and event invariant checks.
# @PRE run_id exists.
# @POST Returns dict with run details.
# [DEF:get_run_status:Function]
# @PURPOSE: Get run status with statistics.
# @PRE: run_id exists.
# @POST: Returns dict with run details.
def get_run_status(self, run_id: str) -> Dict[str, Any]:
with belief_scope("TranslationOrchestrator.get_run_status"):
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
@@ -715,8 +694,8 @@ class TranslationOrchestrator:
.count()
)
# Get event invariants (lightweight — only fetches event_type column)
invariants = self.event_log.get_run_event_invariants_lightweight(run_id)
# Get event summary
event_summary = self.event_log.get_run_event_summary(run_id)
return {
"id": run.id,
@@ -732,16 +711,20 @@ class TranslationOrchestrator:
"insert_status": run.insert_status,
"superset_execution_id": run.superset_execution_id,
"batch_count": batch_count,
"event_invariants": invariants,
"event_invariants": {
"has_run_started": event_summary["has_run_started"],
"terminal_event_count": event_summary["terminal_event_count"],
"invariant_valid": event_summary["invariant_valid"],
},
"created_by": run.created_by,
"created_at": run.created_at.isoformat() if run.created_at else None,
}
# #endregion get_run_status
# [/DEF:get_run_status:Function]
# #region get_run_records [C:5] [TYPE Function] [SEMANTICS translate,run,records]
# @BRIEF Get paginated records for a run with optional status filter.
# @PRE run_id exists.
# @POST Returns dict with records and pagination info.
# [DEF:get_run_records:Function]
# @PURPOSE: Get paginated records for a run.
# @PRE: run_id exists.
# @POST: Returns dict with records and pagination info.
def get_run_records(
self,
run_id: str,
@@ -787,12 +770,12 @@ class TranslationOrchestrator:
"page_size": page_size,
"status_filter": status_filter,
}
# #endregion get_run_records
# [/DEF:get_run_records:Function]
# #region get_run_history [C:5] [TYPE Function] [SEMANTICS translate,run,history]
# @BRIEF Get paginated run history for a job.
# @PRE job_id exists.
# @POST Returns tuple of (total_count, list_of_runs).
# [DEF:get_run_history:Function]
# @PURPOSE: Get run history for a job.
# @PRE: job_id exists.
# @POST: Returns list of runs.
def get_run_history(
self,
job_id: str,
@@ -830,10 +813,10 @@ class TranslationOrchestrator:
}
for r in runs
]
# #endregion get_run_history
# [/DEF:get_run_history:Function]
# #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison.
# [DEF:_compute_config_hash:Function]
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
@staticmethod
def _compute_config_hash(job: TranslationJob) -> str:
import hashlib
@@ -849,10 +832,10 @@ class TranslationOrchestrator:
"upsert_strategy": job.upsert_strategy,
}, sort_keys=True)
return hashlib.sha256(config_str.encode()).hexdigest()[:16]
# #endregion _compute_config_hash
# [/DEF:_compute_config_hash:Function]
# #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @BRIEF Compute a SHA-256 hash of dictionary state for snapshot comparison.
# [DEF:_compute_dict_snapshot_hash:Function]
# @PURPOSE: Compute a hash of dictionary state for snapshot comparison.
def _compute_dict_snapshot_hash(self, job_id: str) -> str:
import hashlib
from ...models.translate import TranslationJobDictionary
@@ -864,7 +847,7 @@ class TranslationOrchestrator:
dict_ids = sorted([dl.dictionary_id for dl in dict_links])
hash_input = ",".join(dict_ids)
return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
# #endregion _compute_dict_snapshot_hash
# [/DEF:_compute_dict_snapshot_hash:Function]
# #endregion TranslationOrchestrator

10
backend/src/plugins/translate/plugin.py
View File

@@ -1,15 +1,17 @@
# #region TranslatePlugin [C:3] [TYPE Module] [SEMANTICS plugin,translate,llm,sql,dashboard]
# #region TranslatePlugin [C:2] [TYPE Module] [SEMANTICS plugin, translate, llm, sql, dashboard]
# @BRIEF TranslatePlugin skeleton for cross-dialect SQL and dashboard translation.
# @LAYER Domain
# @LAYER: Domain
# @RELATION INHERITS -> [PluginBase]
# @RATIONALE: Separate plugin avoids bloating LLMAnalysisPlugin beyond fractal limit (<400 lines).
# @REJECTED: Extending LLMAnalysisPlugin would conflate two distinct feature domains.
from typing import Dict, Any, Optional
from ...core.plugin_base import PluginBase
# #region TranslatePlugin [C:3] [TYPE Class] [SEMANTICS plugin,translate]
# #region TranslatePlugin [TYPE Class]
# @BRIEF Plugin for translating SQL queries and dashboard definitions across database dialects.
# @RELATION IMPLEMENTS -> [PluginBase]
# @RELATION IMPLEMENTS -> backend.src.core.plugin_base.PluginBase
class TranslatePlugin(PluginBase):
@property
def id(self) -> str:

242
backend/src/plugins/translate/preview.py
View File

@@ -1,21 +1,19 @@
# #region TranslationPreview [C:5] [TYPE Module] [SEMANTICS translate,preview,llm,session]
# #region TranslationPreview [C:4] [TYPE Module] [SEMANTICS translate, preview, llm, session]
# @BRIEF Preview session management: fetch sample rows from Superset, send to LLM with context + filtered dictionary, return side-by-side results.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION DEPENDS_ON -> [TranslationPreviewSession]
# @RELATION DEPENDS_ON -> [TranslationPreviewRecord]
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationJob:Class]
# @RELATION DEPENDS_ON -> [TranslationPreviewSession:Class]
# @RELATION DEPENDS_ON -> [TranslationPreviewRecord:Class]
# @RELATION DEPENDS_ON -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [DictionaryManager]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [render_prompt]
# @RELATION DEPENDS_ON -> [ConfigManager]
# @PRE Database session and config manager are available.
# @POST Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected.
# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates DB rows.
# @DATA_CONTRACT Input[job_id, sample_size] -> Output[Dict with session, records, cost_estimate]
# @INVARIANT Preview sessions expire after 24 hours; exactly one ACTIVE session per job at a time.
# @RATIONALE C5 because preview is stateful with approve/edit/reject lifecycle and LLM API calls requiring structured error handling.
# @REJECTED Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably.
# @PRE: Database session and config manager are available.
# @POST: Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected.
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows.
# @RATIONALE: C4 because preview is stateful with approve/edit/reject lifecycle and LLM API calls.
# @REJECTED: Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably.
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.orm import Session
@@ -25,8 +23,7 @@ import json
import hashlib
import re
from ...core.cot_logger import MarkerLogger
from ...core.logger import belief_scope
from ...core.logger import logger, belief_scope
from ...core.config_manager import ConfigManager
from ...core.superset_client import SupersetClient
from ...models.translate import (
@@ -39,9 +36,8 @@ from ...services.llm_provider import LLMProviderService
from ...services.llm_prompt_templates import render_prompt
from .dictionary import DictionaryManager
log = MarkerLogger("TranslationPreview")
# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [C:1] [TYPE Constant] [SEMANTICS translate,prompt]
# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [TYPE Constant]
# @BRIEF Default prompt template for batch LLM translation execution (no context columns — faster).
DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
"Translate the following database content from {source_language} to {target_language}.\n\n"
"Source dialect: {source_dialect}\n"
@@ -57,7 +53,8 @@ DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
# #endregion DEFAULT_EXECUTION_PROMPT_TEMPLATE
# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [C:1] [TYPE Constant] [SEMANTICS translate,prompt]
# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [TYPE Constant]
# @BRIEF Default prompt template for LLM translation preview.
DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
"Translate the following database content from {source_language} to {target_language}.\n\n"
"Source dialect: {source_dialect}\n"
@@ -74,8 +71,10 @@ DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
# #endregion DEFAULT_PREVIEW_PROMPT_TEMPLATE
# #region TokenEstimator [C:2] [TYPE Class] [SEMANTICS translate,tokens,estimate]
# @BRIEF Estimate token counts and costs for LLM translation operations using heuristic chars/token ratio.
# #region TokenEstimator [TYPE Class]
# @BRIEF Estimate token counts and costs for LLM translation operations.
# @RATIONALE: Token estimation uses a heuristic (chars/token ratio) since exact tokenization depends on the LLM model.
# @REJECTED: Using an external tokenizer library would introduce a heavy dependency for estimation only.
class TokenEstimator:
"""Estimate token counts and costs for LLM operations."""
@@ -83,38 +82,45 @@ class TokenEstimator:
OUTPUT_TOKENS_PER_ROW_ESTIMATE: int = 50
TOKEN_COST_PER_1K: float = 0.002 # Default cost per 1K tokens
# #region estimate_prompt_tokens [C:1] [TYPE Function] [SEMANTICS translate,tokens]
# [DEF:estimate_prompt_tokens:Function]
# @PURPOSE: Estimate token count for a prompt string.
# @PRE: prompt is a non-empty string.
# @POST: Returns estimated token count (integer).
@staticmethod
def estimate_prompt_tokens(prompt: str) -> int:
if not prompt:
return 0
return max(1, int(len(prompt) / TokenEstimator.CHARS_PER_TOKEN_ESTIMATE))
# #endregion estimate_prompt_tokens
# [/DEF:estimate_prompt_tokens:Function]
# #region estimate_output_tokens [C:1] [TYPE Function] [SEMANTICS translate,tokens]
# [DEF:estimate_output_tokens:Function]
# @PURPOSE: Estimate output token count for translating N rows.
# @PRE: row_count >= 0.
# @POST: Returns estimated output token count.
@staticmethod
def estimate_output_tokens(row_count: int) -> int:
return row_count * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE
# #endregion estimate_output_tokens
# [/DEF:estimate_output_tokens:Function]
# #region estimate_cost [C:1] [TYPE Function] [SEMANTICS translate,cost]
# [DEF:estimate_cost:Function]
# @PURPOSE: Estimate cost for a given number of tokens.
# @PRE: total_tokens >= 0.
# @POST: Returns estimated cost in USD.
@staticmethod
def estimate_cost(total_tokens: int, cost_per_1k: Optional[float] = None) -> float:
rate = cost_per_1k if cost_per_1k is not None else TokenEstimator.TOKEN_COST_PER_1K
return round((total_tokens / 1000) * rate, 6)
# #endregion estimate_cost
# [/DEF:estimate_cost:Function]
# #endregion TokenEstimator
# #region TranslationPreview [C:5] [TYPE Class] [SEMANTICS translate,preview,lifecycle]
# #region TranslationPreview [C:4] [TYPE Class]
# @BRIEF Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate.
# @PRE Database session and config manager are available.
# @POST Preview sessions created with persisted records; full execution gates on accepted session.
# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates DB rows.
# @DATA_CONTRACT Input[job_id, sample_size] -> Output[Dict with session, records, cost_estimate]
# @INVARIANT Preview sessions expire after 24 hours; exactly one ACTIVE session per job at a time.
# @PRE: Database session and config manager are available.
# @POST: Preview sessions created with persisted records; full execution gates on accepted session.
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows.
class TranslationPreview:
def __init__(
@@ -127,11 +133,12 @@ class TranslationPreview:
self.config_manager = config_manager
self.current_user = current_user
# #region preview_rows [C:4] [TYPE Function] [SEMANTICS translate,preview,rows]
# @BRIEF Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.
# @PRE job_id exists and job has source_datasource_id, translation_column configured.
# @POST Returns TranslationPreviewResponse with records, cost estimation, and persistent session.
# @SIDE_EFFECT Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.
# [DEF:preview_rows:Function]
# @COMPLEXITY: 4
# @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.
# @PRE: job_id exists and job has source_datasource_id, translation_column configured.
# @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session.
# @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.
def preview_rows(
self,
job_id: str,
@@ -140,7 +147,7 @@ class TranslationPreview:
env_id: Optional[str] = None,
) -> Dict[str, Any]:
with belief_scope("TranslationPreview.preview_rows"):
log.reason("Starting preview for job", payload={"job_id": job_id, "sample_size": sample_size})
logger.reason("Starting preview for job", {"job_id": job_id, "sample_size": sample_size})
# 1. Load job
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
@@ -156,7 +163,7 @@ class TranslationPreview:
dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id)
# 3. Fetch sample rows from Superset
log.reason("Fetching sample rows from Superset", payload={
logger.reason("Fetching sample rows from Superset", {
"datasource_id": job.source_datasource_id,
"sample_size": sample_size,
"translation_column": job.translation_column,
@@ -170,12 +177,12 @@ class TranslationPreview:
raise ValueError("No rows returned from datasource for preview")
actual_row_count = len(source_rows)
log.reason(f"Fetched {actual_row_count} sample row(s)")
logger.reason(f"Fetched {actual_row_count} sample row(s)")
# Debug: log first row keys and translation column value
if source_rows:
first_row = source_rows[0]
log.reason(
logger.reason(
f"First source row keys={list(first_row.keys())} "
f"translation_col={job.translation_column} "
f"val='{first_row.get(job.translation_column, '')}'"
@@ -249,7 +256,7 @@ class TranslationPreview:
total_est_cost = TokenEstimator.estimate_cost(total_est_tokens)
# 8. Call LLM
log.reason("Calling LLM for preview translation", payload={
logger.reason("Calling LLM for preview translation", {
"provider_id": job.provider_id,
"row_count": actual_row_count,
"estimated_tokens": sample_total_tokens,
@@ -293,7 +300,6 @@ class TranslationPreview:
source_object_name=f"Row {idx + 1}",
status=status,
feedback=feedback,
source_data=meta.get("context_data"),
created_at=datetime.now(timezone.utc),
)
self.db.add(record)
@@ -333,19 +339,18 @@ class TranslationPreview:
"dict_snapshot_hash": dict_snapshot_hash,
}
log.reflect("Preview completed", payload={
logger.reflect("Preview completed", {
"session_id": session.id,
"row_count": actual_row_count,
"sample_cost": sample_cost,
})
return result
# #endregion preview_rows
# [/DEF:preview_rows:Function]
# #region update_preview_row [C:4] [TYPE Function] [SEMANTICS translate,preview,row,update]
# @BRIEF Approve, edit, or reject an individual preview row.
# @PRE session_id and row_id exist, session is ACTIVE.
# @POST PreviewRecord status is updated (APPROVED, REJECTED, or APPROVED with edited translation).
# @SIDE_EFFECT DB write.
# [DEF:update_preview_row:Function]
# @PURPOSE: Approve, edit, or reject an individual preview row.
# @PRE: session_id and row_id exist, session is ACTIVE.
# @POST: PreviewRecord status is updated.
def update_preview_row(
self,
job_id: str,
@@ -396,7 +401,7 @@ class TranslationPreview:
self.db.commit()
self.db.refresh(record)
log.reason(f"Preview row {action}d", payload={
logger.reason(f"Preview row {action}d", {
"row_id": row_id,
"session_id": session.id,
"status": record.status,
@@ -409,13 +414,13 @@ class TranslationPreview:
"status": record.status,
"feedback": record.feedback,
}
# #endregion update_preview_row
# [/DEF:update_preview_row:Function]
# #region accept_preview_session [C:4] [TYPE Function] [SEMANTICS translate,preview,accept]
# @BRIEF Mark a preview session as accepted (APPLIED), which gates full execution.
# @PRE job_id has an ACTIVE preview session.
# @POST Session status changes to APPLIED; future full execution calls check for accepted session.
# @SIDE_EFFECT DB write.
# [DEF:accept_preview_session:Function]
# @PURPOSE: Mark a preview session as accepted, which gates full execution.
# @PRE: job_id has an ACTIVE preview session.
# @POST: Session status changes to APPLIED.
# @SIDE_EFFECT: Future full execution calls will check for accepted session.
def accept_preview_session(self, job_id: str) -> Dict[str, Any]:
with belief_scope("TranslationPreview.accept_preview_session"):
session = (
@@ -434,7 +439,7 @@ class TranslationPreview:
self.db.commit()
self.db.refresh(session)
log.reason("Preview session accepted", payload={
logger.reason("Preview session accepted", {
"session_id": session.id,
"job_id": job_id,
})
@@ -464,11 +469,12 @@ class TranslationPreview:
for r in records
],
}
# #endregion accept_preview_session
# [/DEF:accept_preview_session:Function]
# #region get_preview_session [C:3] [TYPE Function] [SEMANTICS translate,preview,get]
# @BRIEF Get the latest preview session for a job with its records.
# @RELATION DEPENDS_ON -> [TranslationPreviewSession]
# [DEF:get_preview_session:Function]
# @PURPOSE: Get the latest preview session for a job with its records.
# @PRE: job_id exists.
# @POST: Returns session data with records or raises ValueError.
def get_preview_session(self, job_id: str) -> Dict[str, Any]:
with belief_scope("TranslationPreview.get_preview_session"):
session = (
@@ -507,13 +513,14 @@ class TranslationPreview:
for r in records
],
}
# #endregion get_preview_session
# [/DEF:get_preview_session:Function]
# #region _fetch_sample_rows [C:4] [TYPE Function] [SEMANTICS translate,superset,sample]
# @BRIEF Fetch sample rows from the Superset dataset for preview.
# @PRE job has source_datasource_id and translation_column.
# @POST Returns list of dicts with row data from Superset chart data API.
# @SIDE_EFFECT Calls Superset chart data endpoint with pagination.
# [DEF:_fetch_sample_rows:Function]
# @COMPLEXITY: 4
# @PURPOSE: Fetch sample rows from the Superset dataset for preview.
# @PRE: job has source_datasource_id and translation_column.
# @POST: Returns list of dicts with row data.
# @SIDE_EFFECT: Calls Superset chart data endpoint.
def _fetch_sample_rows(self, job: TranslationJob, sample_size: int = 10, env_id: Optional[str] = None) -> List[Dict[str, Any]]:
with belief_scope("TranslationPreview._fetch_sample_rows"):
# Determine environment: prefer explicit env_id, then job.environment_id, then job.source_dialect (legacy)
@@ -524,15 +531,13 @@ class TranslationPreview:
None,
)
if not env_config:
log.explore("Could not find environment for datasource", error="Environment not found for datasource",
payload={
logger.explore("Could not find environment for datasource", {
"env_id": target_env_id,
})
# Fallback: try first environment
if environments:
env_config = environments[0]
log.explore("Falling back to first available environment", error="Environment fallback used",
payload={
logger.explore("Falling back to first available environment", {
"env_id": env_config.id,
})
else:
@@ -577,28 +582,29 @@ class TranslationPreview:
headers={"Content-Type": "application/json"},
)
except Exception as e:
log.explore("Chart data API failed", error=str(e))
logger.explore("Chart data API failed", {"error": str(e)})
raise ValueError(f"Failed to fetch sample data from Superset: {e}")
# Parse response
rows = self._extract_data_rows(response)
log.reason(f"Extracted {len(rows)} data row(s)")
logger.reason(f"Extracted {len(rows)} data row(s)")
# Debug: log first row keys and translation column value
if rows:
first_row = rows[0]
log.reason(
logger.reason(
f"Row keys={list(first_row.keys())} "
f"target_col={job.translation_column} "
f"val='{first_row.get(job.translation_column, '')}'"
)
return rows
# #endregion _fetch_sample_rows
# [/DEF:_fetch_sample_rows:Function]
# #region _extract_data_rows [C:3] [TYPE Function] [SEMANTICS translate,superset,data]
# @BRIEF Extract data rows from Superset chart data response, trying multiple response formats.
# @RELATION DEPENDS_ON -> [SupersetClient]
# [DEF:_extract_data_rows:Function]
# @PURPOSE: Extract data rows from Superset chart data response.
# @PRE: response is a dict from Superset API.
# @POST: Returns list of row dicts.
@staticmethod
def _extract_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]:
with belief_scope("TranslationPreview._extract_data_rows"):
@@ -627,13 +633,14 @@ class TranslationPreview:
return result
return []
# #endregion _extract_data_rows
# [/DEF:_extract_data_rows:Function]
# #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call]
# @BRIEF Call the configured LLM provider with the preview prompt.
# @PRE job has a valid provider_id.
# @POST Returns raw LLM response string.
# @SIDE_EFFECT Makes HTTP call to LLM provider.
# [DEF:_call_llm:Function]
# @COMPLEXITY: 4
# @PURPOSE: Call the configured LLM provider with the preview prompt.
# @PRE: job has a valid provider_id.
# @POST: Returns raw LLM response string.
# @SIDE_EFFECT: Makes HTTP call to LLM provider.
def _call_llm(self, job: TranslationJob, prompt: str) -> str:
with belief_scope("TranslationPreview._call_llm"):
if not job.provider_id:
@@ -663,19 +670,19 @@ class TranslationPreview:
else:
raise ValueError(f"Unsupported provider type '{provider_type}' for preview")
log.reason("LLM call completed", payload={
logger.reason("LLM call completed", {
"provider_id": job.provider_id,
"model": model,
"response_length": len(response_text),
})
return response_text
# #endregion _call_llm
# [/DEF:_call_llm:Function]
# #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai]
# @BRIEF Call an OpenAI-compatible API for translation with structured output support.
# @PRE base_url, api_key, model, prompt are valid.
# @POST Returns response text.
# @SIDE_EFFECT Makes HTTP POST to LLM API.
# [DEF:_call_openai_compatible:Function]
# @PURPOSE: Call an OpenAI-compatible API for translation.
# @PRE: base_url, api_key, model, prompt are valid.
# @POST: Returns response text.
# @SIDE_EFFECT: Makes HTTP POST to LLM API.
@staticmethod
def _call_openai_compatible(
base_url: str,
@@ -706,7 +713,7 @@ class TranslationPreview:
if provider_type in ("openai", "openai_compatible"):
payload["response_format"] = {"type": "json_object"}
log.reason(
logger.reason(
f"LLM request model={payload.get('model')} "
f"provider_type={provider_type} "
f"response_format={'yes' if 'response_format' in payload else 'no'} "
@@ -714,11 +721,11 @@ class TranslationPreview:
)
response = http_requests.post(url, headers=headers, json=payload, timeout=120)
if not response.ok:
log.explore("LLM API error", error=f"LLM API returned status {response.status_code}", payload={
"status_code": response.status_code,
"model": payload.get('model'),
"body": response.text[:2000],
})
logger.explore(
f"LLM API error status={response.status_code} "
f"model={payload.get('model')} "
f"body={response.text[:2000]}"
)
response.raise_for_status()
data = response.json()
@@ -731,17 +738,16 @@ class TranslationPreview:
raise ValueError("LLM returned empty content")
return content
# #endregion _call_openai_compatible
# [/DEF:_call_openai_compatible:Function]
# #region _parse_llm_response [C:4] [TYPE Function] [SEMANTICS translate,llm,parse]
# @BRIEF Parse the LLM JSON response into a dict of row_id -> translation, with markdown code block fallback.
# @PRE response_text is valid JSON with {"rows": [...]} structure.
# @POST Returns dict mapping string row_id to translation text; warns if fewer translations than expected.
# @SIDE_EFFECT Logs warnings for short responses.
# [DEF:_parse_llm_response:Function]
# @PURPOSE: Parse the LLM JSON response into a dict of row_id -> translation.
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
# @POST: Returns dict mapping string row_id to translation text.
@staticmethod
def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:
with belief_scope("TranslationPreview._parse_llm_response"):
log.reason(f"Raw LLM response length={len(response_text)} preview={response_text[:500]}")
logger.reason(f"Raw LLM response length={len(response_text)} preview={response_text[:500]}")
try:
data = json.loads(response_text)
@@ -759,7 +765,7 @@ class TranslationPreview:
rows = data.get("rows", [])
if not isinstance(rows, list):
log.explore("LLM response has no 'rows' array", error="LLM response missing 'rows' key")
logger.explore(f"LLM response has no 'rows' array, keys={list(data.keys())} text_preview={response_text[:300]}")
raise ValueError("LLM response missing 'rows' array")
translations: Dict[str, str] = {}
@@ -770,17 +776,16 @@ class TranslationPreview:
translations[row_id] = translation
if len(translations) < expected_count:
log.explore("LLM returned fewer translations than expected", error=f"Expected {expected_count} translations, got {len(translations)}", payload={
"expected_count": expected_count,
"actual_count": len(translations),
"response_preview": response_text[:600],
})
logger.explore(
f"LLM returned fewer translations expected={expected_count} "
f"got={len(translations)} response_preview={response_text[:600]}"
)
return translations
# #endregion _parse_llm_response
# [/DEF:_parse_llm_response:Function]
# #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison.
# [DEF:_compute_config_hash:Function]
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
@staticmethod
def _compute_config_hash(job: TranslationJob) -> str:
config_str = json.dumps({
@@ -795,10 +800,10 @@ class TranslationPreview:
"upsert_strategy": job.upsert_strategy,
}, sort_keys=True)
return hashlib.sha256(config_str.encode()).hexdigest()[:16]
# #endregion _compute_config_hash
# [/DEF:_compute_config_hash:Function]
# #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]
# @BRIEF Compute a SHA-256 hash of the dictionary state for snapshot comparison.
# [DEF:_compute_dict_snapshot_hash:Function]
# @PURPOSE: Compute a hash of the dictionary state for snapshot comparison.
def _compute_dict_snapshot_hash(self, job_id: str) -> str:
dict_links = (
self.db.query(TranslationJobDictionary)
@@ -808,8 +813,9 @@ class TranslationPreview:
dict_ids = sorted([dl.dictionary_id for dl in dict_links])
hash_input = ",".join(dict_ids)
return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
# #endregion _compute_dict_snapshot_hash
# [/DEF:_compute_dict_snapshot_hash:Function]
# #endregion TranslationPreview
# #endregion TranslationPreview

150
backend/src/plugins/translate/scheduler.py
View File

@@ -1,18 +1,16 @@
# #region TranslationScheduler [C:5] [TYPE Module] [SEMANTICS translate,scheduler,apscheduler,cron]
# @BRIEF Manage TranslationSchedule rows and register them with core SchedulerService for cron-based execution.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationSchedule]
# @RELATION DEPENDS_ON -> [SchedulerService]
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
# @RELATION DEPENDS_ON -> [TranslationEventLog]
# @PRE Database session and SchedulerService are available.
# @POST TranslationSchedule CRUD persisted; translation runs triggered on schedule.
# @SIDE_EFFECT Registers APScheduler jobs; runs translations on trigger; creates events.
# @DATA_CONTRACT Input[job_id, cron_expression] -> Output[TranslationSchedule]
# @INVARIANT Concurrency guard: max 1 pending/running run per job at scheduled trigger time.
# @RATIONALE Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance.
# @REJECTED Separate scheduler instance would create resource contention.
# @REJECTED Polling-based approach — event-driven APScheduler is more precise.
# #region TranslationScheduler [C:4] [TYPE Module] [SEMANTICS translate, scheduler, apscheduler, cron]
# @BRIEF Manage TranslationSchedule rows and register them with core SchedulerService.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [SchedulerService:Class]
# @RELATION DEPENDS_ON -> [TranslationOrchestrator:Class]
# @RELATION DEPENDS_ON -> [TranslationEventLog:Class]
# @PRE: Database session and SchedulerService are available.
# @POST: TranslationSchedule CRUD persisted; APScheduler jobs registered/updated/removed.
# @SIDE_EFFECT: Registers APScheduler jobs; runs translations on trigger; creates events.
# @RATIONALE: Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance.
# @REJECTED: Separate scheduler instance would create resource contention.
# @REJECTED: Polling-based approach — event-driven APScheduler is more precise.
import uuid
from datetime import datetime, timezone, timedelta
@@ -20,22 +18,14 @@ from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from ...core.cot_logger import MarkerLogger
from ...core.logger import belief_scope
from ...core.logger import logger, belief_scope
from ...core.config_manager import ConfigManager
from ...models.translate import TranslationSchedule, TranslationJob, TranslationRun
from .events import TranslationEventLog
log = MarkerLogger("TranslationScheduler")
# #region TranslationScheduler [C:5] [TYPE Class] [SEMANTICS translate,scheduler,crud]
# #region TranslationScheduler [C:4] [TYPE Class]
# @BRIEF CRUD for TranslationSchedule rows + APScheduler registration wrappers.
# @PRE Database session and ConfigManager are available.
# @POST Schedule rows created/updated/deleted with event logging.
# @SIDE_EFFECT Writes TranslationSchedule rows; logs SCHEDULE_CREATED/UPDATED/DELETED events.
# @DATA_CONTRACT Input[job_id, cron_expression, timezone] -> Output[TranslationSchedule]
# @INVARIANT Exactly one schedule per job (upsert pattern).
class TranslationScheduler:
def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None):
@@ -44,11 +34,10 @@ class TranslationScheduler:
self.current_user = current_user
self.event_log = TranslationEventLog(db)
# #region create_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,create]
# @BRIEF Create a new schedule for a job with cron expression and event logging.
# @PRE job_id exists. cron_expression is valid.
# @POST TranslationSchedule row created; SCHEDULE_CREATED event logged.
# @SIDE_EFFECT DB write; event logging.
# [DEF:create_schedule:Function]
# @PURPOSE: Create a new schedule for a job.
# @PRE: job_id exists. cron_expression is valid.
# @POST: TranslationSchedule row created.
def create_schedule(
self,
job_id: str,
@@ -85,15 +74,14 @@ class TranslationScheduler:
created_by=self.current_user,
)
log.reflect("Schedule created", payload={"schedule_id": schedule.id, "job_id": job_id})
logger.reflect("Schedule created", {"schedule_id": schedule.id, "job_id": job_id})
return schedule
# #endregion create_schedule
# [/DEF:create_schedule:Function]
# #region update_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,update]
# @BRIEF Update an existing schedule's cron expression, timezone, or active state.
# @PRE job_id has an existing schedule.
# @POST Schedule updated; SCHEDULE_UPDATED event logged.
# @SIDE_EFFECT DB write; event logging.
# [DEF:update_schedule:Function]
# @PURPOSE: Update an existing schedule.
# @PRE: job_id has an existing schedule.
# @POST: Schedule updated.
def update_schedule(
self,
job_id: str,
@@ -130,15 +118,14 @@ class TranslationScheduler:
created_by=self.current_user,
)
log.reflect("Schedule updated", payload={"schedule_id": schedule.id, "job_id": job_id})
logger.reflect("Schedule updated", {"schedule_id": schedule.id, "job_id": job_id})
return schedule
# #endregion update_schedule
# [/DEF:update_schedule:Function]
# #region delete_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,delete]
# @BRIEF Delete a schedule for a job with event logging.
# @PRE job_id has an existing schedule.
# @POST Schedule deleted; SCHEDULE_DELETED event logged.
# @SIDE_EFFECT DB write; event logging.
# [DEF:delete_schedule:Function]
# @PURPOSE: Delete a schedule for a job.
# @PRE: job_id has an existing schedule.
# @POST: Schedule deleted.
def delete_schedule(self, job_id: str) -> None:
with belief_scope("TranslationScheduler.delete_schedule"):
schedule = self.db.query(TranslationSchedule).filter(
@@ -158,14 +145,13 @@ class TranslationScheduler:
created_by=self.current_user,
)
log.reflect("Schedule deleted", payload={"schedule_id": schedule_id, "job_id": job_id})
# #endregion delete_schedule
logger.reflect("Schedule deleted", {"schedule_id": schedule_id, "job_id": job_id})
# [/DEF:delete_schedule:Function]
# #region set_schedule_active [C:4] [TYPE Function] [SEMANTICS translate,schedule,activate]
# @BRIEF Enable or disable a schedule by setting is_active flag.
# @PRE job_id has an existing schedule.
# @POST Schedule is_active updated.
# @SIDE_EFFECT DB write.
# [DEF:enable_disable_schedule:Function]
# @PURPOSE: Enable or disable a schedule.
# @PRE: job_id has an existing schedule.
# @POST: Schedule is_active updated.
def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule:
with belief_scope("TranslationScheduler.set_schedule_active"):
schedule = self.db.query(TranslationSchedule).filter(
@@ -179,16 +165,18 @@ class TranslationScheduler:
self.db.commit()
self.db.refresh(schedule)
log.reflect("Schedule active state set", payload={
logger.reflect("Schedule active state set", {
"schedule_id": schedule.id,
"job_id": job_id,
"is_active": is_active,
})
return schedule
# #endregion set_schedule_active
# [/DEF:enable_disable_schedule:Function]
# #region get_schedule [C:2] [TYPE Function] [SEMANTICS translate,schedule,get]
# @BRIEF Get the schedule for a job by job_id.
# [DEF:get_schedule:Function]
# @PURPOSE: Get schedule for a job.
# @PRE: job_id exists.
# @POST: Returns TranslationSchedule or raises ValueError.
def get_schedule(self, job_id: str) -> TranslationSchedule:
with belief_scope("TranslationScheduler.get_schedule"):
schedule = self.db.query(TranslationSchedule).filter(
@@ -197,10 +185,11 @@ class TranslationScheduler:
if not schedule:
raise ValueError(f"No schedule found for job '{job_id}'")
return schedule
# #endregion get_schedule
# [/DEF:get_schedule:Function]
# #region list_active_schedules [C:2] [TYPE Function] [SEMANTICS translate,schedule,list]
# @BRIEF List all active schedules.
# [DEF:list_active_schedules:Function]
# @PURPOSE: List all active schedules.
# @POST: Returns list of active TranslationSchedule rows.
@staticmethod
def list_active_schedules(db: Session) -> List[TranslationSchedule]:
return (
@@ -208,10 +197,13 @@ class TranslationScheduler:
.filter(TranslationSchedule.is_active == True)
.all()
)
# #endregion list_active_schedules
# [/DEF:list_active_schedules:Function]
# #region get_next_executions [C:2] [TYPE Function] [SEMANTICS translate,schedule,next]
# @BRIEF Compute next N execution times from a cron expression using APScheduler's CronTrigger.
# [DEF:get_next_executions:Function]
# @PURPOSE: Compute next N execution times from cron expression.
# @PRE: cron_expression is valid.
# @POST: Returns list of ISO datetime strings.
@staticmethod
def get_next_executions(cron_expression: str, timezone_str: str = "UTC", n: int = 3) -> List[str]:
from zoneinfo import ZoneInfo
from apscheduler.triggers.cron import CronTrigger
@@ -221,7 +213,7 @@ class TranslationScheduler:
tz = ZoneInfo(timezone_str)
trigger = CronTrigger.from_crontab(cron_expression, timezone=tz)
except (ValueError, KeyError) as e:
log.explore("Invalid cron", error=str(e))
logger.warning(f"[get_next_executions] Invalid cron: {e}")
return []
now = datetime.now(tz)
@@ -236,20 +228,18 @@ class TranslationScheduler:
prev = ft
next_time = ft
return results
# #endregion get_next_executions
# [/DEF:get_next_executions:Function]
# #endregion TranslationScheduler
# #region execute_scheduled_translation [C:5] [TYPE Function] [SEMANTICS translate,schedule,execute]
# @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and baseline expiry detection.
# @PRE schedule_id is valid and job exists.
# @POST Translation run created and executed if no concurrent run exists.
# @SIDE_EFFECT DB writes; LLM calls; Superset API calls; event logging.
# @DATA_CONTRACT Input[schedule_id, job_id, db_session_maker, config_manager] -> Output[None]
# @INVARIANT Concurrency guard: max 1 pending/running run per job at trigger time.
# @RATIONALE New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.
# #region execute_scheduled_translation [TYPE Function]
# @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and new-key-only fallback.
# @PRE: schedule_id is valid.
# @POST: Translation run created and executed if no concurrent run exists.
# @SIDE_EFFECT: DB writes; LLM calls; Superset API calls.
# @RATIONALE: New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.
def execute_scheduled_translation(
schedule_id: str,
job_id: str,
@@ -266,10 +256,10 @@ def execute_scheduled_translation(
TranslationSchedule.job_id == job_id,
).first()
if not schedule or not schedule.is_active:
log.reason("Schedule inactive/missing", payload={"schedule_id": schedule_id})
logger.info(f"[scheduled_translation] Schedule inactive/missing: {schedule_id}")
return
log.reason("Scheduled translation triggered", payload={
logger.reason("Scheduled translation triggered", {
"schedule_id": schedule_id,
"job_id": job_id,
})
@@ -284,8 +274,7 @@ def execute_scheduled_translation(
.first()
)
if active_run:
log.explore("Skipping scheduled run — concurrent run in progress", error="Concurrent run detected",
payload={
logger.explore("Skipping scheduled run — concurrent run in progress", {
"job_id": job_id,
"existing_run_id": active_run.id,
})
@@ -312,7 +301,7 @@ def execute_scheduled_translation(
age = datetime.now(timezone.utc) - most_recent.created_at
if age > timedelta(days=90):
baseline_expired = True
log.reason("Baseline expired — full translation", payload={
logger.reason("Baseline expired — full translation", {
"job_id": job_id,
"last_run": most_recent.created_at.isoformat(),
"age_days": age.days,
@@ -343,7 +332,10 @@ def execute_scheduled_translation(
orch.execute_run(run)
run.insert_status = run.insert_status or "succeeded"
except Exception as exec_err:
log.explore("Scheduled translation execution failed", error=str(exec_err))
logger.explore("Scheduled translation execution failed", {
"run_id": run.id,
"error": str(exec_err),
})
# Leave schedule enabled — schedule continues on failure
run.status = "FAILED"
run.error_message = str(exec_err)
@@ -353,13 +345,13 @@ def execute_scheduled_translation(
schedule.last_run_at = datetime.now(timezone.utc)
db.commit()
log.reflect("Scheduled translation complete", payload={
logger.reflect("Scheduled translation complete", {
"schedule_id": schedule_id,
"run_id": run.id,
"status": run.status,
})
except Exception as e:
log.explore("Unexpected error", error=str(e))
logger.error(f"[scheduled_translation] Unexpected error: {e}")
finally:
db.close()
# #endregion execute_scheduled_translation

197
backend/src/plugins/translate/service.py
View File

@@ -1,16 +1,14 @@
# #region TranslateJobService [C:5] [TYPE Module] [SEMANTICS translate,service,crud,validation]
# #region TranslateJobService [C:4] [TYPE Module] [SEMANTICS translate, service, crud, validation]
# @BRIEF Service layer for translation job CRUD with datasource column validation and database dialect detection.
# @LAYER Domain
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [ConfigManager]
# @PRE Database session and config manager are available.
# @POST Translation jobs are created/updated/deleted with column validation and dialect caching.
# @SIDE_EFFECT Queries Superset for column metadata and database dialect at save time.
# @DATA_CONTRACT Input[TranslateJobCreate/Update] -> Output[TranslationJob/TranslateJobResponse]
# @INVARIANT Snapshot isolation: in-progress runs use config snapshot; config edits affect future runs only.
# @RATIONALE Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity.
# @PRE: Database session and config manager are available.
# @POST: Translation jobs are created/updated/deleted with column validation and dialect caching.
# @SIDE_EFFECT: Queries Superset for column metadata and database dialect at save time.
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity.
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.orm import Session
@@ -18,7 +16,6 @@ from datetime import datetime, timezone
import uuid
from ...core.logger import logger
from ...core.cot_logger import MarkerLogger
from ...core.config_manager import ConfigManager
from ...core.superset_client import SupersetClient
from ...models.translate import TranslationJob, TranslationJobDictionary
@@ -30,8 +27,6 @@ from ...schemas.translate import (
DatasourceColumnResponse,
)
log = MarkerLogger("TranslateJobService")
# Supported database dialects for translation
SUPPORTED_DIALECTS = {
"postgresql", "mysql", "clickhouse", "sqlite", "mssql",
@@ -40,16 +35,12 @@ SUPPORTED_DIALECTS = {
}
# #region get_dialect_from_database [C:4] [TYPE Function] [SEMANTICS translate,dialect,validation]
# @BRIEF Extract normalized dialect string from a Superset database record and validate it is supported.
# @PRE database_record is a dict from Superset API with 'backend' or 'engine' key.
# @POST Returns normalized dialect string or raises ValueError if unsupported.
# @SIDE_EFFECT None — pure data extraction.
# #region get_dialect_from_database [TYPE Function]
# @BRIEF Extract normalized dialect string from a Superset database record.
# @PRE: database_record is a dict from Superset API.
# @POST: Returns normalized dialect string or raises ValueError if unsupported.
def get_dialect_from_database(database_record: Dict[str, Any]) -> str:
"""Extract and validate dialect from Superset database record."""
log.reason("Extracting dialect from database record", payload={
"has_backend": bool(database_record.get("backend") or database_record.get("engine")),
})
backend = (
database_record.get("backend")
or database_record.get("engine")
@@ -57,9 +48,6 @@ def get_dialect_from_database(database_record: Dict[str, Any]) -> str:
).lower().strip()
if not backend:
log.explore("Could not determine database dialect", error="No backend/engine field in database record", payload={
"record_keys": list(database_record.keys()),
})
raise ValueError("Could not determine database dialect from connection")
# Map Superset backend names to normalized dialect
@@ -85,36 +73,28 @@ def get_dialect_from_database(database_record: Dict[str, Any]) -> str:
normalized = dialect_map.get(backend, backend)
if normalized not in SUPPORTED_DIALECTS:
log.explore("Unsupported dialect", payload={"backend": backend, "normalized": normalized}, error=f"Unsupported database dialect: {backend}")
raise ValueError(
f"Unsupported database dialect: '{backend}'. "
f"Supported dialects: {', '.join(sorted(SUPPORTED_DIALECTS))}"
)
log.reflect("Dialect validated", payload={"dialect": normalized})
return normalized
# #endregion get_dialect_from_database
# #region fetch_datasource_metadata [C:4] [TYPE Function] [SEMANTICS translate,datasource,metadata]
# @BRIEF Fetch datasource columns and database dialect from Superset for a given dataset.
# @PRE dataset_id is a valid Superset dataset ID and environment has valid credentials.
# @POST Returns (columns_list, dialect_string) or raises on failure.
# @SIDE_EFFECT Queries Superset API for dataset detail and database info.
# #region fetch_datasource_metadata [TYPE Function]
# @BRIEF Fetch datasource columns and database dialect from Superset.
# @PRE: dataset_id is a valid Superset dataset ID and environment has valid credentials.
# @POST: Returns (columns_list, dialect_string) or raises on failure.
def fetch_datasource_metadata(
dataset_id: int,
env_id: str,
config_manager: ConfigManager,
) -> Tuple[List[Dict[str, Any]], str]:
"""Fetch column metadata and database dialect for a datasource from Superset."""
log.reason("Fetching datasource metadata", payload={
"dataset_id": dataset_id,
"env_id": env_id,
})
# Find environment config
environments = config_manager.get_environments()
env_config = next((e for e in environments if e.id == env_id), None)
if not env_config:
log.explore("Environment not found", payload={"env_id": env_id}, error=f"Environment {env_id} not configured")
raise ValueError(f"Superset environment '{env_id}' not found in configuration")
# Create Superset client and fetch dataset detail
@@ -151,29 +131,25 @@ def fetch_datasource_metadata(
else:
raise ValueError("No database information available for this datasource")
except Exception as e:
log.explore("Could not fetch database dialect", error=str(e))
logger.warning(f"[translate_service] Could not fetch database dialect: {e}")
raise ValueError(f"Could not determine database dialect: {e}")
log.reflect("Metadata fetched", payload={"columns": len(columns), "dialect": dialect})
return columns, dialect
# #endregion fetch_datasource_metadata
# #region detect_virtual_columns [C:2] [TYPE Function] [SEMANTICS translate,columns]
# @BRIEF Identify virtual (calculated) columns from column metadata by filtering non-physical entries.
# #region detect_virtual_columns [TYPE Function]
# @BRIEF Identify virtual (calculated) columns from column metadata.
# @PRE: columns is a list of dicts with 'is_physical' key.
# @POST: Returns list of virtual column names.
def detect_virtual_columns(columns: List[Dict[str, Any]]) -> List[str]:
"""Return names of columns that are virtual (not physical)."""
return [col["name"] for col in columns if not col.get("is_physical", True)]
# #endregion detect_virtual_columns
# #region TranslateJobService [C:5] [TYPE Class] [SEMANTICS translate,service,crud]
# #region TranslateJobService [TYPE Class]
# @BRIEF Service for translation job CRUD with validation and Superset integration.
# @PRE Database session and config manager are available.
# @POST Translation jobs are managed with full lifecycle (create/update/delete/duplicate).
# @SIDE_EFFECT Queries Superset for dialect detection and column validation.
# @DATA_CONTRACT Input[TranslateJobCreate/Update] -> Output[TranslationJob]
# @INVARIANT Snapshot isolation: in-progress runs unaffected by config edits.
class TranslateJobService:
def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None):
@@ -181,8 +157,9 @@ class TranslateJobService:
self.config_manager = config_manager
self.current_user = current_user
# #region list_jobs [C:2] [TYPE Function] [SEMANTICS translate,jobs,query]
# @BRIEF List translation jobs with optional status filter and pagination.
# [DEF:list_jobs:Function]
# @PURPOSE: List translation jobs with optional status filter and pagination.
# @POST: Returns tuple of (total_count, list_of_jobs).
def list_jobs(
self,
page: int = 1,
@@ -199,30 +176,29 @@ class TranslateJobService:
jobs = query.order_by(TranslationJob.created_at.desc()).offset(offset).limit(page_size).all()
return total, jobs
# #endregion list_jobs
# [/DEF:list_jobs:Function]
# #region get_job [C:2] [TYPE Function] [SEMANTICS translate,jobs,query]
# @BRIEF Get a single translation job by ID, raising ValueError if not found.
# [DEF:get_job:Function]
# @PURPOSE: Get a single translation job by ID.
# @POST: Returns TranslationJob or raises ValueError.
def get_job(self, job_id: str) -> TranslationJob:
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
if not job:
raise ValueError(f"Translation job '{job_id}' not found")
return job
# #endregion get_job
# [/DEF:get_job:Function]
# #region create_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,create]
# @BRIEF Create a new translation job with column validation and dialect detection.
# @PRE payload contains valid job configuration (name, upsert_strategy, etc).
# @POST Returns the created TranslationJob with database_dialect cached.
# @SIDE_EFFECT Validates columns via SupersetClient if source_datasource_id is provided; caches database_dialect.
# [DEF:create_job:Function]
# @PURPOSE: Create a new translation job with column validation.
# @PRE: payload contains valid job configuration.
# @POST: Returns the created TranslationJob with database_dialect cached.
# @SIDE_EFFECT: Validates columns via SupersetClient if source_datasource_id is provided.
# @SIDE_EFFECT: Caches database_dialect from Superset connection.
def create_job(self, payload: TranslateJobCreate) -> TranslationJob:
log.reason("Creating translation job", payload={"name": payload.name})
logger.info(f"[TranslateJobService] Creating job '{payload.name}'")
# Validate: must have a translation column if datasource is configured
if payload.source_datasource_id and not payload.translation_column:
log.explore("Missing translation column", error="translation_column required when source_datasource_id is set", payload={
"source_datasource_id": payload.source_datasource_id,
})
raise ValueError("A translation column is required when a datasource is selected")
# Validate upsert strategy
@@ -247,7 +223,7 @@ class TranslateJobService:
)
dialect = detected_dialect
except Exception as e:
log.explore("Dialect detection failed", error=str(e))
logger.warning(f"[TranslateJobService] Dialect detection failed: {e}")
dialect = payload.source_dialect
# Build job instance
@@ -271,7 +247,6 @@ class TranslateJobService:
batch_size=payload.batch_size,
upsert_strategy=payload.upsert_strategy,
environment_id=payload.environment_id,
target_database_id=payload.target_database_id,
status="DRAFT",
created_by=self.current_user,
)
@@ -290,17 +265,17 @@ class TranslateJobService:
self.db.commit()
self.db.refresh(job)
log.reflect("Job created", payload={"job_id": job.id})
logger.info(f"[TranslateJobService] Created job '{job.id}'")
return job
# #endregion create_job
# [/DEF:create_job:Function]
# #region update_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,update]
# @BRIEF Update an existing translation job with optional dialect re-detection.
# @PRE payload contains fields to update; job_id exists.
# @POST Returns the updated TranslationJob with re-detected dialect if datasource changed.
# @SIDE_EFFECT Re-detects database_dialect if source_datasource_id changed.
# [DEF:update_job:Function]
# @PURPOSE: Update an existing translation job.
# @PRE: payload contains fields to update.
# @POST: Returns the updated TranslationJob.
# @SIDE_EFFECT: Re-detects database_dialect if source_datasource_id changed.
def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob:
log.reason("Updating translation job", payload={"job_id": job_id})
logger.info(f"[TranslateJobService] Updating job '{job_id}'")
job = self.get_job(job_id)
update_data = payload.model_dump(exclude_unset=True)
@@ -321,7 +296,7 @@ class TranslateJobService:
)
job.database_dialect = detected_dialect
except Exception as e:
log.explore("Dialect re-detection failed", error=str(e))
logger.warning(f"[TranslateJobService] Dialect re-detection failed: {e}")
job.updated_at = datetime.now(timezone.utc)
@@ -340,17 +315,16 @@ class TranslateJobService:
self.db.commit()
self.db.refresh(job)
log.reflect("Job updated", payload={"job_id": job_id})
logger.info(f"[TranslateJobService] Updated job '{job_id}'")
return job
# #endregion update_job
# [/DEF:update_job:Function]
# #region delete_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,delete]
# @BRIEF Delete a translation job and its dictionary associations.
# @PRE job_id must exist.
# @POST Job and all related TranslationJobDictionary records are deleted.
# @SIDE_EFFECT Deletes TranslationJob and TranslationJobDictionary rows.
# [DEF:delete_job:Function]
# @PURPOSE: Delete a translation job and its associations.
# @PRE: job_id must exist.
# @POST: Job and all related records are deleted.
def delete_job(self, job_id: str) -> None:
log.reason("Deleting translation job", payload={"job_id": job_id})
logger.info(f"[TranslateJobService] Deleting job '{job_id}'")
job = self.get_job(job_id)
# Delete dictionary associations
@@ -360,16 +334,15 @@ class TranslateJobService:
self.db.delete(job)
self.db.commit()
log.reflect("Job deleted", payload={"job_id": job_id})
# #endregion delete_job
logger.info(f"[TranslateJobService] Deleted job '{job_id}'")
# [/DEF:delete_job:Function]
# #region duplicate_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,duplicate]
# @BRIEF Duplicate a translation job with a new name, copying all fields and dictionary associations.
# @PRE job_id must exist.
# @POST Returns the duplicated TranslationJob with status DRAFT.
# @SIDE_EFFECT Creates new TranslationJob and TranslationJobDictionary rows.
# [DEF:duplicate_job:Function]
# @PURPOSE: Duplicate a translation job with a new name.
# @PRE: job_id must exist.
# @POST: Returns the duplicated TranslationJob with status DRAFT.
def duplicate_job(self, job_id: str, new_name: Optional[str] = None) -> TranslationJob:
log.reason("Duplicating translation job", payload={"job_id": job_id, "new_name": new_name})
logger.info(f"[TranslateJobService] Duplicating job '{job_id}'")
source = self.get_job(job_id)
# Copy all fields except id, created_at, updated_at, status
@@ -392,8 +365,6 @@ class TranslateJobService:
provider_id=source.provider_id,
batch_size=source.batch_size,
upsert_strategy=source.upsert_strategy,
environment_id=source.environment_id,
target_database_id=source.target_database_id,
status="DRAFT",
created_by=self.current_user,
)
@@ -414,20 +385,22 @@ class TranslateJobService:
self.db.commit()
self.db.refresh(new_job)
log.reflect("Job duplicated", payload={"new_job_id": new_job.id, "source_job_id": job_id})
logger.info(f"[TranslateJobService] Duplicated job '{job_id}' -> '{new_job.id}'")
return new_job
# #endregion duplicate_job
# [/DEF:duplicate_job:Function]
# #region get_job_dictionary_ids [C:1] [TYPE Function] [SEMANTICS translate,jobs,dictionaries]
# [DEF:get_job_dictionary_ids:Function]
# @PURPOSE: Get dictionary IDs attached to a job.
# @POST: Returns list of dictionary IDs.
def get_job_dictionary_ids(self, job_id: str) -> List[str]:
assocs = self.db.query(TranslationJobDictionary).filter(
TranslationJobDictionary.job_id == job_id
).all()
return [a.dictionary_id for a in assocs]
# #endregion get_job_dictionary_ids
# [/DEF:get_job_dictionary_ids:Function]
# #region fetch_available_datasources [C:2] [TYPE Function] [SEMANTICS translate,datasources,query]
# @BRIEF List available Superset datasets for translation job creation with optional search filter.
# [DEF:fetch_available_datasources:Function]
# @PURPOSE: List available Superset datasets for translation job creation.
def fetch_available_datasources(self, env_id: str, search: Optional[str] = None) -> list:
"""List Superset datasets available for translation."""
from ...core.superset_client import SupersetClient
@@ -453,11 +426,12 @@ class TranslateJobService:
"description": ds.get("description", ""),
})
return result
# #endregion fetch_available_datasources
# [/DEF:fetch_available_datasources:Function]
# #endregion TranslateJobService
# #region _extract_dialect [C:1] [TYPE Function] [SEMANTICS translate,dialect]
# #region _extract_dialect [TYPE Function]
# @BRIEF Extract database dialect from Superset backend URI.
def _extract_dialect(backend: str) -> str:
"""Extract dialect name from a Superset database backend URI (e.g. 'postgresql+psycopg2://...')."""
if not backend:
@@ -468,10 +442,9 @@ def _extract_dialect(backend: str) -> str:
return dialect.lower()
except Exception:
return "unknown"
# #endregion _extract_dialect
# #region job_to_response [C:2] [TYPE Function] [SEMANTICS translate,serialization]
# #region job_to_response [TYPE Function]
# @BRIEF Convert a TranslationJob ORM model to a TranslateJobResponse schema with dictionary_ids.
def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -> TranslateJobResponse:
return TranslateJobResponse(
@@ -499,29 +472,27 @@ def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -
updated_at=job.updated_at,
dictionary_ids=dict_ids or [],
environment_id=job.environment_id,
target_database_id=job.target_database_id,
)
# #endregion job_to_response
# #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS translate,datasource,columns]
# @BRIEF Fetch datasource column metadata from Superset and return structured DatasourceColumnsResponse.
# @PRE datasource_id is a valid Superset dataset ID.
# @POST Returns DatasourceColumnsResponse with column metadata and database dialect.
# @SIDE_EFFECT Queries Superset API for dataset detail and database info.
# #region DatasourceColumnsService [TYPE Function]
# @BRIEF Fetch datasource column metadata from Superset and return structured response.
# @PRE: datasource_id is a valid Superset dataset ID.
# @POST: Returns DatasourceColumnsResponse with column metadata and database dialect.
# @SIDE_EFFECT: Queries Superset API for dataset detail and database info.
def get_datasource_columns(
datasource_id: int,
env_id: str,
config_manager: ConfigManager,
) -> DatasourceColumnsResponse:
"""Fetch and return column metadata for a given Superset datasource."""
log.reason("Fetching datasource columns", payload={"datasource_id": datasource_id, "env_id": env_id})
logger.info(f"[get_datasource_columns] Fetching columns for datasource {datasource_id}")
# Find environment config
environments = config_manager.get_environments()
env_config = next((e for e in environments if e.id == env_id), None)
if not env_config:
log.explore("Environment not found", payload={"env_id": env_id}, error=f"Environment {env_id} not configured")
raise ValueError(f"Superset environment '{env_id}' not found")
# Create Superset client
@@ -543,7 +514,6 @@ def get_datasource_columns(
db_result = db_record.get("result", db_record) if isinstance(db_record, dict) else db_record
dialect = get_dialect_from_database(db_result)
else:
log.explore("Could not determine dialect", payload={"datasource_id": datasource_id}, error=f"Could not determine dialect for datasource {datasource_id}")
raise ValueError("Could not determine database dialect for this datasource")
# Extract columns
@@ -561,19 +531,14 @@ def get_datasource_columns(
description=col.get("description", ""),
))
result = DatasourceColumnsResponse(
return DatasourceColumnsResponse(
datasource_id=datasource_id,
datasource_name=dataset_detail.get("table_name"),
schema_name=dataset_detail.get("schema"),
database_dialect=dialect,
columns=columns,
)
log.reflect("Datasource columns fetched", payload={
"datasource_id": datasource_id,
"columns_count": len(columns),
"dialect": dialect,
})
return result
# #endregion get_datasource_columns
# #endregion DatasourceColumnsService
# #endregion TranslateJobService
# #endregion TranslateJobService

78
backend/src/plugins/translate/sql_generator.py
View File

@@ -1,18 +1,18 @@
# #region SQLGenerator [C:4] [TYPE Module] [SEMANTICS translate,sql,generator,dialect]
# @BRIEF Dialect-aware safe SQL generation for INSERT/UPSERT operations with identifier quoting and value encoding.
# @LAYER Domain
# #region SQLGenerator [C:3] [TYPE Module] [SEMANTICS translate, sql, generator, dialect]
# @BRIEF Dialect-aware safe SQL generation for INSERT/UPSERT operations.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION DEPENDS_ON -> [TranslationRun]
# @PRE Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS.
# @POST Returns safe SQL strings for the target dialect.
# @PRE: Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS.
# @POST: Returns safe SQL strings for the target dialect.
# @SIDE_EFFECT: None — pure code generation.
# @RATIONALE: Dialect-aware SQL uses ON CONFLICT for PostgreSQL; plain INSERT for ClickHouse with documented limitations.
# @REJECTED: UPDATE statements — source is append-only; UPSERT covers overwrite case.
# @REJECTED: ORM-based insert bypasses Superset's SQL Lab audit trail.
from typing import Any, Dict, List, Optional, Tuple
from datetime import datetime, timezone
from ...core.cot_logger import MarkerLogger
from ...core.logger import belief_scope
log = MarkerLogger("SqlGenerator")
from typing import Any, Dict, List, Optional, Tuple
from ...core.logger import logger, belief_scope
# PostgreSQL/Greenplum dialects that support ON CONFLICT
POSTGRESQL_DIALECTS = {"postgresql", "redshift", "greenplum"}
@@ -57,8 +57,10 @@ def _normalize_timestamp_value(value: Any) -> Optional[str]:
# #endregion _normalize_timestamp_value
# #region _quote_identifier [C:2] [TYPE Function] [SEMANTICS translate,sql,quoting]
# @BRIEF Quote a SQL identifier per dialect rules — double quotes for PostgreSQL, backticks for ClickHouse.
# #region _quote_identifier [TYPE Function]
# @BRIEF Quote an identifier per dialect rules. PostgreSQL uses double quotes; ClickHouse uses backticks.
# @PRE: identifier is a non-empty string.
# @POST: Returns safely quoted identifier.
def _quote_identifier(identifier: str, dialect: str) -> str:
"""Quote a SQL identifier per dialect rules."""
if not identifier:
@@ -155,17 +157,16 @@ def generate_insert_sql(
# #endregion generate_insert_sql
# #region generate_upsert_sql [C:3] [TYPE Function] [SEMANTICS translate,sql,upsert]
# @BRIEF Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE / DO NOTHING.
# @RELATION DEPENDS_ON -> [_quote_identifier]
# @RELATION DEPENDS_ON -> [_build_values_clause]
# #region generate_upsert_sql [TYPE Function]
# @BRIEF Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE.
# @PRE: dialect is postgresql-compatible. target_table, columns, key_columns are non-empty.
# @POST: Returns UPSERT SQL string or raises ValueError.
def generate_upsert_sql(
target_schema: Optional[str],
target_table: str,
columns: List[str],
key_columns: List[str],
rows: List[Dict[str, Any]],
dialect: Optional[str] = None,
) -> str:
"""Generate INSERT ... ON CONFLICT (key_cols) DO UPDATE SET ... SQL."""
with belief_scope("generate_upsert_sql"):
@@ -180,7 +181,7 @@ def generate_upsert_sql(
col_list = ", ".join(columns)
key_list = ", ".join(key_columns)
values = _build_values_clause(columns, rows, dialect=dialect)
values = _build_values_clause(columns, rows)
table_ref = target_table
if target_schema:
@@ -205,20 +206,17 @@ def generate_upsert_sql(
# #endregion generate_upsert_sql
# #region SQLGenerator [C:4] [TYPE Class] [SEMANTICS translate,sql,generator]
# @BRIEF Generate safe, dialect-appropriate SQL INSERT/UPSERT statements with batching support.
# @PRE Job has target_schema, target_table, key columns configured.
# @POST Returns generated SQL string for the target dialect.
# @SIDE_EFFECT None — pure SQL generation.
# @RELATION DEPENDS_ON -> [generate_insert_sql]
# @RELATION DEPENDS_ON -> [generate_upsert_sql]
# #region SQLGenerator [C:3] [TYPE Class]
# @BRIEF Generate safe, dialect-appropriate SQL INSERT/UPSERT statements.
# @PRE: Job has target_schema, target_table, key columns configured.
# @POST: Returns generated SQL string for the target dialect.
class SQLGenerator:
# #region SQLGenerator.generate [C:4] [TYPE Function] [SEMANTICS translate,sql,generate]
# @BRIEF Generate SQL for a set of rows, detecting dialect from the job configuration.
# @PRE dialect is a supported database dialect. columns list is non-empty. rows is non-empty.
# @POST Returns tuple of (sql_string, statement_count).
# @SIDE_EFFECT None — pure SQL generation.
# [DEF:SQLGenerator.generate:Function]
# @PURPOSE: Generate SQL for a set of rows, detecting dialect from the job configuration.
# @PRE: dialect is a supported database dialect. columns list is non-empty. rows is non-empty.
# @POST: Returns tuple of (sql_string, statement_count).
# @SIDE_EFFECT: None — pure SQL generation.
@staticmethod
def generate(
dialect: str,
@@ -244,7 +242,7 @@ class SQLGenerator:
Tuple of (sql_string, row_count).
"""
with belief_scope("SQLGenerator.generate"):
log.reason("Generating SQL", payload={
logger.reason("Generating SQL", {
"dialect": dialect,
"schema": target_schema,
"table": target_table,
@@ -290,7 +288,6 @@ class SQLGenerator:
columns=quoted_columns,
key_columns=quoted_key_columns,
rows=rows,
dialect=dialect,
)
else:
sql = generate_insert_sql(
@@ -310,7 +307,7 @@ class SQLGenerator:
dialect=dialect,
)
if use_upsert:
log.reason("ClickHouse UPSERT not supported; using plain INSERT", payload={
logger.reason("ClickHouse UPSERT not supported; using plain INSERT", {
"note": "ClickHouse does not support ON CONFLICT. Use ReplacingMergeTree for dedup.",
})
else:
@@ -323,17 +320,18 @@ class SQLGenerator:
dialect=dialect,
)
log.reflect("SQL generated", payload={
logger.reflect("SQL generated", {
"dialect": dialect,
"row_count": len(rows),
"sql_length": len(sql),
})
return sql, len(rows)
# #endregion SQLGenerator.generate
# [/DEF:SQLGenerator.generate:Function]
# #region SQLGenerator.generate_batch [C:3] [TYPE Function] [SEMANTICS translate,sql,batch]
# @BRIEF Generate separate INSERT statements for each row chunk (batch-safe version).
# @RELATION DEPENDS_ON -> [SQLGenerator.generate]
# [DEF:SQLGenerator.generate_batch:Function]
# @PURPOSE: Generate separate INSERT statements for each row (batch-safe version).
# @PRE: Same as generate().
# @POST: Returns list of (sql_string, row_index) tuples.
@staticmethod
def generate_batch(
dialect: str,
@@ -369,7 +367,7 @@ class SQLGenerator:
statements.append((sql, count))
return statements
# #endregion SQLGenerator.generate_batch
# [/DEF:SQLGenerator.generate_batch:Function]
# #endregion SQLGenerator

196
backend/src/plugins/translate/superset_executor.py
View File

@@ -1,49 +1,43 @@
# #region SupersetSqlLabExecutor [C:5] [TYPE Module] [SEMANTICS translate,superset,sqllab,execute]
# @BRIEF Submit SQL to Superset SQL Lab API and poll execution status with retry.
# @LAYER Infra
# #region SupersetSqlLabExecutor [C:4] [TYPE Module] [SEMANTICS translate, superset, sqllab, execute]
# @BRIEF Submit SQL to Superset SQL Lab API and poll execution status.
# @LAYER: Infra
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [ConfigManager]
# @PRE Valid Superset environment configuration and authenticated client.
# @POST SQL is submitted to Superset SQL Lab; execution reference is returned.
# @SIDE_EFFECT Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status.
# @DATA_CONTRACT Input[sql:str, database_id:Optional] -> Output[Dict with query_id, status, error_message]
# @INVARIANT Polling respects max_polls limit; timeout returns structured error.
# @RATIONALE Direct SQL Lab API submission provides audit trail and execution monitoring within Superset.
# @REJECTED Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC.
# @PRE: Valid Superset environment configuration and authenticated client.
# @POST: SQL is submitted to Superset SQL Lab; execution reference is returned.
# @SIDE_EFFECT: Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status.
# @RATIONALE: Direct SQL Lab API submission provides audit trail and execution monitoring within Superset.
# @REJECTED: Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC.
from typing import Any, Dict, List, Optional, Tuple, Union
from typing import Any, Dict, List, Optional, Tuple
from datetime import datetime, timezone
import json
import time
import uuid
from ...core.logger import logger, belief_scope
from ...core.cot_logger import MarkerLogger
from ...core.config_manager import ConfigManager
from ...core.superset_client import SupersetClient
log = MarkerLogger("SupersetExecutor")
# #region SupersetSqlLabExecutor [C:5] [TYPE Class] [SEMANTICS translate,superset,sqllab]
# #region SupersetSqlLabExecutor [C:4] [TYPE Class]
# @BRIEF Submit SQL to Superset SQL Lab API with polling and status tracking.
# @PRE Valid environment ID and ConfigManager.
# @POST SQL submitted to Superset; execution reference recorded; polling completes.
# @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status.
# @DATA_CONTRACT Input[env_id:str, database_id:Optional] -> Output[Dict with query_id, status]
# @INVARIANT database_id resolved lazily; polling capped at max_polls.
# @PRE: Valid environment ID and ConfigManager.
# @POST: SQL submitted to Superset; execution reference recorded.
# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status.
class SupersetSqlLabExecutor:
def __init__(self, config_manager: ConfigManager, env_id: str, database_id: Optional[int] = None):
def __init__(self, config_manager: ConfigManager, env_id: str):
self.config_manager = config_manager
self.env_id = env_id
self._client: Optional[SupersetClient] = None
self._database_id: Optional[int] = database_id
self._database_id: Optional[int] = None
# #region _get_client [C:4] [TYPE Function] [SEMANTICS translate,superset,client]
# @BRIEF Lazy-initialize SupersetClient for the configured environment.
# @PRE env_id must correspond to a valid environment config.
# @POST Returns authenticated SupersetClient.
# @SIDE_EFFECT Authenticates against Superset API on first call.
# [DEF:_get_client:Function]
# @PURPOSE: Lazy-initialize SupersetClient for the configured environment.
# @PRE: env_id must correspond to a valid environment config.
# @POST: Returns authenticated SupersetClient.
# @SIDE_EFFECT: Authenticates against Superset API.
def _get_client(self) -> SupersetClient:
with belief_scope("SupersetSqlLabExecutor._get_client"):
if self._client is not None:
@@ -56,13 +50,13 @@ class SupersetSqlLabExecutor:
self._client = SupersetClient(env_config)
return self._client
# #endregion _get_client
# [/DEF:_get_client:Function]
# #region resolve_database_id [C:4] [TYPE Function] [SEMANTICS translate,database,resolve]
# @BRIEF Resolve the target database ID from the environment by name or default.
# @PRE Superset environment is reachable and has databases.
# @POST Returns database_id integer or raises ValueError.
# @SIDE_EFFECT Fetches databases list from Superset API.
# [DEF:resolve_database_id:Function]
# @PURPOSE: Resolve the target database ID from the environment.
# @PRE: database_name or database_id should be known.
# @POST: Returns database_id integer or raises ValueError.
# @SIDE_EFFECT: Fetches databases list from Superset.
def resolve_database_id(self, database_name: Optional[str] = None) -> int:
with belief_scope("SupersetSqlLabExecutor.resolve_database_id"):
client = self._get_client()
@@ -76,7 +70,7 @@ class SupersetSqlLabExecutor:
for db in databases:
if db.get("database_name", "").lower() == database_name.lower():
self._database_id = db["id"]
log.reason("Resolved database ID by name", payload={
logger.reason("Resolved database ID by name", {
"database_name": database_name,
"database_id": self._database_id,
})
@@ -87,76 +81,42 @@ class SupersetSqlLabExecutor:
# Default: use first database
self._database_id = databases[0]["id"]
log.reason("Using default database", payload={
logger.reason("Using default database", {
"database_name": databases[0].get("database_name"),
"database_id": self._database_id,
})
return self._database_id
# #endregion resolve_database_id
# [/DEF:resolve_database_id:Function]
# #region resolve_database_uuid [C:4] [TYPE Function] [SEMANTICS translate,database,uuid]
# @BRIEF Resolve a database UUID to a numeric database ID from Superset.
# @PRE uuid_str is a valid Superset database UUID.
# @POST Returns numeric database_id or raises ValueError if not found.
# @SIDE_EFFECT Fetches databases list from Superset API.
def resolve_database_uuid(self, uuid_str: str) -> int:
with belief_scope("SupersetSqlLabExecutor.resolve_database_uuid"):
client = self._get_client()
_, databases = client.get_databases(
query={"columns": ["id", "uuid", "database_name"]}
)
if not databases:
raise ValueError("No databases found in Superset environment")
for db in databases:
db_uuid = db.get("uuid") or ""
if db_uuid.lower() == uuid_str.lower():
db_id = db["id"]
self._database_id = db_id
log.reason("Resolved database ID by UUID", payload={
"uuid": uuid_str,
"database_id": db_id,
"database_name": db.get("database_name"),
})
return db_id
raise ValueError(
f"Database with UUID '{uuid_str}' not found in Superset environment"
)
# #endregion resolve_database_uuid
# #region execute_sql [C:4] [TYPE Function] [SEMANTICS translate,sql,execute]
# @BRIEF Submit SQL to Superset SQL Lab and return execution tracking info.
# @PRE sql is valid SQL string. database_id is a valid Superset DB ID.
# @POST Returns execution result dict with query_id and status.
# @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/.
# [DEF:execute_sql:Function]
# @PURPOSE: Submit SQL to Superset SQL Lab and return execution tracking info.
# @PRE: sql is valid SQL string. database_id is a valid Superset DB ID.
# @POST: Returns execution result dict with query_id and status.
# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/.
def execute_sql(
self,
sql: str,
database_id: Optional[Union[int, str]] = None,
database_id: Optional[int] = None,
run_async: bool = True,
) -> Dict[str, Any]:
with belief_scope("SupersetSqlLabExecutor.execute_sql"):
client = self._get_client()
db_id = database_id or self._database_id
if db_id is None:
if not db_id:
db_id = self.resolve_database_id()
# Handle UUID string passed as database_id — resolve to numeric ID
if isinstance(db_id, str):
try:
db_id = int(db_id)
except (ValueError, TypeError):
db_id = self.resolve_database_uuid(db_id)
log.reason("Submitting SQL to Superset SQL Lab", payload={
logger.reason("Submitting SQL to Superset SQL Lab", {
"database_id": db_id,
"sql_length": len(sql),
"run_async": run_async,
})
payload = {
"database_id": db_id,
"sql": sql,
"client_id": uuid.uuid4().hex[:10],
"runAsync": run_async,
"schema": None,
"tab": "translation-insert",
}
try:
@@ -167,25 +127,17 @@ class SupersetSqlLabExecutor:
headers={"Content-Type": "application/json"},
)
except Exception as e:
log.explore("SQL Lab execute failed", error=str(e))
logger.explore("SQL Lab execute failed", {"error": str(e)})
raise ValueError(f"Superset SQL Lab execute failed: {e}")
# Parse response — try multiple formats
# Parse response
result = response if isinstance(response, dict) else {}
query_id = (result.get("query_id")
or result.get("id")
or result.get("sql_lab_session_ref")
or (result.get("result") or {}).get("query_id")
or (result.get("result") or {}).get("id"))
query_id = result.get("query_id") or result.get("id")
status = result.get("status", "unknown")
log.reason("SQL Lab execute response", payload={
logger.reason("SQL Lab execute response", {
"query_id": query_id,
"status": status,
"has_query_id": query_id is not None,
"has_errors": "errors" in result or "error" in result,
"response_keys": list(result.keys()) if isinstance(result, dict) else type(result).__name__,
"response_preview": str(result)[:1000] if isinstance(result, dict) else str(result)[:1000],
})
return {
@@ -194,13 +146,13 @@ class SupersetSqlLabExecutor:
"raw_response": result,
"database_id": db_id,
}
# #endregion execute_sql
# [/DEF:execute_sql:Function]
# #region poll_execution_status [C:4] [TYPE Function] [SEMANTICS translate,poll,status]
# @BRIEF Poll Superset for SQL execution status until completion or timeout.
# @PRE query_id is a valid Superset query ID.
# @POST Returns final execution status dict with success/failure/timeout.
# @SIDE_EFFECT Makes HTTP GET requests to Superset API.
# [DEF:poll_execution_status:Function]
# @PURPOSE: Poll Superset for SQL execution status until completion or timeout.
# @PRE: query_id is a valid Superset query ID.
# @POST: Returns final execution status dict.
# @SIDE_EFFECT: Makes HTTP GET requests to Superset.
def poll_execution_status(
self,
query_id: str,
@@ -210,7 +162,7 @@ class SupersetSqlLabExecutor:
with belief_scope("SupersetSqlLabExecutor.poll_execution_status"):
client = self._get_client()
log.reason("Polling SQL execution status", payload={
logger.reason("Polling SQL execution status", {
"query_id": query_id,
"max_polls": max_polls,
"interval": poll_interval_seconds,
@@ -228,7 +180,7 @@ class SupersetSqlLabExecutor:
# Terminal states
if state in ("success", "finished", "completed"):
log.reason("SQL execution completed", payload={
logger.reason("SQL execution completed", {
"query_id": query_id,
"attempt": attempt + 1,
"rows_affected": result.get("rows"),
@@ -245,8 +197,7 @@ class SupersetSqlLabExecutor:
if state in ("failed", "error", "stopped"):
error_msg = result.get("error_message", "Unknown error")
log.explore("SQL execution failed", error="SQL query execution returned failed/error state",
payload={
logger.explore("SQL execution failed", {
"query_id": query_id,
"state": state,
"error": error_msg,
@@ -267,8 +218,7 @@ class SupersetSqlLabExecutor:
time.sleep(poll_interval_seconds)
except Exception as e:
log.explore("Polling error, retrying", error="Polling encountered error, will retry",
payload={
logger.explore("Polling error, retrying", {
"query_id": query_id,
"attempt": attempt + 1,
"error": str(e),
@@ -277,7 +227,7 @@ class SupersetSqlLabExecutor:
continue
# Timeout
log.explore("SQL execution polling timed out", error="Polling timed out", payload={
logger.explore("SQL execution polling timed out", {
"query_id": query_id,
"max_polls": max_polls,
})
@@ -287,17 +237,17 @@ class SupersetSqlLabExecutor:
"error_message": f"Polling timed out after {max_polls} attempts",
"results": None,
}
# #endregion poll_execution_status
# [/DEF:poll_execution_status:Function]
# #region execute_and_poll [C:4] [TYPE Function] [SEMANTICS translate,sql,execute,poll]
# @BRIEF Execute SQL and wait for completion. One-shot convenience method.
# @PRE sql is valid SQL.
# @POST Returns final execution result.
# @SIDE_EFFECT Makes HTTP calls to Superset API.
# [DEF:execute_and_poll:Function]
# @PURPOSE: Execute SQL and wait for completion. One-shot convenience method.
# @PRE: sql is valid SQL.
# @POST: Returns final execution result.
# @SIDE_EFFECT: Makes HTTP calls to Superset API.
def execute_and_poll(
self,
sql: str,
database_id: Optional[Union[int, str]] = None,
database_id: Optional[int] = None,
max_polls: int = 60,
poll_interval_seconds: float = 2.0,
) -> Dict[str, Any]:
@@ -305,11 +255,12 @@ class SupersetSqlLabExecutor:
exec_result = self.execute_sql(
sql=sql,
database_id=database_id,
run_async=True,
)
query_id = exec_result.get("query_id")
if not query_id:
log.explore("No query_id from SQL Lab execute", error="No query_id returned", payload={
logger.explore("No query_id from SQL Lab execute", {
"raw_response": exec_result.get("raw_response"),
})
return {
@@ -323,13 +274,13 @@ class SupersetSqlLabExecutor:
max_polls=max_polls,
poll_interval_seconds=poll_interval_seconds,
)
# #endregion execute_and_poll
# [/DEF:execute_and_poll:Function]
# #region get_query_results [C:4] [TYPE Function] [SEMANTICS translate,query,results]
# @BRIEF Fetch the results of a completed query from Superset.
# @PRE query_id is a valid Superset query ID.
# @POST Returns query results if available, or error dict.
# @SIDE_EFFECT Makes HTTP GET to Superset API.
# [DEF:get_query_results:Function]
# @PURPOSE: Fetch the results of a completed query.
# @PRE: query_id is a valid Superset query ID.
# @POST: Returns query results if available.
# @SIDE_EFFECT: Makes HTTP GET to Superset.
def get_query_results(self, query_id: str) -> Dict[str, Any]:
with belief_scope("SupersetSqlLabExecutor.get_query_results"):
client = self._get_client()
@@ -345,8 +296,7 @@ class SupersetSqlLabExecutor:
"results": result.get("results"),
}
except Exception as e:
log.explore("Failed to fetch query results", error="Failed to fetch query results from Superset",
payload={
logger.explore("Failed to fetch query results", {
"query_id": query_id,
"error": str(e),
})
@@ -356,7 +306,7 @@ class SupersetSqlLabExecutor:
"error_message": str(e),
"results": None,
}
# #endregion get_query_results
# [/DEF:get_query_results:Function]
# #endregion SupersetSqlLabExecutor