busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

semantics

Browse Source
This commit is contained in:
busya
2026-05-26 09:30:41 +03:00
parent 1e7bcecaea
commit 9ffa8af1dc
623 changed files with 28045 additions and 26557 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
backend/src/plugins/backup.py
View File

@@ -1,10 +1,10 @@
# #region BackupPlugin [TYPE Module] [SEMANTICS backup, export, archive, superset, dashboard]
# @BRIEF A plugin that provides functionality to back up Superset dashboards.
# @LAYER: App
# @LAYER App
# @RELATION IMPLEMENTS -> PluginBase
# @RELATION DEPENDS_ON -> superset_tool.client
# @RELATION DEPENDS_ON -> superset_tool.utils
# @RELATION USES -> TaskContext
# @RELATION DEPENDS_ON -> [EXT:Library:superset_tool.client]
# @RELATION DEPENDS_ON -> [EXT:Library:superset_tool.utils]
# @RELATION DEPENDS_ON -> [TaskContext]
from pathlib import Path
from typing import Any
@@ -30,9 +30,9 @@ class BackupPlugin(PluginBase):
@property
# region id [TYPE Function]
# @PURPOSE: Returns the unique identifier for the backup plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string ID.
# @RETURN: str - "superset-backup"
# @PRE Plugin instance exists.
# @POST Returns string ID.
# @RETURN str - "superset-backup"
def id(self) -> str:
with belief_scope("id"):
return "superset-backup"
@@ -41,9 +41,9 @@ class BackupPlugin(PluginBase):
@property
# region name [TYPE Function]
# @PURPOSE: Returns the human-readable name of the backup plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string name.
# @RETURN: str - Plugin name.
# @PRE Plugin instance exists.
# @POST Returns string name.
# @RETURN str - Plugin name.
def name(self) -> str:
with belief_scope("name"):
return "Superset Dashboard Backup"
@@ -52,9 +52,9 @@ class BackupPlugin(PluginBase):
@property
# region description [TYPE Function]
# @PURPOSE: Returns a description of the backup plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string description.
# @RETURN: str - Plugin description.
# @PRE Plugin instance exists.
# @POST Returns string description.
# @RETURN str - Plugin description.
def description(self) -> str:
with belief_scope("description"):
return "Backs up all dashboards from a Superset instance."
@@ -63,9 +63,9 @@ class BackupPlugin(PluginBase):
@property
# region version [TYPE Function]
# @PURPOSE: Returns the version of the backup plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string version.
# @RETURN: str - "1.0.0"
# @PRE Plugin instance exists.
# @POST Returns string version.
# @RETURN str - "1.0.0"
def version(self) -> str:
with belief_scope("version"):
return "1.0.0"
@@ -74,7 +74,7 @@ class BackupPlugin(PluginBase):
@property
# region ui_route [TYPE Function]
# @PURPOSE: Returns the frontend route for the backup plugin.
# @RETURN: str - "/tools/backups"
# @RETURN str - "/tools/backups"
def ui_route(self) -> str:
with belief_scope("ui_route"):
return "/tools/backups"
@@ -82,9 +82,9 @@ class BackupPlugin(PluginBase):
# region get_schema [TYPE Function]
# @PURPOSE: Returns the JSON schema for backup plugin parameters.
# @PRE: Plugin instance exists.
# @POST: Returns dictionary schema.
# @RETURN: Dict[str, Any] - JSON schema.
# @PRE Plugin instance exists.
# @POST Returns dictionary schema.
# @RETURN Dict[str, Any] - JSON schema.
def get_schema(self) -> dict[str, Any]:
with belief_scope("get_schema"):
config_manager = get_config_manager()
@@ -107,10 +107,10 @@ class BackupPlugin(PluginBase):
# region execute [TYPE Function]
# @PURPOSE: Executes the dashboard backup logic with TaskContext support.
# @PARAM: params (Dict[str, Any]) - Backup parameters (env, backup_path, dashboard_ids).
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE: Target environment must be configured. params must be a dictionary.
# @POST: All dashboards are exported and archived.
# @PARAM params (Dict[str, Any]) - Backup parameters (env, backup_path, dashboard_ids).
# @PARAM context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE Target environment must be configured. params must be a dictionary.
# @POST All dashboards are exported and archived.
async def execute(self, params: dict[str, Any], context: TaskContext | None = None):
with belief_scope("execute"):
config_manager = get_config_manager()

68
backend/src/plugins/debug.py
View File

@@ -1,8 +1,8 @@
# #region DebugPluginModule [TYPE Module] [SEMANTICS debug, diagnostic, superset, api, system]
# @BRIEF Implements a plugin for system diagnostics and debugging Superset API responses.
# @LAYER: Plugins
# @RELATION Inherits from PluginBase. Uses SupersetClient from core.
# @RELATION USES -> TaskContext
# @LAYER Plugin
# @BRIEF Plugin for diagnostics. Inherits PluginBase.
# @RELATION DEPENDS_ON -> [TaskContext]
from typing import Any
@@ -22,9 +22,9 @@ class DebugPlugin(PluginBase):
@property
# region id [TYPE Function]
# @PURPOSE: Returns the unique identifier for the debug plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string ID.
# @RETURN: str - "system-debug"
# @PRE Plugin instance exists.
# @POST Returns string ID.
# @RETURN str - "system-debug"
def id(self) -> str:
with belief_scope("id"):
return "system-debug"
@@ -33,9 +33,9 @@ class DebugPlugin(PluginBase):
@property
# region name [TYPE Function]
# @PURPOSE: Returns the human-readable name of the debug plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string name.
# @RETURN: str - Plugin name.
# @PRE Plugin instance exists.
# @POST Returns string name.
# @RETURN str - Plugin name.
def name(self) -> str:
with belief_scope("name"):
return "System Debug"
@@ -44,9 +44,9 @@ class DebugPlugin(PluginBase):
@property
# region description [TYPE Function]
# @PURPOSE: Returns a description of the debug plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string description.
# @RETURN: str - Plugin description.
# @PRE Plugin instance exists.
# @POST Returns string description.
# @RETURN str - Plugin description.
def description(self) -> str:
with belief_scope("description"):
return "Run system diagnostics and debug Superset API responses."
@@ -55,9 +55,9 @@ class DebugPlugin(PluginBase):
@property
# region version [TYPE Function]
# @PURPOSE: Returns the version of the debug plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string version.
# @RETURN: str - "1.0.0"
# @PRE Plugin instance exists.
# @POST Returns string version.
# @RETURN str - "1.0.0"
def version(self) -> str:
with belief_scope("version"):
return "1.0.0"
@@ -66,7 +66,7 @@ class DebugPlugin(PluginBase):
@property
# region ui_route [TYPE Function]
# @PURPOSE: Returns the frontend route for the debug plugin.
# @RETURN: str - "/tools/debug"
# @RETURN str - "/tools/debug"
def ui_route(self) -> str:
with belief_scope("ui_route"):
return "/tools/debug"
@@ -74,9 +74,9 @@ class DebugPlugin(PluginBase):
# region get_schema [TYPE Function]
# @PURPOSE: Returns the JSON schema for the debug plugin parameters.
# @PRE: Plugin instance exists.
# @POST: Returns dictionary schema.
# @RETURN: Dict[str, Any] - JSON schema.
# @PRE Plugin instance exists.
# @POST Returns dictionary schema.
# @RETURN Dict[str, Any] - JSON schema.
def get_schema(self) -> dict[str, Any]:
with belief_scope("get_schema"):
return {
@@ -115,11 +115,11 @@ class DebugPlugin(PluginBase):
# region execute [TYPE Function]
# @PURPOSE: Executes the debug logic with TaskContext support.
# @PARAM: params (Dict[str, Any]) - Debug parameters.
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE: action must be provided in params.
# @POST: Debug action is executed and results returned.
# @RETURN: Dict[str, Any] - Execution results.
# @PARAM params (Dict[str, Any]) - Debug parameters.
# @PARAM context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE action must be provided in params.
# @POST Debug action is executed and results returned.
# @RETURN Dict[str, Any] - Execution results.
async def execute(self, params: dict[str, Any], context: TaskContext | None = None) -> dict[str, Any]:
with belief_scope("execute"):
action = params.get("action")
@@ -142,11 +142,11 @@ class DebugPlugin(PluginBase):
# region _test_db_api [TYPE Function]
# @PURPOSE: Tests database API connectivity for source and target environments.
# @PRE: source_env and target_env params exist in params.
# @POST: Returns DB counts for both envs.
# @PARAM: params (Dict) - Plugin parameters.
# @PARAM: log - Logger instance for superset_api source.
# @RETURN: Dict - Comparison results.
# @PRE source_env and target_env params exist in params.
# @POST Returns DB counts for both envs.
# @PARAM params (Dict) - Plugin parameters.
# @PARAM log - Logger instance for superset_api source.
# @RETURN Dict - Comparison results.
async def _test_db_api(self, params: dict[str, Any], log) -> dict[str, Any]:
with belief_scope("_test_db_api"):
source_env_name = params.get("source_env")
@@ -180,11 +180,11 @@ class DebugPlugin(PluginBase):
# region _get_dataset_structure [TYPE Function]
# @PURPOSE: Retrieves the structure of a dataset.
# @PRE: env and dataset_id params exist in params.
# @POST: Returns dataset JSON structure.
# @PARAM: params (Dict) - Plugin parameters.
# @PARAM: log - Logger instance for superset_api source.
# @RETURN: Dict - Dataset structure.
# @PRE env and dataset_id params exist in params.
# @POST Returns dataset JSON structure.
# @PARAM params (Dict) - Plugin parameters.
# @PARAM log - Logger instance for superset_api source.
# @RETURN Dict - Dataset structure.
async def _get_dataset_structure(self, params: dict[str, Any], log) -> dict[str, Any]:
with belief_scope("_get_dataset_structure"):
env_name = params.get("env")

8
backend/src/plugins/git/llm_extension.py
View File

@@ -1,6 +1,6 @@
# #region GitLLMExtensionModule [C:3] [TYPE Module] [SEMANTICS git, llm, commit, message, generation]
# @BRIEF LLM-based extensions for the Git plugin, specifically for commit message generation.
# @LAYER: Domain
# @LAYER Domain
# @RELATION DEPENDS_ON -> [LLMClient]
@@ -19,9 +19,9 @@ class GitLLMExtension:
# region suggest_commit_message [TYPE Function]
# @PURPOSE: Generates a suggested commit message based on a diff and history.
# @PARAM: diff (str) - The git diff of staged changes.
# @PARAM: history (List[str]) - Recent commit messages for context.
# @RETURN: str - The suggested commit message.
# @PARAM diff (str) - The git diff of staged changes.
# @PARAM history (List[str]) - Recent commit messages for context.
# @RETURN str - The suggested commit message.
@retry(
stop=stop_after_attempt(2),
wait=wait_exponential(multiplier=1, min=2, max=10),

56
backend/src/plugins/git_plugin.py
View File

@@ -1,16 +1,16 @@
# #region GitPluginModule [C:4] [TYPE Module] [SEMANTICS git, versioning, deploy, sync, superset, backup, transactional]
#
# @BRIEF Предоставляет плагин для версионирования и развертывания дашбордов Superset.
# @LAYER: Plugin
# @RELATION INHERITS_FROM -> src.core.plugin_base.PluginBase
# @RELATION USES -> src.services.git_service.GitService
# @RELATION USES -> src.core.superset_client.SupersetClient
# @RELATION USES -> src.core.config_manager.ConfigManager
# @RELATION USES -> TaskContext
# @LAYER Plugin
# @RELATION INHERITS -> [EXT:path:src.core.plugin_base.PluginBase]
# @RELATION DEPENDS_ON -> [GitService]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [ConfigManager]
# @RELATION DEPENDS_ON -> [TaskContext]
#
# @INVARIANT: Все операции с Git должны выполняться через GitService.
# @INVARIANT Все операции с Git должны выполняться через GitService.
# @CONSTRAINT: Плагин работает только с распакованными YAML-экспортами Superset.
# @INVARIANT: _handle_sync сохраняет backup управляемых директорий перед удалением;
# @INVARIANT _handle_sync сохраняет backup управляемых директорий перед удалением;
# при ошибке распаковки backup восстанавливается.
import io
@@ -35,8 +35,8 @@ class GitPlugin(PluginBase):
# region __init__ [TYPE Function]
# @PURPOSE: Инициализирует плагин и его зависимости.
# @PRE: shared config_manager доступен через src.dependencies.
# @POST: Инициализированы git_service и config_manager.
# @PRE shared config_manager доступен через src.dependencies.
# @POST Инициализированы git_service и config_manager.
def __init__(self):
with belief_scope("GitPlugin.__init__"):
app_logger.info("Initializing GitPlugin.")
@@ -85,9 +85,9 @@ class GitPlugin(PluginBase):
# region get_schema [TYPE Function]
# @PURPOSE: Возвращает JSON-схему параметров для выполнения задач плагина.
# @PRE: GitPlugin is initialized.
# @POST: Returns a JSON schema dictionary.
# @RETURN: Dict[str, Any] - Схема параметров.
# @PRE GitPlugin is initialized.
# @POST Returns a JSON schema dictionary.
# @RETURN Dict[str, Any] - Схема параметров.
def get_schema(self) -> dict[str, Any]:
with belief_scope("GitPlugin.get_schema"):
return {
@@ -109,8 +109,8 @@ class GitPlugin(PluginBase):
# region execute [C:3] [TYPE Function]
# @PURPOSE: Main task executor with TaskContext support.
# @RELATION: CALLS -> self._handle_sync
# @RELATION: CALLS -> self._handle_deploy
# @RELATION CALLS -> self._handle_sync
# @RELATION CALLS -> self._handle_deploy
async def execute(self, task_data: dict[str, Any], context: TaskContext | None = None) -> dict[str, Any]:
with belief_scope("GitPlugin.execute"):
operation = task_data.get("operation")
@@ -143,14 +143,14 @@ class GitPlugin(PluginBase):
# region _handle_sync [C:4] [TYPE Function] [SEMANTICS git,sync,backup,transactional]
# @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий с backup/restore.
# @PRE: Репозиторий для дашборда должен существовать.
# @POST: Файлы в репозитории обновлены до текущего состояния в Superset.
# @PRE Репозиторий для дашборда должен существовать.
# @POST Файлы в репозитории обновлены до текущего состояния в Superset.
# Управляемые директории backup-ируются перед удалением; при ошибке распаковки backup восстанавливается.
# @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория.
# @SIDE_EFFECT Изменяет файлы в локальной рабочей директории репозитория.
# Создаёт временный backup в /tmp/ss-tools-backup-{dashboard_id}-{timestamp}/.
# @RETURN: Dict[str, str] - Результат синхронизации.
# @RELATION: CALLS -> src.services.git_service.GitService.get_repo
# @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard
# @RETURN Dict[str, str] - Результат синхронизации.
# @RELATION CALLS -> src.services.git_service.GitService.get_repo
# @RELATION CALLS -> src.core.superset_client.SupersetClient.export_dashboard
async def _handle_sync(self, dashboard_id: int, source_env_id: str | None = None, log=None, git_log=None, superset_log=None) -> dict[str, str]:
with belief_scope("GitPlugin._handle_sync"):
try:
@@ -247,9 +247,9 @@ class GitPlugin(PluginBase):
# region _handle_deploy [C:4] [TYPE Function] [SEMANTICS git,deploy,zip]
# @PURPOSE: Packages repository into ZIP and imports into target Superset environment.
# @POST: Dashboard imported into target Superset.
# @SIDE_EFFECT: Creates and removes temporary ZIP file.
# @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard
# @POST Dashboard imported into target Superset.
# @SIDE_EFFECT Creates and removes temporary ZIP file.
# @RELATION CALLS -> src.core.superset_client.SupersetClient.import_dashboard
async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> dict[str, Any]:
with belief_scope("GitPlugin._handle_deploy"):
try:
@@ -297,11 +297,11 @@ class GitPlugin(PluginBase):
# region _get_env [C:4] [TYPE Function] [SEMANTICS env,config,strict]
# @PURPOSE: Вспомогательный метод для получения конфигурации окружения.
# @PARAM: env_id (Optional[str]) - ID окружения.
# @PRE: env_id is a string or None.
# @POST: Returns an Environment object from config or DB.
# @PARAM env_id (Optional[str]) - ID окружения.
# @PRE env_id is a string or None.
# @POST Returns an Environment object from config or DB.
# When env_id is explicitly provided and not found, raises ValueError (no fallback).
# @RETURN: Environment - Объект конфигурации окружения.
# @RETURN Environment - Объект конфигурации окружения.
def _get_env(self, env_id: str | None = None):
with belief_scope("GitPlugin._get_env"):
app_logger.reason(f"Fetching environment for ID: {env_id}", extra={"src": "_get_env"})

14
backend/src/plugins/llm_analysis/__tests__/test_client_headers.py
View File

@@ -1,5 +1,5 @@
# region TestClientHeaders [TYPE Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @RELATION BELONGS_TO -> SrcRoot
# @SEMANTICS: tests, llm-client, openrouter, headers
# @PURPOSE: Verify OpenRouter client initialization includes provider-specific headers.
@@ -8,10 +8,10 @@ from src.plugins.llm_analysis.service import LLMClient
# region test_openrouter_client_includes_referer_and_title_headers [TYPE Function]
# @RELATION: BINDS_TO -> TestClientHeaders
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: OpenRouter requests should carry site/app attribution headers for compatibility.
# @PRE: Client is initialized for OPENROUTER provider.
# @POST: Async client headers include Authorization, HTTP-Referer, and X-Title.
# @PRE Client is initialized for OPENROUTER provider.
# @POST Async client headers include Authorization, HTTP-Referer, and X-Title.
def test_openrouter_client_includes_referer_and_title_headers(monkeypatch):
monkeypatch.setenv("OPENROUTER_SITE_URL", "http://localhost:8000")
monkeypatch.setenv("OPENROUTER_APP_NAME", "ss-tools-test")
@@ -31,10 +31,10 @@ def test_openrouter_client_includes_referer_and_title_headers(monkeypatch):
# region test_litellm_client_uses_default_bearer_auth [TYPE Function]
# @RELATION: BINDS_TO -> TestClientHeaders
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: LiteLLM proxy uses standard OpenAI-compatible Bearer auth — no special headers needed.
# @PRE: Client is initialized for LITELLM provider.
# @POST: Async client headers include only Authorization — no extra provider-specific headers.
# @PRE Client is initialized for LITELLM provider.
# @POST Async client headers include only Authorization — no extra provider-specific headers.
def test_litellm_client_uses_default_bearer_auth():
"""Verify LiteLLM client initialization uses standard Bearer auth without extra headers."""
client = LLMClient(

44
backend/src/plugins/llm_analysis/__tests__/test_screenshot_service.py
View File

@@ -1,5 +1,5 @@
# region TestScreenshotService [TYPE Module]
# @RELATION: VERIFIES ->[src.plugins.llm_analysis.service.ScreenshotService]
# @RELATION BINDS_TO ->[ScreenshotService]
# @SEMANTICS: tests, screenshot-service, navigation, timeout-regression
# @PURPOSE: Protect dashboard screenshot navigation from brittle networkidle waits.
@@ -9,10 +9,10 @@ from src.plugins.llm_analysis.service import ScreenshotService
# region test_iter_login_roots_includes_child_frames [TYPE Function]
# @RELATION: BINDS_TO ->[TestScreenshotService]
# @RELATION BINDS_TO ->[EXT:frontend:TestScreenshotService]
# @PURPOSE: Login discovery must search embedded auth frames, not only the main page.
# @PRE: Page exposes child frames list.
# @POST: Returned roots include page plus child frames in order.
# @PRE Page exposes child frames list.
# @POST Returned roots include page plus child frames in order.
def test_iter_login_roots_includes_child_frames():
frame_a = object()
frame_b = object()
@@ -28,10 +28,10 @@ def test_iter_login_roots_includes_child_frames():
# region test_response_looks_like_login_page_detects_login_markup [TYPE Function]
# @RELATION: BINDS_TO ->[TestScreenshotService]
# @RELATION BINDS_TO ->[EXT:frontend:TestScreenshotService]
# @PURPOSE: Direct login fallback must reject responses that render the login screen again.
# @PRE: Response body contains stable login-page markers.
# @POST: Helper returns True so caller treats fallback as failed authentication.
# @PRE Response body contains stable login-page markers.
# @POST Helper returns True so caller treats fallback as failed authentication.
def test_response_looks_like_login_page_detects_login_markup():
service = ScreenshotService(env=type("Env", (), {})())
@@ -55,10 +55,10 @@ def test_response_looks_like_login_page_detects_login_markup():
# region test_find_first_visible_locator_skips_hidden_first_match [TYPE Function]
# @RELATION: BINDS_TO ->[TestScreenshotService]
# @RELATION BINDS_TO ->[EXT:frontend:TestScreenshotService]
# @PURPOSE: Locator helper must not reject a selector collection just because its first element is hidden.
# @PRE: First matched element is hidden and second matched element is visible.
# @POST: Helper returns the second visible candidate.
# @PRE First matched element is hidden and second matched element is visible.
# @POST Helper returns the second visible candidate.
@pytest.mark.anyio
async def test_find_first_visible_locator_skips_hidden_first_match():
class _FakeElement:
@@ -96,10 +96,10 @@ async def test_find_first_visible_locator_skips_hidden_first_match():
# region test_submit_login_via_form_post_uses_browser_context_request [TYPE Function]
# @RELATION: BINDS_TO ->[TestScreenshotService]
# @RELATION BINDS_TO ->[EXT:frontend:TestScreenshotService]
# @PURPOSE: Fallback login must submit hidden fields and credentials through the context request cookie jar.
# @PRE: Login DOM exposes csrf hidden field and request context returns authenticated HTML.
# @POST: Helper returns True and request payload contains csrf_token plus credentials plus request options.
# @PRE Login DOM exposes csrf hidden field and request context returns authenticated HTML.
# @POST Helper returns True and request payload contains csrf_token plus credentials plus request options.
@pytest.mark.anyio
async def test_submit_login_via_form_post_uses_browser_context_request():
class _FakeInput:
@@ -207,10 +207,10 @@ async def test_submit_login_via_form_post_uses_browser_context_request():
# region test_submit_login_via_form_post_accepts_authenticated_redirect [TYPE Function]
# @RELATION: BINDS_TO ->[TestScreenshotService]
# @RELATION BINDS_TO ->[EXT:frontend:TestScreenshotService]
# @PURPOSE: Fallback login must treat non-login 302 redirect as success without waiting for redirect target.
# @PRE: Request response is 302 with Location outside login path.
# @POST: Helper returns True.
# @PRE Request response is 302 with Location outside login path.
# @POST Helper returns True.
@pytest.mark.anyio
async def test_submit_login_via_form_post_accepts_authenticated_redirect():
class _FakeInput:
@@ -282,10 +282,10 @@ async def test_submit_login_via_form_post_accepts_authenticated_redirect():
# region test_submit_login_via_form_post_rejects_login_markup_response [TYPE Function]
# @RELATION: BINDS_TO ->[TestScreenshotService]
# @RELATION BINDS_TO ->[EXT:frontend:TestScreenshotService]
# @PURPOSE: Fallback login must fail when POST response still contains login form content.
# @PRE: Login DOM exposes csrf hidden field and request response renders login markup.
# @POST: Helper returns False.
# @PRE Login DOM exposes csrf hidden field and request response renders login markup.
# @POST Helper returns False.
@pytest.mark.anyio
async def test_submit_login_via_form_post_rejects_login_markup_response():
class _FakeInput:
@@ -365,10 +365,10 @@ async def test_submit_login_via_form_post_rejects_login_markup_response():
# region test_goto_resilient_falls_back_from_domcontentloaded_to_load [TYPE Function]
# @RELATION: BINDS_TO ->[TestScreenshotService]
# @RELATION BINDS_TO ->[EXT:frontend:TestScreenshotService]
# @PURPOSE: Pages with unstable primary wait must retry with fallback wait strategy.
# @PRE: First page.goto call raises; second succeeds.
# @POST: Helper returns second response and attempts both wait modes in order.
# @PRE First page.goto call raises; second succeeds.
# @POST Helper returns second response and attempts both wait modes in order.
@pytest.mark.anyio
async def test_goto_resilient_falls_back_from_domcontentloaded_to_load():
class _FakePage:

14
backend/src/plugins/llm_analysis/__tests__/test_service.py
View File

@@ -1,5 +1,5 @@
# region TestService [TYPE Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @RELATION BELONGS_TO -> SrcRoot
# @SEMANTICS: tests, llm-analysis, fallback, provider-error, unknown-status
# @PURPOSE: Verify LLM analysis transport/provider failures do not masquerade as dashboard FAIL results.
@@ -10,10 +10,10 @@ from src.plugins.llm_analysis.service import LLMClient
# region test_test_runtime_connection_uses_json_completion_transport [TYPE Function]
# @RELATION: BINDS_TO -> TestService
# @RELATION BINDS_TO -> TestService
# @PURPOSE: Provider self-test must exercise the same chat completion transport as runtime analysis.
# @PRE: get_json_completion is available on initialized client.
# @POST: Self-test forwards a lightweight user message into get_json_completion and returns its payload.
# @PRE get_json_completion is available on initialized client.
# @POST Self-test forwards a lightweight user message into get_json_completion and returns its payload.
@pytest.mark.anyio
async def test_test_runtime_connection_uses_json_completion_transport(monkeypatch):
client = LLMClient(
@@ -39,10 +39,10 @@ async def test_test_runtime_connection_uses_json_completion_transport(monkeypatc
# region test_analyze_dashboard_provider_error_maps_to_unknown [TYPE Function]
# @RELATION: BINDS_TO -> TestService
# @RELATION BINDS_TO -> TestService
# @PURPOSE: Infrastructure/provider failures must produce UNKNOWN analysis status rather than FAIL.
# @PRE: LLMClient.get_json_completion raises provider/auth exception.
# @POST: Returned payload uses status=UNKNOWN and issue severity UNKNOWN.
# @PRE LLMClient.get_json_completion raises provider/auth exception.
# @POST Returned payload uses status=UNKNOWN and issue severity UNKNOWN.
@pytest.mark.anyio
async def test_analyze_dashboard_provider_error_maps_to_unknown(monkeypatch, tmp_path):
screenshot_path = tmp_path / "shot.jpg"

4
backend/src/plugins/llm_analysis/models.py
View File

@@ -1,7 +1,7 @@
# #region LLMAnalysisModels [C:3] [TYPE Module] [SEMANTICS pydantic, llm, plugin, model, provider-type]
# @BRIEF Define Pydantic models for LLM Analysis plugin.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> pydantic
# @LAYER Domain
# @RELATION DEPENDS_ON -> [EXT:Library:pydantic]
from datetime import datetime
from enum import Enum

40
backend/src/plugins/llm_analysis/plugin.py
View File

@@ -1,13 +1,13 @@
# #region LLMAnalysisPlugin [C:5] [TYPE Module] [SEMANTICS llm, analysis, dashboard, validation, documentation]
# @BRIEF Implements DashboardValidationPlugin and DocumentationPlugin.
# @LAYER: Plugin
# @LAYER Plugin
# @RELATION INHERITS -> [PluginBase]
# @RELATION CALLS -> [ScreenshotService]
# @RELATION CALLS -> [LLMClient]
# @RELATION CALLS -> [LLMProviderService]
# @RELATION USES -> TaskContext
# @INVARIANT: All LLM interactions must be executed as asynchronous tasks.
# @DATA_CONTRACT: AnalysisRequest -> AnalysisResult
# @RELATION DEPENDS_ON -> [TaskContext]
# @INVARIANT All LLM interactions must be executed as asynchronous tasks.
# @DATA_CONTRACT AnalysisRequest -> AnalysisResult
from datetime import datetime, timedelta
import json
@@ -33,8 +33,8 @@ from .service import LLMClient, ScreenshotService
# #region _is_masked_or_invalid_api_key [TYPE Function]
# @BRIEF Guards against placeholder or malformed API keys in runtime.
# @PRE: value may be None.
# @POST: Returns True when value cannot be used for authenticated provider calls.
# @PRE value may be None.
# @POST Returns True when value cannot be used for authenticated provider calls.
def _is_masked_or_invalid_api_key(value: str | None) -> bool:
key = (value or "").strip()
if not key:
@@ -47,8 +47,8 @@ def _is_masked_or_invalid_api_key(value: str | None) -> bool:
# #region _json_safe_value [TYPE Function]
# @BRIEF Recursively normalize payload values for JSON serialization.
# @PRE: value may be nested dict/list with datetime values.
# @POST: datetime values are converted to ISO strings.
# @PRE value may be nested dict/list with datetime values.
# @POST datetime values are converted to ISO strings.
def _json_safe_value(value: Any):
if isinstance(value, datetime):
return value.isoformat()
@@ -61,7 +61,7 @@ def _json_safe_value(value: Any):
# #region DashboardValidationPlugin [TYPE Class]
# @BRIEF Plugin for automated dashboard health analysis using LLMs.
# @RELATION IMPLEMENTS -> backend.src.core.plugin_base.PluginBase
# @RELATION IMPLEMENTS -> [EXT:path:backend.src.core.plugin_base.PluginBase]
class DashboardValidationPlugin(PluginBase):
@property
def id(self) -> str:
@@ -92,11 +92,11 @@ class DashboardValidationPlugin(PluginBase):
# region DashboardValidationPlugin.execute [TYPE Function]
# @PURPOSE: Executes the dashboard validation task with TaskContext support.
# @PARAM: params (Dict[str, Any]) - Validation parameters.
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE: params contains dashboard_id, environment_id, and provider_id.
# @POST: Returns a dictionary with validation results and persists them to the database.
# @SIDE_EFFECT: Captures a screenshot, calls LLM API, and writes to the database.
# @PARAM params (Dict[str, Any]) - Validation parameters.
# @PARAM context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE params contains dashboard_id, environment_id, and provider_id.
# @POST Returns a dictionary with validation results and persists them to the database.
# @SIDE_EFFECT Captures a screenshot, calls LLM API, and writes to the database.
async def execute(self, params: dict[str, Any], context: TaskContext | None = None):
with belief_scope("execute", f"plugin_id={self.id}"):
validation_started_at = datetime.utcnow()
@@ -326,7 +326,7 @@ class DashboardValidationPlugin(PluginBase):
# #region DocumentationPlugin [TYPE Class]
# @BRIEF Plugin for automated dataset documentation using LLMs.
# @RELATION IMPLEMENTS -> backend.src.core.plugin_base.PluginBase
# @RELATION IMPLEMENTS -> [EXT:path:backend.src.core.plugin_base.PluginBase]
class DocumentationPlugin(PluginBase):
@property
def id(self) -> str:
@@ -357,11 +357,11 @@ class DocumentationPlugin(PluginBase):
# region DocumentationPlugin.execute [TYPE Function]
# @PURPOSE: Executes the dataset documentation task with TaskContext support.
# @PARAM: params (Dict[str, Any]) - Documentation parameters.
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE: params contains dataset_id, environment_id, and provider_id.
# @POST: Returns generated documentation and updates the dataset in Superset.
# @SIDE_EFFECT: Calls LLM API and updates dataset metadata in Superset.
# @PARAM params (Dict[str, Any]) - Documentation parameters.
# @PARAM context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE params contains dataset_id, environment_id, and provider_id.
# @POST Returns generated documentation and updates the dataset in Superset.
# @SIDE_EFFECT Calls LLM API and updates dataset metadata in Superset.
async def execute(self, params: dict[str, Any], context: TaskContext | None = None):
with belief_scope("execute", f"plugin_id={self.id}"):
# Use TaskContext logger if available, otherwise fall back to app logger

4
backend/src/plugins/llm_analysis/scheduler.py
View File

@@ -1,6 +1,6 @@
# #region LLMAnalysisScheduler [C:3] [TYPE Module] [SEMANTICS scheduler, llm, validation, task, cron]
# @BRIEF Provides helper functions to schedule LLM-based validation tasks.
# @LAYER: Domain
# @LAYER Domain
# @RELATION DEPENDS_ON -> [SchedulerService]
from typing import Any
@@ -11,7 +11,7 @@ from ...dependencies import get_scheduler_service, get_task_manager
# #region schedule_dashboard_validation [TYPE Function]
# @BRIEF Schedules a recurring dashboard validation task.
# @SIDE_EFFECT: Adds a job to the scheduler service.
# @SIDE_EFFECT Adds a job to the scheduler service.
def schedule_dashboard_validation(dashboard_id: str, cron_expression: str, params: dict[str, Any]):
with belief_scope("schedule_dashboard_validation", f"dashboard_id={dashboard_id}"):
scheduler = get_scheduler_service()

6
backend/src/plugins/llm_analysis/service.py
View File

@@ -1,9 +1,9 @@
# #region LLMAnalysisService [C:5] [TYPE Module] [SEMANTICS llm, screenshot, playwright, openai, tenacity]
# @BRIEF Services for LLM interaction and dashboard screenshots.
# @LAYER Plugin
# @RELATION DEPENDS_ON -> tenacity
# @RELATION DEPENDS_ON -> tenacity
# @RELATION DEPENDS_ON -> tenacity
# @RELATION DEPENDS_ON -> [EXT:Library:tenacity]
# @RELATION DEPENDS_ON -> [EXT:Library:tenacity]
# @RELATION DEPENDS_ON -> [EXT:Library:tenacity]
# @INVARIANT Screenshots must be 1920px width and capture full page height.
# @DATA_CONTRACT DashboardSpec -> Screenshot + Analysis
# @RATIONALE Extracted all hardcoded timeouts into named module-level constants (PLAYWRIGHT_NAVIGATION_TIMEOUT_MS, PLAYWRIGHT_WAIT_TIMEOUT_MS, PLAYWRIGHT_SHORT_TIMEOUT_MS, HTTP_REQUEST_TIMEOUT_MS, SCREENSHOT_SERVICE_TIMEOUT_MS, LLM_HTTP_TIMEOUT_S) and DEFAULT_USER_AGENT. Zero remaining numeric timeout literals.

2
backend/src/plugins/maintenance_banner.py
View File

@@ -3,7 +3,7 @@
# Dispatches to MaintenanceService based on operation type in params.
# @LAYER Plugin
# @RELATION IMPLEMENTS -> [PluginBase]
# @RELATION DEPENDS_ON -> [MaintenanceServiceModule]
# @RELATION DEPENDS_ON -> [EXT:frontend:MaintenanceServiceModule]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [TaskContext]
# @INVARIANT TaskManager executes this plugin with params containing operation and event_id.

48
backend/src/plugins/mapper.py
View File

@@ -1,8 +1,8 @@
# #region MapperPluginModule [TYPE Module] [SEMANTICS mapper, dataset, column, sqllab, excel, mapping]
# @BRIEF Implements a plugin for mapping dataset columns using Superset SQL Lab or Excel files.
# @LAYER: Plugins
# @RELATION Inherits from PluginBase. Uses DatasetMapper and SupersetSqlLabExecutor.
# @RELATION USES -> TaskContext
# @LAYER Plugin
# @BRIEF Plugin for dataset column mapping. Inherits PluginBase.
# @RELATION DEPENDS_ON -> [TaskContext]
from typing import Any
@@ -23,9 +23,9 @@ class MapperPlugin(PluginBase):
@property
# region id [TYPE Function]
# @PURPOSE: Returns the unique identifier for the mapper plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string ID.
# @RETURN: str - "dataset-mapper"
# @PRE Plugin instance exists.
# @POST Returns string ID.
# @RETURN str - "dataset-mapper"
def id(self) -> str:
with belief_scope("id"):
return "dataset-mapper"
@@ -34,9 +34,9 @@ class MapperPlugin(PluginBase):
@property
# region name [TYPE Function]
# @PURPOSE: Returns the human-readable name of the mapper plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string name.
# @RETURN: str - Plugin name.
# @PRE Plugin instance exists.
# @POST Returns string name.
# @RETURN str - Plugin name.
def name(self) -> str:
with belief_scope("name"):
return "Dataset Mapper"
@@ -45,9 +45,9 @@ class MapperPlugin(PluginBase):
@property
# region description [TYPE Function]
# @PURPOSE: Returns a description of the mapper plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string description.
# @RETURN: str - Plugin description.
# @PRE Plugin instance exists.
# @POST Returns string description.
# @RETURN str - Plugin description.
def description(self) -> str:
with belief_scope("description"):
return "Map dataset column verbose names using Superset SQL Lab or Excel files."
@@ -56,9 +56,9 @@ class MapperPlugin(PluginBase):
@property
# region version [TYPE Function]
# @PURPOSE: Returns the version of the mapper plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string version.
# @RETURN: str - "1.0.0"
# @PRE Plugin instance exists.
# @POST Returns string version.
# @RETURN str - "1.0.0"
def version(self) -> str:
with belief_scope("version"):
return "1.0.0"
@@ -67,7 +67,7 @@ class MapperPlugin(PluginBase):
@property
# region ui_route [TYPE Function]
# @PURPOSE: Returns the frontend route for the mapper plugin.
# @RETURN: str - "/tools/mapper"
# @RETURN str - "/tools/mapper"
def ui_route(self) -> str:
with belief_scope("ui_route"):
return "/tools/mapper"
@@ -75,9 +75,9 @@ class MapperPlugin(PluginBase):
# region get_schema [TYPE Function]
# @PURPOSE: Returns the JSON schema for the mapper plugin parameters.
# @PRE: Plugin instance exists.
# @POST: Returns dictionary schema.
# @RETURN: Dict[str, Any] - JSON schema.
# @PRE Plugin instance exists.
# @POST Returns dictionary schema.
# @RETURN Dict[str, Any] - JSON schema.
def get_schema(self) -> dict[str, Any]:
with belief_scope("get_schema"):
return {
@@ -122,11 +122,11 @@ class MapperPlugin(PluginBase):
# region execute [TYPE Function]
# @PURPOSE: Executes the dataset mapping logic with TaskContext support.
# @PARAM: params (Dict[str, Any]) - Mapping parameters.
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE: Params contain valid 'env', 'dataset_id', and 'source'. params must be a dictionary.
# @POST: Updates the dataset in Superset.
# @RETURN: Dict[str, Any] - Execution status.
# @PARAM params (Dict[str, Any]) - Mapping parameters.
# @PARAM context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE Params contain valid 'env', 'dataset_id', and 'source'. params must be a dictionary.
# @POST Updates the dataset in Superset.
# @RETURN Dict[str, Any] - Execution status.
async def execute(self, params: dict[str, Any], context: TaskContext | None = None) -> dict[str, Any]:
with belief_scope("execute"):
env_name = params.get("env")

90
backend/src/plugins/migration.py
View File

@@ -1,16 +1,16 @@
# #region MigrationPlugin [C:5] [TYPE Module] [SEMANTICS migration, export, import, mapping, superset]
# @BRIEF Orchestrates export, DB-mapping transformation, and import of Superset dashboards across environments.
# @LAYER: App
# @LAYER App
# @RELATION IMPLEMENTS -> PluginBase
# @RELATION DEPENDS_ON -> SupersetClient
# @RELATION DEPENDS_ON -> MigrationEngine
# @RELATION DEPENDS_ON -> IdMappingService
# @RELATION USES -> TaskContext
# @PRE: Plugin loader can resolve infrastructure dependencies and execution requests provide validated migration context.
# @POST: Plugin metadata remains stable and migration execution preserves mapped-environment import guarantees.
# @SIDE_EFFECT: Reads config, opens database sessions, creates temporary artifacts, and triggers Superset export/import workflows.
# @DATA_CONTRACT: Input[TaskContext{from_env,to_env,dashboard_regex,replace_db_config,from_db_id,to_db_id,passwords?}] -> Output[MigrationResult|artifact set]
# @INVARIANT: Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution.
# @RELATION DEPENDS_ON -> [TaskContext]
# @PRE Plugin loader can resolve infrastructure dependencies and execution requests provide validated migration context.
# @POST Plugin metadata remains stable and migration execution preserves mapped-environment import guarantees.
# @SIDE_EFFECT Reads config, opens database sessions, creates temporary artifacts, and triggers Superset export/import workflows.
# @DATA_CONTRACT Input[TaskContext{from_env,to_env,dashboard_regex,replace_db_config,from_db_id,to_db_id,passwords?}] -> Output[MigrationResult|artifact set]
# @INVARIANT Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution.
import re
from typing import Any
@@ -29,14 +29,14 @@ from ..models.mapping import DatabaseMapping, Environment
# #region MigrationPlugin [TYPE Class]
# @BRIEF Implementation of the migration plugin workflow and transformation orchestration.
# @PRE: SupersetClient authenticated, database session active
# @POST: Returns MigrationResult with success/failure status and artifact list
# @TEST_FIXTURE: superset_export_zip -> file:backend/tests/fixtures/migration/dashboard_export.zip
# @TEST_FIXTURE: db_mapping_payload -> INLINE_JSON: {"db_mappings": {"source_uuid_1": "target_uuid_2"}}
# @TEST_FIXTURE: password_inject_payload -> INLINE_JSON: {"passwords": {"PostgreSQL": "secret123"}}
# @TEST_INVARIANT: strict_db_isolation -> VERIFIED_BY: [successful_dashboard_transfer, missing_mapping_resolution]
# @SIDE_EFFECT: Writes migration artifacts to database, triggers dashboard imports
# @DATA_CONTRACT: MigrationPlan AST, DryRunResult, RiskAssessment
# @PRE SupersetClient authenticated, database session active
# @POST Returns MigrationResult with success/failure status and artifact list
# @TEST_FIXTURE superset_export_zip -> file:backend/tests/fixtures/migration/dashboard_export.zip
# @TEST_FIXTURE db_mapping_payload -> INLINE_JSON: {"db_mappings": {"source_uuid_1": "target_uuid_2"}}
# @TEST_FIXTURE password_inject_payload -> INLINE_JSON: {"passwords": {"PostgreSQL": "secret123"}}
# @TEST_INVARIANT strict_db_isolation -> VERIFIED_BY: [successful_dashboard_transfer, missing_mapping_resolution]
# @SIDE_EFFECT Writes migration artifacts to database, triggers dashboard imports
# @DATA_CONTRACT MigrationPlan AST, DryRunResult, RiskAssessment
class MigrationPlugin(PluginBase):
"""
A plugin to migrate Superset dashboards between environments.
@@ -45,9 +45,9 @@ class MigrationPlugin(PluginBase):
@property
# region id [TYPE Function]
# @PURPOSE: Returns the unique identifier for the migration plugin.
# @PRE: None.
# @POST: Returns stable string "superset-migration".
# @RETURN: str
# @PRE None.
# @POST Returns stable string "superset-migration".
# @RETURN str
def id(self) -> str:
with belief_scope("MigrationPlugin.id"):
return "superset-migration"
@@ -56,9 +56,9 @@ class MigrationPlugin(PluginBase):
@property
# region name [TYPE Function]
# @PURPOSE: Returns the human-readable name of the plugin.
# @PRE: None.
# @POST: Returns "Superset Dashboard Migration".
# @RETURN: str
# @PRE None.
# @POST Returns "Superset Dashboard Migration".
# @RETURN str
def name(self) -> str:
with belief_scope("MigrationPlugin.name"):
return "Superset Dashboard Migration"
@@ -67,9 +67,9 @@ class MigrationPlugin(PluginBase):
@property
# region description [TYPE Function]
# @PURPOSE: Returns the semantic description of the plugin.
# @PRE: None.
# @POST: Returns description string.
# @RETURN: str
# @PRE None.
# @POST Returns description string.
# @RETURN str
def description(self) -> str:
with belief_scope("MigrationPlugin.description"):
return "Migrates dashboards between Superset environments."
@@ -78,9 +78,9 @@ class MigrationPlugin(PluginBase):
@property
# region version [TYPE Function]
# @PURPOSE: Returns the semantic version of the migration plugin.
# @PRE: None.
# @POST: Returns "1.0.0".
# @RETURN: str
# @PRE None.
# @POST Returns "1.0.0".
# @RETURN str
def version(self) -> str:
with belief_scope("MigrationPlugin.version"):
return "1.0.0"
@@ -89,9 +89,9 @@ class MigrationPlugin(PluginBase):
@property
# region ui_route [TYPE Function]
# @PURPOSE: Returns the frontend routing anchor for the plugin.
# @PRE: None.
# @POST: Returns "/migration".
# @RETURN: str
# @PRE None.
# @POST Returns "/migration".
# @RETURN str
def ui_route(self) -> str:
with belief_scope("MigrationPlugin.ui_route"):
return "/migration"
@@ -99,9 +99,9 @@ class MigrationPlugin(PluginBase):
# region get_schema [TYPE Function]
# @PURPOSE: Generates the JSON Schema for the plugin execution form dynamically.
# @PRE: ConfigManager is accessible and environments are defined.
# @POST: Returns a JSON Schema dict matching current system environments.
# @RETURN: Dict[str, Any]
# @PRE ConfigManager is accessible and environments are defined.
# @POST Returns a JSON Schema dict matching current system environments.
# @RETURN Dict[str, Any]
def get_schema(self) -> dict[str, Any]:
with belief_scope("MigrationPlugin.get_schema"):
app_logger.reason("Generating migration UI schema")
@@ -153,18 +153,18 @@ class MigrationPlugin(PluginBase):
# region execute [TYPE Function]
# @PURPOSE: Orchestrates the dashboard migration pipeline including extraction, AST mutation, and ingestion.
# @PARAM: params (Dict[str, Any]) - Extracted parameters from UI/API execution request.
# @PARAM: context (Optional[TaskContext]) - Dependency injected TaskContext for IO tracing.
# @PRE: Source and target environments must resolve. Matching dashboards must exist.
# @POST: Dashboard ZIP bundles are transformed and imported. ID mappings are synchronized.
# @SIDE_EFFECT: Creates temp files, mutates target Superset state, blocks on user input (passwords/mappings).
# @TEST_CONTRACT: Dict[str, Any] -> Dict[str, Any]
# @TEST_SCENARIO: successful_dashboard_transfer -> ZIP is downloaded, DB mappings applied via AST, target import succeeds.
# @TEST_SCENARIO: missing_password_injection -> Target import fails on auth, TaskManager pauses for user input, retries with password successfully.
# @TEST_SCENARIO: empty_selection -> Returns NO_MATCHES gracefully when regex finds zero dashboards.
# @TEST_EDGE: missing_env_field -> [ValueError: Could not resolve source or target environment]
# @TEST_EDGE: invalid_regex_pattern -> [Regex compilation exception is thrown or caught gracefully]
# @TEST_EDGE: target_api_timeout -> [Dashboard added to failed_dashboards, task concludes with PARTIAL_SUCCESS]
# @PARAM params (Dict[str, Any]) - Extracted parameters from UI/API execution request.
# @PARAM context (Optional[TaskContext]) - Dependency injected TaskContext for IO tracing.
# @PRE Source and target environments must resolve. Matching dashboards must exist.
# @POST Dashboard ZIP bundles are transformed and imported. ID mappings are synchronized.
# @SIDE_EFFECT Creates temp files, mutates target Superset state, blocks on user input (passwords/mappings).
# @TEST_CONTRACT Dict[str, Any] -> Dict[str, Any]
# @TEST_SCENARIO successful_dashboard_transfer -> ZIP is downloaded, DB mappings applied via AST, target import succeeds.
# @TEST_SCENARIO missing_password_injection -> Target import fails on auth, TaskManager pauses for user input, retries with password successfully.
# @TEST_SCENARIO empty_selection -> Returns NO_MATCHES gracefully when regex finds zero dashboards.
# @TEST_EDGE missing_env_field -> [ValueError: Could not resolve source or target environment]
# @TEST_EDGE invalid_regex_pattern -> [Regex compilation exception is thrown or caught gracefully]
# @TEST_EDGE target_api_timeout -> [Dashboard added to failed_dashboards, task concludes with PARTIAL_SUCCESS]
async def execute(self, params: dict[str, Any], context: TaskContext | None = None):
with belief_scope("MigrationPlugin.execute"):
app_logger.reason("Evaluating migration task parameters", extra={"params": params})

60
backend/src/plugins/search.py
View File

@@ -1,8 +1,8 @@
# #region SearchPluginModule [TYPE Module] [SEMANTICS search, dataset, text, pattern, superset]
# @BRIEF Implements a plugin for searching text patterns across all datasets in a specific Superset environment.
# @LAYER: Plugins
# @RELATION Inherits from PluginBase. Uses SupersetClient from core.
# @RELATION USES -> TaskContext
# @LAYER Plugin
# @BRIEF Plugin for text search across datasets. Inherits PluginBase.
# @RELATION DEPENDS_ON -> [TaskContext]
import re
from typing import Any
@@ -23,9 +23,9 @@ class SearchPlugin(PluginBase):
@property
# region id [TYPE Function]
# @PURPOSE: Returns the unique identifier for the search plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string ID.
# @RETURN: str - "search-datasets"
# @PRE Plugin instance exists.
# @POST Returns string ID.
# @RETURN str - "search-datasets"
def id(self) -> str:
with belief_scope("id"):
return "search-datasets"
@@ -34,9 +34,9 @@ class SearchPlugin(PluginBase):
@property
# region name [TYPE Function]
# @PURPOSE: Returns the human-readable name of the search plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string name.
# @RETURN: str - Plugin name.
# @PRE Plugin instance exists.
# @POST Returns string name.
# @RETURN str - Plugin name.
def name(self) -> str:
with belief_scope("name"):
return "Search Datasets"
@@ -45,9 +45,9 @@ class SearchPlugin(PluginBase):
@property
# region description [TYPE Function]
# @PURPOSE: Returns a description of the search plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string description.
# @RETURN: str - Plugin description.
# @PRE Plugin instance exists.
# @POST Returns string description.
# @RETURN str - Plugin description.
def description(self) -> str:
with belief_scope("description"):
return "Search for text patterns across all datasets in a specific environment."
@@ -56,9 +56,9 @@ class SearchPlugin(PluginBase):
@property
# region version [TYPE Function]
# @PURPOSE: Returns the version of the search plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string version.
# @RETURN: str - "1.0.0"
# @PRE Plugin instance exists.
# @POST Returns string version.
# @RETURN str - "1.0.0"
def version(self) -> str:
with belief_scope("version"):
return "1.0.0"
@@ -67,7 +67,7 @@ class SearchPlugin(PluginBase):
@property
# region ui_route [TYPE Function]
# @PURPOSE: Returns the frontend route for the search plugin.
# @RETURN: str - "/tools/search"
# @RETURN str - "/tools/search"
def ui_route(self) -> str:
with belief_scope("ui_route"):
return "/tools/search"
@@ -75,9 +75,9 @@ class SearchPlugin(PluginBase):
# region get_schema [TYPE Function]
# @PURPOSE: Returns the JSON schema for the search plugin parameters.
# @PRE: Plugin instance exists.
# @POST: Returns dictionary schema.
# @RETURN: Dict[str, Any] - JSON schema.
# @PRE Plugin instance exists.
# @POST Returns dictionary schema.
# @RETURN Dict[str, Any] - JSON schema.
def get_schema(self) -> dict[str, Any]:
with belief_scope("get_schema"):
return {
@@ -100,11 +100,11 @@ class SearchPlugin(PluginBase):
# region execute [TYPE Function]
# @PURPOSE: Executes the dataset search logic with TaskContext support.
# @PARAM: params (Dict[str, Any]) - Search parameters.
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE: Params contain valid 'env' and 'query'.
# @POST: Returns a dictionary with count and results list.
# @RETURN: Dict[str, Any] - Search results.
# @PARAM params (Dict[str, Any]) - Search parameters.
# @PARAM context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE Params contain valid 'env' and 'query'.
# @POST Returns a dictionary with count and results list.
# @RETURN Dict[str, Any] - Search results.
async def execute(self, params: dict[str, Any], context: TaskContext | None = None) -> dict[str, Any]:
with belief_scope("SearchPlugin.execute", f"params={params}"):
env_name = params.get("env")
@@ -179,12 +179,12 @@ class SearchPlugin(PluginBase):
# region _get_context [TYPE Function]
# @PURPOSE: Extracts a small context around the match for display.
# @PARAM: text (str) - The full text to extract context from.
# @PARAM: match_text (str) - The matched text pattern.
# @PARAM: context_lines (int) - Number of lines of context to include.
# @PRE: text and match_text must be strings.
# @POST: Returns context string.
# @RETURN: str - Extracted context.
# @PARAM text (str) - The full text to extract context from.
# @PARAM match_text (str) - The matched text pattern.
# @PARAM context_lines (int) - Number of lines of context to include.
# @PRE text and match_text must be strings.
# @POST Returns context string.
# @RETURN str - Extracted context.
def _get_context(self, text: str, match_text: str, context_lines: int = 1) -> str:
"""
Extracts a small context around the match for display.

6
backend/src/plugins/storage/plugin.py
View File

@@ -1,9 +1,9 @@
# #region StoragePlugin [TYPE Module] [SEMANTICS fastapi, storage, filesystem, backup, archive]
# @BRIEF Provides core filesystem operations for managing backups and repositories.
# @LAYER App
# @RELATION USES -> TaskContext
# @RELATION USES -> TaskContext
# @RELATION USES -> TaskContext
# @RELATION DEPENDS_ON -> [TaskContext]
# @RELATION DEPENDS_ON -> [TaskContext]
# @RELATION DEPENDS_ON -> [TaskContext]
# @INVARIANT All file operations must be restricted to the configured storage root.
# @RATIONALE Replaced Path(__file__).parents[3] with BASE_DIR import from database.py for path resolution consistency.

18
backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py
View File

@@ -1,15 +1,15 @@
# region ClickHouseInsertIntegration [TYPE Module]
# @SEMANTICS: test, clickhouse, integration, insert, join
# @PURPOSE: Integration tests for ClickHouse INSERT with timestamp normalization and JOIN verification.
# @LAYER: Test
# @RELATION: BINDS_TO -> [SQLGenerator:Module]
# @RELATION: BINDS_TO -> [TranslationOrchestrator]
# @TEST_CONTRACT: SQLGenerator.generate(clickhouse) -> valid INSERT SQL with YYYY-MM-DD dates
# @TEST_SCENARIO: timestamp_in_date_column -> ClickHouse parses date correctly
# @TEST_SCENARIO: join_after_insert -> financial_comments_translated JOIN financial_arrears works
# @TEST_EDGE: unix_millis_timestamp -> converted to YYYY-MM-DD
# @TEST_EDGE: unix_seconds_timestamp -> converted to YYYY-MM-DD
# @TEST_EDGE: non_timestamp_string -> passed through unchanged
# @LAYER Test
# @RELATION BINDS_TO -> [SQLGenerator]
# @RELATION BINDS_TO -> [TranslationOrchestrator]
# @TEST_CONTRACT SQLGenerator.generate(clickhouse) -> valid INSERT SQL with YYYY-MM-DD dates
# @TEST_SCENARIO timestamp_in_date_column -> ClickHouse parses date correctly
# @TEST_SCENARIO join_after_insert -> financial_comments_translated JOIN financial_arrears works
# @TEST_EDGE unix_millis_timestamp -> converted to YYYY-MM-DD
# @TEST_EDGE unix_seconds_timestamp -> converted to YYYY-MM-DD
# @TEST_EDGE non_timestamp_string -> passed through unchanged
import logging

4
backend/src/plugins/translate/__tests__/test_dictionary.py
View File

@@ -5,7 +5,7 @@
# test_dictionary_filter.py — Batch filter + migration
# test_dictionary_correction.py — Correction context capture
# test_dictionary_prompt_builder.py — Prompt builder operations
# test_dictionary_utils.py — Utility functions
# test_dictionary[EXT:internal:_utils].py — Utility functions
# @RELATION BINDS_TO -> [DictionaryManager]
# @RATIONALE Split monolithic test module into domain-specific files per INV_7.
# @REJECTED Keeping 1199-line test module violates module < 400 lines constraint.
@@ -16,5 +16,5 @@ from .test_dictionary_crud import * # noqa: F401, F403
from .test_dictionary_filter import * # noqa: F401, F403
from .test_dictionary_import import * # noqa: F401, F403
from .test_dictionary_prompt_builder import * # noqa: F401, F403
from .test_dictionary_utils import * # noqa: F401, F403
from .test_dictionary[EXT:internal:_utils] import * # noqa: F401, F403
# #endregion TestDictionaryLegacyHub

36
backend/src/plugins/translate/__tests__/test_dictionary_crud.py
View File

@@ -2,10 +2,10 @@
# @BRIEF Validate DictionaryCRUD and DictionaryEntryCRUD operations.
# @RELATION BINDS_TO -> [DictionaryCRUD]
# @RELATION BINDS_TO -> [DictionaryEntryCRUD]
# @TEST_EDGE: duplicate_entry -> ValueError on repeated (dictionary_id, source_term_norm, source_lang, target_lang)
# @TEST_EDGE: delete_active_job -> ValueError with active/scheduled message
# @TEST_EDGE: same_term_different_lang_pair -> allowed (not duplicate)
# @TEST_INVARIANT: unique_normalized -> verifies: [duplicate_entry, same_term_different_lang_pair allowed]
# @TEST_EDGE duplicate_entry -> ValueError on repeated (dictionary_id, source_term_norm, source_lang, target_lang)
# @TEST_EDGE delete_active_job -> ValueError with active/scheduled message
# @TEST_EDGE same_term_different_lang_pair -> allowed (not duplicate)
# @TEST_INVARIANT unique_normalized -> verifies: [duplicate_entry, same_term_different_lang_pair allowed]
import pytest
@@ -42,7 +42,7 @@ class TestDictionaryCRUD:
"""Verify dictionary-level CRUD operations."""
# region test_create_dictionary [C:2] [TYPE Function]
# @BRIEF: Verify dictionary creation and read-back.
# @BRIEF Verify dictionary creation and read-back.
def test_create_dictionary(self, db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Finance Terms",
@@ -62,7 +62,7 @@ class TestDictionaryCRUD:
# endregion test_create_dictionary
# region test_update_dictionary [C:2] [TYPE Function]
# @BRIEF: Verify dictionary metadata update.
# @BRIEF Verify dictionary metadata update.
def test_update_dictionary(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Old Name", source_dialect="a", target_dialect="b")
updated = DictionaryManager.update_dictionary(
@@ -74,7 +74,7 @@ class TestDictionaryCRUD:
# endregion test_update_dictionary
# region test_delete_dictionary [C:2] [TYPE Function]
# @BRIEF: Verify dictionary deletion also removes entries.
# @BRIEF Verify dictionary deletion also removes entries.
def test_delete_dictionary(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="To Delete", source_dialect="a", target_dialect="b")
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
@@ -85,7 +85,7 @@ class TestDictionaryCRUD:
# endregion test_delete_dictionary
# region test_list_dictionaries [C:2] [TYPE Function]
# @BRIEF: Verify paginated dictionary listing.
# @BRIEF Verify paginated dictionary listing.
def test_list_dictionaries(self, db_session: Session):
for i in range(5):
DictionaryManager.create_dictionary(db_session, name=f"Dict {i}", source_dialect="a", target_dialect="b")
@@ -95,7 +95,7 @@ class TestDictionaryCRUD:
# endregion test_list_dictionaries
# region test_delete_dictionary_blocked_by_active_job [C:2] [TYPE Function]
# @BRIEF: Verify deletion is blocked when attached to active/scheduled jobs.
# @BRIEF Verify deletion is blocked when attached to active/scheduled jobs.
def test_delete_dictionary_blocked_by_active_job(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Test", source_dialect="a", target_dialect="b")
job = TranslationJob(name="Active Job", source_dialect="a", target_dialect="b", status="ACTIVE", created_by="test_user")
@@ -109,7 +109,7 @@ class TestDictionaryCRUD:
# endregion test_delete_dictionary_blocked_by_active_job
# region test_delete_dictionary_allowed_with_completed_job [C:2] [TYPE Function]
# @BRIEF: Verify deletion is allowed when only completed/failed jobs reference the dictionary.
# @BRIEF Verify deletion is allowed when only completed/failed jobs reference the dictionary.
def test_delete_dictionary_allowed_with_completed_job(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Test", source_dialect="a", target_dialect="b")
job = TranslationJob(name="Completed Job", source_dialect="a", target_dialect="b", status="COMPLETED", created_by="test_user")
@@ -128,7 +128,7 @@ class TestDictionaryEntryCRUD:
"""Verify entry-level CRUD operations."""
# region test_add_entry_duplicate [C:2] [TYPE Function]
# @BRIEF: Verify duplicate entry raises ValueError.
# @BRIEF Verify duplicate entry raises ValueError.
def test_add_entry_duplicate(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Test", source_dialect="a", target_dialect="b")
DictionaryManager.add_entry(db_session, d.id, "Hello", "Hola", source_language="en", target_language="es")
@@ -139,7 +139,7 @@ class TestDictionaryEntryCRUD:
# endregion test_add_entry_duplicate
# region test_add_entry_duplicate_per_dictionary [C:2] [TYPE Function]
# @BRIEF: Verify duplicate is per-dictionary (same term in different dicts is OK).
# @BRIEF Verify duplicate is per-dictionary (same term in different dicts is OK).
def test_add_entry_duplicate_per_dictionary(self, db_session: Session):
d1 = DictionaryManager.create_dictionary(db_session, name="Dict1", source_dialect="a", target_dialect="b")
d2 = DictionaryManager.create_dictionary(db_session, name="Dict2", source_dialect="a", target_dialect="b")
@@ -149,7 +149,7 @@ class TestDictionaryEntryCRUD:
# endregion test_add_entry_duplicate_per_dictionary
# region test_edit_entry [C:2] [TYPE Function]
# @BRIEF: Verify entry edit updates fields and enforces uniqueness.
# @BRIEF Verify entry edit updates fields and enforces uniqueness.
def test_edit_entry(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Test", source_dialect="a", target_dialect="b")
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
@@ -161,7 +161,7 @@ class TestDictionaryEntryCRUD:
# endregion test_edit_entry
# region test_delete_entry [C:2] [TYPE Function]
# @BRIEF: Verify entry deletion.
# @BRIEF Verify entry deletion.
def test_delete_entry(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Test", source_dialect="a", target_dialect="b")
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
@@ -171,7 +171,7 @@ class TestDictionaryEntryCRUD:
# endregion test_delete_entry
# region test_clear_entries [C:2] [TYPE Function]
# @BRIEF: Verify clearing all entries for a dictionary.
# @BRIEF Verify clearing all entries for a dictionary.
def test_clear_entries(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Test", source_dialect="a", target_dialect="b")
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
@@ -183,7 +183,7 @@ class TestDictionaryEntryCRUD:
# endregion test_clear_entries
# region test_add_entry_with_language_pair [C:2] [TYPE Function]
# @BRIEF: Verify creating entry with language pair stores correctly.
# @BRIEF Verify creating entry with language pair stores correctly.
def test_add_entry_with_language_pair(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Lang Test", source_dialect="a", target_dialect="b")
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")
@@ -196,7 +196,7 @@ class TestDictionaryEntryCRUD:
# endregion test_add_entry_with_language_pair
# region test_duplicate_same_language_pair [C:2] [TYPE Function]
# @BRIEF: Verify duplicate with same language pair raises conflict.
# @BRIEF Verify duplicate with same language pair raises conflict.
def test_duplicate_same_language_pair(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Dup Test", source_dialect="a", target_dialect="b")
DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")
@@ -205,7 +205,7 @@ class TestDictionaryEntryCRUD:
# endregion test_duplicate_same_language_pair
# region test_same_term_different_language_pair [C:2] [TYPE Function]
# @BRIEF: Verify same term with different language pair is allowed.
# @BRIEF Verify same term with different language pair is allowed.
def test_same_term_different_language_pair(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Multi Lang", source_dialect="a", target_dialect="b")
entry1 = DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")

2
backend/src/plugins/translate/__tests__/test_dictionary_import.py
View File

@@ -1,7 +1,7 @@
# #region TestDictionaryImport [C:3] [TYPE Module] [SEMANTICS test, dictionary, import, export]
# @BRIEF Validate DictionaryImportExport operations.
# @RELATION BINDS_TO -> [DictionaryImportExport]
# @TEST_EDGE: import_invalid_format -> ValueError for missing required columns
# @TEST_EDGE import_invalid_format -> ValueError for missing required columns
import csv
import io

4
backend/src/plugins/translate/__tests__/test_dictionary_utils.py
View File

@@ -1,8 +1,8 @@
# #region TestDictionaryUtils [C:3] [TYPE Module] [SEMANTICS test, dictionary, utils, normalization]
# @BRIEF Validate utility functions: _normalize_term and _detect_delimiter.
# @RELATION BINDS_TO -> [_utils]
# @RELATION BINDS_TO -> [[EXT:internal:_utils]]
from src.plugins.translate._utils import _detect_delimiter, _normalize_term
from src.plugins.translate.[EXT:internal:_utils] import _detect_delimiter, _normalize_term
class TestNormalizeTerm:

16
backend/src/plugins/translate/__tests__/test_executor.py
View File

@@ -1,11 +1,11 @@
# region ExecutorTests [TYPE Module]
# @SEMANTICS: test, translate, executor, null-handling, cancellation
# @PURPOSE: Tests for TranslationExecutor: null content handling, cancellation flag during execution.
# @LAYER: Test
# @RELATION: BINDS_TO -> [TranslationExecutor:Module]
# @TEST_CONTRACT: TranslationExecutor -> execute_run, _call_openai_compatible
# @TEST_EDGE: null_llm_content -> raises ValueError instead of TypeError
# @TEST_EDGE: cancellation_flag_during_execution -> stops batch processing
# @LAYER Test
# @RELATION BINDS_TO -> [TranslationExecutor]
# @TEST_CONTRACT TranslationExecutor -> execute_run, _call_openai_compatible
# @TEST_EDGE null_llm_content -> raises ValueError instead of TypeError
# @TEST_EDGE cancellation_flag_during_execution -> stops batch processing
import pytest
from unittest.mock import MagicMock, patch
@@ -267,7 +267,7 @@ class TestCancellationFlag:
# region TestEstimateRowTokens [TYPE Class]
# @PURPOSE: Tests for estimate_row_tokens — per-row token estimation for adaptive batch sizing.
# @RELATION: BINDS_TO -> [estimate_row_tokens]
# @RELATION BINDS_TO -> [estimate_row_tokens]
class TestEstimateRowTokens:
"""Unit tests for estimate_row_tokens()."""
@@ -344,8 +344,8 @@ class TestEstimateRowTokens:
# region TestAutoSizeBatches [TYPE Class]
# @PURPOSE: Tests for _auto_size_batches — variable-sized batch splitting based on content length.
# @RELATION: BINDS_TO -> [TranslationExecutor._auto_size_batches]
# @RELATION: BINDS_TO -> [estimate_token_budget]
# @RELATION BINDS_TO -> [EXT:method:TranslationExecutor._auto_size_batches]
# @RELATION BINDS_TO -> [estimate_token_budget]
class TestAutoSizeBatches:
"""Tests for TranslationExecutor._auto_size_batches()."""

44
backend/src/plugins/translate/__tests__/test_inline_correction.py
View File

@@ -1,15 +1,15 @@
# region InlineCorrectionTests [TYPE Module]
# @SEMANTICS: test, translate, correction, inline, bulk
# @PURPOSE: Tests for inline correction (T109-T116, T117): single correction, dictionary submission, bulk replace.
# @LAYER: Domain
# @RELATION: BINDS_TO -> [InlineCorrectionService:Module]
# @RELATION: BINDS_TO -> [BulkFindReplaceService:Module]
# @TEST_CONTRACT: InlineCorrectionService -> submit_correction, duplicate detection
# @TEST_CONTRACT: BulkFindReplaceService -> preview, apply, atomic
# @TEST_EDGE: single_correction -> Creates dictionary entry with origin tracking
# @TEST_EDGE: duplicate_detection -> Conflict detected for existing term
# @TEST_EDGE: bulk_replace_preview -> Returns accurate affected records
# @TEST_EDGE: bulk_replace_apply -> Updates values and optionally submits to dictionary
# @LAYER Domain
# @RELATION BINDS_TO -> [InlineCorrectionService]
# @RELATION BINDS_TO -> [BulkFindReplaceService]
# @TEST_CONTRACT InlineCorrectionService -> submit_correction, duplicate detection
# @TEST_CONTRACT BulkFindReplaceService -> preview, apply, atomic
# @TEST_EDGE single_correction -> Creates dictionary entry with origin tracking
# @TEST_EDGE duplicate_detection -> Conflict detected for existing term
# @TEST_EDGE bulk_replace_preview -> Returns accurate affected records
# @TEST_EDGE bulk_replace_apply -> Updates values and optionally submits to dictionary
from unittest.mock import MagicMock, patch
@@ -282,19 +282,19 @@ class TestBulkFindReplaceService:
# region TestContextAwareCorrection [TYPE Module]
# @SEMANTICS: test, translate, correction, context, jaccard, priority
# @PURPOSE: Tests for context-aware correction (T132): context capture, Jaccard similarity, priority flagging, truncation, context_source tagging.
# @LAYER: Test
# @RELATION: BINDS_TO -> [InlineCorrectionService:Class]
# @RELATION: BINDS_TO -> [ContextAwarePromptBuilder:Class]
# @TEST_CONTRACT: InlineCorrectionService -> context capture from source row, context editing/removal
# @TEST_CONTRACT: ContextAwarePromptBuilder -> Jaccard similarity, priority flagging, truncation, render_entry
# @TEST_EDGE: context_capture -> Auto-populates context_data from source row columns
# @TEST_EDGE: context_removal -> keep_context=false clears context_data and sets has_context=False
# @TEST_EDGE: jaccard_zero -> 0% overlap returns 0.0
# @TEST_EDGE: jaccard_half -> 50% overlap returns 0.5
# @TEST_EDGE: jaccard_full -> 100% overlap returns 1.0
# @TEST_EDGE: priority_flagging -> Similarity >=0.5 triggers priority prefix
# @TEST_EDGE: truncation_500 -> Context rendering capped at ~500 tokens with annotation
# @TEST_EDGE: context_source_tagging -> auto/bulk/manual tags set correctly
# @LAYER Test
# @RELATION BINDS_TO -> [InlineCorrectionService]
# @RELATION BINDS_TO -> [ContextAwarePromptBuilder]
# @TEST_CONTRACT InlineCorrectionService -> context capture from source row, context editing/removal
# @TEST_CONTRACT ContextAwarePromptBuilder -> Jaccard similarity, priority flagging, truncation, render_entry
# @TEST_EDGE context_capture -> Auto-populates context_data from source row columns
# @TEST_EDGE context_removal -> keep_context=false clears context_data and sets has_context=False
# @TEST_EDGE jaccard_zero -> 0% overlap returns 0.0
# @TEST_EDGE jaccard_half -> 50% overlap returns 0.5
# @TEST_EDGE jaccard_full -> 100% overlap returns 1.0
# @TEST_EDGE priority_flagging -> Similarity >=0.5 triggers priority prefix
# @TEST_EDGE truncation_500 -> Context rendering capped at ~500 tokens with annotation
# @TEST_EDGE context_source_tagging -> auto/bulk/manual tags set correctly
from unittest.mock import MagicMock

18
backend/src/plugins/translate/__tests__/test_orchestrator.py
View File

@@ -1,15 +1,15 @@
# region OrchestratorTests [TYPE Module]
# @SEMANTICS: test, translate, orchestrator, events
# @PURPOSE: Tests for TranslationOrchestrator: run lifecycle, partial failure, batch retry, event invariants, NULL handling.
# @LAYER: Test
# @RELATION: BINDS_TO -> [TranslationOrchestrator:Module]
# @RELATION: BINDS_TO -> [TranslationEventLog:Module]
# @TEST_CONTRACT: TranslationOrchestrator -> start_run, execute_run, cancel_run, retry_failed_batches
# @TEST_FIXTURE: mock_db -> MagicMock SQLAlchemy session
# @TEST_FIXTURE: mock_config_manager -> MagicMock ConfigManager
# @TEST_EDGE: missing_preview -> raises ValueError
# @TEST_EDGE: invalid_run_status -> raises ValueError
# @TEST_EDGE: executor_failure -> run is marked FAILED
# @LAYER Test
# @RELATION BINDS_TO -> [TranslationOrchestrator]
# @RELATION BINDS_TO -> [TranslationEventLog]
# @TEST_CONTRACT TranslationOrchestrator -> start_run, execute_run, cancel_run, retry_failed_batches
# @TEST_FIXTURE mock_db -> MagicMock SQLAlchemy session
# @TEST_FIXTURE mock_config_manager -> MagicMock ConfigManager
# @TEST_EDGE missing_preview -> raises ValueError
# @TEST_EDGE invalid_run_status -> raises ValueError
# @TEST_EDGE executor_failure -> run is marked FAILED
from datetime import UTC, datetime
import json

20
backend/src/plugins/translate/__tests__/test_orthogonal_fixes.py
View File

@@ -1,18 +1,18 @@
# region OrthogonalTranslationFixes [TYPE Module] [SEMANTICS test, translate, orthogonal, verification]
# @BRIEF Orthogonal verification of translation system fixes: cancel lock timeout, null content, periodic commit, async->def migration.
# @LAYER: Test
# @LAYER Test
# @RELATION BINDS_TO -> [TranslationOrchestrator]
# @RELATION BINDS_TO -> [TranslationExecutor]
# @RELATION BINDS_TO -> [TranslateRunRoutesModule]
# @TEST_EDGE: cancel_lock_timeout -> lock_timeout expiry falls back to CANCEL_REQUESTED flag
# @TEST_EDGE: cancel_nonexistent_run -> raises ValueError for missing run
# @TEST_EDGE: cancel_pending_run -> PENDING runs can be cancelled
# @TEST_EDGE: cancel_completed_at_set -> completed_at is populated after cancel
# @TEST_EDGE: cancel_event_log -> RUN_CANCELLED event exists in event log
# @TEST_EDGE: execute_cancelled_after_executor -> orchestrator handles CANCELLED status from executor
# @TEST_EDGE: execute_zero_rows -> orchestrator handles zero-row completion
# @TEST_EDGE: async_to_def_routes -> sync route handlers work with FastAPI TestClient
# @TEST_EDGE: no_await_in_routes -> no accidental await in sync handlers
# @TEST_EDGE cancel_lock_timeout -> lock_timeout expiry falls back to CANCEL_REQUESTED flag
# @TEST_EDGE cancel_nonexistent_run -> raises ValueError for missing run
# @TEST_EDGE cancel_pending_run -> PENDING runs can be cancelled
# @TEST_EDGE cancel_completed_at_set -> completed_at is populated after cancel
# @TEST_EDGE cancel_event_log -> RUN_CANCELLED event exists in event log
# @TEST_EDGE execute_cancelled_after_executor -> orchestrator handles CANCELLED status from executor
# @TEST_EDGE execute_zero_rows -> orchestrator handles zero-row completion
# @TEST_EDGE async_to_def_routes -> sync route handlers work with FastAPI TestClient
# @TEST_EDGE no_await_in_routes -> no accidental await in sync handlers
from datetime import UTC, datetime
import pytest

4
backend/src/plugins/translate/__tests__/test_preview.py
View File

@@ -1,8 +1,8 @@
# region TranslationPreviewTests [TYPE Module]
# @SEMANTICS: test, translate, preview, session
# @PURPOSE: Test TranslationPreview service: preview, approve/edit/reject state transitions, cost estimation, acceptance gate.
# @LAYER: Test
# @RELATION: BINDS_TO -> [TranslationPreview:Module]
# @LAYER Test
# @RELATION BINDS_TO -> [TranslationPreview]
from datetime import UTC, datetime, timedelta
import json

16
backend/src/plugins/translate/__tests__/test_scheduler.py
View File

@@ -1,14 +1,14 @@
# region TestScheduler [TYPE Module]
# @SEMANTICS: test, translate, scheduler, notification
# @PURPOSE: Tests for TranslationScheduler: CRUD, cron validation, trigger dispatch, failure notification.
# @LAYER: Test
# @RELATION: BINDS_TO -> [TranslationScheduler:Module]
# @TEST_CONTRACT: TranslationScheduler -> create, update, delete, get, get_next_executions
# @TEST_CONTRACT: execute_scheduled_translation -> failure notification, concurrency check
# @TEST_EDGE: missing_schedule -> raise ValueError
# @TEST_EDGE: concurrent_run_skip -> skip and log event
# @TEST_EDGE: execution_failure -> NotificationService called
# @TEST_EDGE: execution_success -> NotificationService NOT called
# @LAYER Test
# @RELATION BINDS_TO -> [TranslationScheduler]
# @TEST_CONTRACT TranslationScheduler -> create, update, delete, get, get_next_executions
# @TEST_CONTRACT execute_scheduled_translation -> failure notification, concurrency check
# @TEST_EDGE missing_schedule -> raise ValueError
# @TEST_EDGE concurrent_run_skip -> skip and log event
# @TEST_EDGE execution_failure -> NotificationService called
# @TEST_EDGE execution_success -> NotificationService NOT called
from datetime import UTC, datetime
import pytest

14
backend/src/plugins/translate/__tests__/test_sql_generator.py
View File

@@ -1,13 +1,13 @@
# region SQLGeneratorTests [TYPE Module]
# @SEMANTICS: test, translate, sql_generator
# @PURPOSE: Tests for SQLGenerator: PostgreSQL INSERT + UPSERT, ClickHouse INSERT, dialect quoting, NULL handling, injection safety.
# @LAYER: Test
# @RELATION: BINDS_TO -> [SQLGenerator:Module]
# @TEST_CONTRACT: SQLGenerator.generate -> SQL string, row_count
# @TEST_FIXTURE: sample_rows -> [{"col1": "val1", "col2": 42}, {"col1": "val2", "col2": 99}]
# @TEST_EDGE: empty_rows -> raises ValueError
# @TEST_EDGE: null_values -> properly encoded as NULL
# @TEST_EDGE: sql_injection -> values with single quotes are escaped
# @LAYER Test
# @RELATION BINDS_TO -> [SQLGenerator]
# @TEST_CONTRACT SQLGenerator.generate -> SQL string, row_count
# @TEST_FIXTURE sample_rows -> [{"col1": "val1", "col2": 42}, {"col1": "val2", "col2": 99}]
# @TEST_EDGE empty_rows -> raises ValueError
# @TEST_EDGE null_values -> properly encoded as NULL
# @TEST_EDGE sql_injection -> values with single quotes are escaped
import pytest
from typing import Any

12
backend/src/plugins/translate/__tests__/test_target_schema.py
View File

@@ -1,12 +1,12 @@
# region TargetSchemaValidationTests [TYPE Module]
# @SEMANTICS: test, translate, target-schema, validation
# @PURPOSE: Tests for target table schema validation.
# @LAYER: Test
# @RELATION: BINDS_TO -> [TargetSchemaValidation]
# @TEST_CONTRACT: _build_expected_columns -> list[TargetSchemaColumnInfo]
# @TEST_CONTRACT: _extract_columns_from_rows -> list[dict]
# @TEST_CONTRACT: _parse_sqllab_result -> (list[dict], bool)
# @TEST_CONTRACT: validate_target_table_schema -> TargetSchemaValidationResponse
# @LAYER Test
# @RELATION BINDS_TO -> [TargetSchemaValidation]
# @TEST_CONTRACT _build_expected_columns -> list[TargetSchemaColumnInfo]
# @TEST_CONTRACT _extract_columns_from_rows -> list[dict]
# @TEST_CONTRACT _parse_sqllab_result -> (list[dict], bool)
# @TEST_CONTRACT validate_target_table_schema -> TargetSchemaValidationResponse
import pytest
from unittest.mock import MagicMock, patch

14
backend/src/plugins/translate/__tests__/test_text_cleaner.py
View File

@@ -1,13 +1,13 @@
# #region TestTextCleaner [C:3] [TYPE Module] [SEMANTICS test, text, cleaner, whitespace, truncation]
# @BRIEF Verify text cleaning contracts — normalize_whitespace, truncate_text, clean_text.
# @RELATION BINDS_TO -> [_text_cleaner:Module]
# @TEST_EDGE: empty_string — empty/whitespace-only input returns ""
# @TEST_EDGE: whitespace_variants — multiple spaces, newlines, tabs all collapse to single space
# @TEST_EDGE: truncation_boundary — text shorter than, equal to, and longer than max_length
# @TEST_EDGE: clean_combined — clean_text correctly chains normalize + truncate
# @TEST_EDGE: zero_max_length — max_length=0 forces truncation on any non-empty text
# @RELATION BINDS_TO -> [[EXT:internal:_text_cleaner]]
# @TEST_EDGE empty_string — empty/whitespace-only input returns ""
# @TEST_EDGE whitespace_variants — multiple spaces, newlines, tabs all collapse to single space
# @TEST_EDGE truncation_boundary — text shorter than, equal to, and longer than max_length
# @TEST_EDGE clean_combined — clean_text correctly chains normalize + truncate
# @TEST_EDGE zero_max_length — max_length=0 forces truncation on any non-empty text
from src.plugins.translate._text_cleaner import clean_text, normalize_whitespace, truncate_text
from src.plugins.translate.[EXT:internal:_text_cleaner] import clean_text, normalize_whitespace, truncate_text
# region TestNormalizeWhitespace [TYPE Class]

26
backend/src/plugins/translate/__tests__/test_token_budget.py
View File

@@ -1,18 +1,18 @@
# #region TestTokenBudget [C:3] [TYPE Module] [SEMANTICS test, token, budget, estimation, batch, translate]
# @BRIEF Verify estimate_token_budget contracts — safe batch sizing, auto-reduction, warning generation.
# @RELATION BINDS_TO -> [estimate_token_budget:Module]
# @TEST_EDGE: empty_rows — empty source rows returns batch_size_adjusted=1
# @TEST_EDGE: small_rows — short text fits in a single batch at requested size
# @TEST_EDGE: large_rows — long text causes batch size reduction
# @TEST_EDGE: multi_language — more target languages increases output estimate
# @TEST_EDGE: context_columns — context columns increase input token estimate
# @TEST_EDGE: dictionary_entries — glossary entries increase input token estimate
# @TEST_EDGE: auto_calc — no batch_size specified, auto-calculates max safe
# @TEST_EDGE: exact_fit — exactly fits context window, batch_adjusted == requested
# @TEST_EDGE: conservative_min — even with huge rows, batch_size_adjusted >= 1
# @TEST_INVARIANT: batch_size_adjusted >= 1 always
# @TEST_INVARIANT: max_output_needed between MIN_MAX_TOKENS(4096) and max_output_tokens(8192)
# @TEST_INVARIANT: warning is None when batch fits, str when reduced
# @RELATION BINDS_TO -> [estimate_token_budget]
# @TEST_EDGE empty_rows — empty source rows returns batch_size_adjusted=1
# @TEST_EDGE small_rows — short text fits in a single batch at requested size
# @TEST_EDGE large_rows — long text causes batch size reduction
# @TEST_EDGE multi_language — more target languages increases output estimate
# @TEST_EDGE context_columns — context columns increase input token estimate
# @TEST_EDGE dictionary_entries — glossary entries increase input token estimate
# @TEST_EDGE auto_calc — no batch_size specified, auto-calculates max safe
# @TEST_EDGE exact_fit — exactly fits context window, batch_adjusted == requested
# @TEST_EDGE conservative_min — even with huge rows, batch_size_adjusted >= 1
# @TEST_INVARIANT batch_size_adjusted >= 1 always
# @TEST_INVARIANT max_output_needed between MIN_MAX_TOKENS(4096) and max_output_tokens(8192)
# @TEST_INVARIANT warning is None when batch fits, str when reduced
from src.plugins.translate._token_budget import DEFAULT_CONTEXT_WINDOW, DEFAULT_MAX_OUTPUT_TOKENS, estimate_token_budget

2
backend/src/plugins/translate/_batch_proc.py
View File

@@ -27,7 +27,7 @@ from ._batch_insert import insert_batch_to_target
from ._lang_detect import batch_detect, get_detector
from ._llm_call import LLMTranslationService
from ._token_budget import estimate_token_budget
from ._utils import _check_translation_cache, _compute_key_hash, _compute_source_hash
from .[EXT:internal:_utils] import _check_translation_cache, _compute_key_hash, _compute_source_hash
from .dictionary import DictionaryManager

2
backend/src/plugins/translate/_batch_sizer.py
View File

@@ -28,7 +28,7 @@ from ._token_budget import (
REASONING_OVERHEAD,
estimate_token_budget,
)
from ._utils import estimate_row_tokens
from .[EXT:internal:_utils] import estimate_row_tokens
# #region AdaptiveBatchSizer [C:3] [TYPE Class]

4
backend/src/plugins/translate/_llm_call.py
View File

@@ -8,7 +8,7 @@
# @RELATION DEPENDS_ON -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [TranslationRecord], [TranslationLanguage]
# @RELATION DEPENDS_ON -> [ContextAwarePromptBuilder]
# @RELATION DEPENDS_ON -> [_llm_http], [_llm_parse]
# @RELATION DEPENDS_ON -> [EXT:method:_llm_http], [_llm_parse]
# @PRE DB session is available. LLM provider is configured on the job.
# @POST TranslationRecord rows created for LLM-processed rows (success/fail/skip).
# @SIDE_EFFECT HTTP calls to LLM provider API; DB writes.
@@ -31,7 +31,7 @@ from ...services.llm_provider import LLMProviderService
from ._llm_http import call_openai_compatible
from ._llm_parse import parse_llm_response
from ._token_budget import estimate_token_budget
from ._utils import _enforce_dictionary
from .[EXT:internal:_utils] import _enforce_dictionary
from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE
from .prompt_builder import ContextAwarePromptBuilder

4
backend/src/plugins/translate/_token_budget.py
View File

@@ -1,8 +1,8 @@
# #region estimate_token_budget [C:3] [TYPE Module] [SEMANTICS translate, token, budget, estimation, llm]
# @BRIEF Calculate safe batch_size and max_tokens for LLM translation calls based on actual content length and model context window limits.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationExecutor:Module]
# @RELATION DEPENDS_ON -> [TranslationExecutor:Module]
# @RELATION DEPENDS_ON -> [TranslationExecutor]
# @RELATION DEPENDS_ON -> [TranslationExecutor]
# @RATIONALE Added comment clarifying PROVIDER_DEFAULTS is a fallback — primary source should be LLMProvider API.
# DeepSeek v4 Flash supports up to 64K context window; output is limited by max_tokens.

6
backend/src/plugins/translate/_utils.py
View File

@@ -1,7 +1,7 @@
# #region TranslationUtils [C:3] [TYPE Module] [SEMANTICS translate, utils, hash, dictionary, cache]
# @BRIEF Shared utility functions for the translation plugin — dictionary enforcement,
# source hashing, cache lookup. Extracted from executor.py to break circular imports.
# @LAYER: Domain
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationRecord]
# @RELATION DEPENDS_ON -> [TranslationLanguage]
# @RATIONALE Extracted from TranslationExecutor to avoid circular imports when sub-services
@@ -22,9 +22,9 @@ from ...models.translate import TranslationLanguage, TranslationRecord
# #region _normalize_term [TYPE Function]
# @BRIEF Normalize a term for case-insensitive unique constraint lookup.
# @RATIONALE: NFC normalization is applied before lowercasing to ensure consistent
# @RATIONALE NFC normalization is applied before lowercasing to ensure consistent
# comparison of Unicode characters (e.g. precomposed vs decomposed forms).
# @REJECTED: Lowercasing without NFC normalization — would cause duplicate entries
# @REJECTED Lowercasing without NFC normalization — would cause duplicate entries
# for semantically identical Unicode strings in different normalization forms.
def _normalize_term(term: str) -> str:
"""Normalize a term by NFC, lowercasing, and removing extra whitespace."""

8
backend/src/plugins/translate/dictionary.py
View File

@@ -1,10 +1,10 @@
# #region DictionaryManagerModule [C:4] [TYPE Module] [SEMANTICS sqlalchemy, translate, dictionary, batch, term]
# @BRIEF Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationJob:Class]
# @RELATION DEPENDS_ON -> [DictionaryEntry:Class]
# @RELATION DEPENDS_ON -> [TerminologyDictionary:Class]
# @RELATION DEPENDS_ON -> [TranslationJobDictionary:Class]
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION DEPENDS_ON -> [DictionaryEntry]
# @RELATION DEPENDS_ON -> [TerminologyDictionary]
# @RELATION DEPENDS_ON -> [TranslationJobDictionary]
# @RATIONALE C4 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion.
# @REJECTED "Keep both" as conflict option — UniqueConstraint prohibits variants; only overwrite/keep existing.
# @REJECTED Monolithic DictionaryManager class — violated INV_7. Decomposed into DictionaryCRUD, DictionaryEntryCRUD, DictionaryImportExport, DictionaryBatchFilter, DictionaryCorrectionService.

2
backend/src/plugins/translate/dictionary_correction.py
View File

@@ -10,7 +10,7 @@ from sqlalchemy.orm import Session
from ...core.logger import belief_scope, logger
from ...models.translate import DictionaryEntry, TerminologyDictionary
from ._utils import _normalize_term
from .[EXT:internal:_utils] import _normalize_term
class DictionaryCorrectionService:

2
backend/src/plugins/translate/dictionary_entries.py
View File

@@ -8,7 +8,7 @@ from sqlalchemy.orm import Session
from ...core.logger import belief_scope, logger
from ...models.translate import DictionaryEntry, TerminologyDictionary
from ._utils import _normalize_term
from .[EXT:internal:_utils] import _normalize_term
from .dictionary_validation import _validate_bcp47

2
backend/src/plugins/translate/dictionary_import_export.py
View File

@@ -11,7 +11,7 @@ from sqlalchemy.orm import Session
from ...core.logger import belief_scope, logger
from ...models.translate import DictionaryEntry, TerminologyDictionary
from ._utils import _detect_delimiter, _normalize_term
from .[EXT:internal:_utils] import _detect_delimiter, _normalize_term
class DictionaryImportExport:

46
backend/src/plugins/translate/events.py
View File

@@ -1,16 +1,16 @@
# #region TranslationEventLog [C:5] [TYPE Module] [SEMANTICS sqlalchemy, translate, event, log, audit]
# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement.
# @LAYER: Domain
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationEvent]
# @RELATION DEPENDS_ON -> [MetricSnapshot]
# @PRE: Database session is open and valid.
# @POST: Events are persisted immutably; terminal events enforce exactly-one invariant per run.
# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion.
# @DATA_CONTRACT: Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]
# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id.
# @RATIONALE: Immutable event log with nullable run_id allows job-level events (no run context) and run-level events.
# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# @REJECTED: Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit.
# @PRE Database session is open and valid.
# @POST Events are persisted immutably; terminal events enforce exactly-one invariant per run.
# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion.
# @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]
# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id.
# @RATIONALE Immutable event log with nullable run_id allows job-level events (no run context) and run-level events.
# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# @REJECTED Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit.
from datetime import UTC, datetime, timedelta
from typing import Any
import uuid
@@ -39,10 +39,10 @@ DEFAULT_RETENTION_DAYS = 90
# #region TranslationEventLog [C:5] [TYPE Class]
# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement.
# @PRE: Database session is available.
# @POST: Events are written immutably; terminal events enforced per run.
# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events.
# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id.
# @PRE Database session is available.
# @POST Events are written immutably; terminal events enforced per run.
# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events.
# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id.
class TranslationEventLog:
def __init__(self, db: Session):
@@ -50,9 +50,9 @@ class TranslationEventLog:
# region log_event [TYPE Function]
# @PURPOSE: Write an immutable event. Enforces terminal event invariant for non-null run_id.
# @PRE: event_type must be a known type. If run_id is not None, enforce terminal invariant.
# @POST: TranslationEvent row is created.
# @SIDE_EFFECT: DB write.
# @PRE event_type must be a known type. If run_id is not None, enforce terminal invariant.
# @POST TranslationEvent row is created.
# @SIDE_EFFECT DB write.
def log_event(
self,
job_id: str,
@@ -123,8 +123,8 @@ class TranslationEventLog:
# region query_events [TYPE Function]
# @PURPOSE: Query events with optional filters.
# @PRE: None.
# @POST: Returns list of TranslationEvent dicts matching filters.
# @PRE None.
# @POST Returns list of TranslationEvent dicts matching filters.
def query_events(
self,
job_id: str | None = None,
@@ -166,9 +166,9 @@ class TranslationEventLog:
# region prune_expired [TYPE Function]
# @PURPOSE: Delete events older than retention_days. Persists MetricSnapshot before pruning.
# @PRE: None.
# @POST: Expired events are deleted; MetricSnapshot is created before deletion.
# @SIDE_EFFECT: Creates MetricSnapshot row; deletes TranslationEvent rows.
# @PRE None.
# @POST Expired events are deleted; MetricSnapshot is created before deletion.
# @SIDE_EFFECT Creates MetricSnapshot row; deletes TranslationEvent rows.
def prune_expired(
self,
retention_days: int = DEFAULT_RETENTION_DAYS,
@@ -247,8 +247,8 @@ class TranslationEventLog:
# region get_run_event_summary [TYPE Function]
# @PURPOSE: Get a summary of events for a run, including invariant check.
# @PRE: run_id is not None.
# @POST: Returns dict with event list and invariant validity.
# @PRE run_id is not None.
# @POST Returns dict with event list and invariant validity.
def get_run_event_summary(self, run_id: str) -> dict[str, Any]:
with belief_scope("TranslationEventLog.get_run_event_summary"):
events = self.query_events(run_id=run_id)

4
backend/src/plugins/translate/executor.py
View File

@@ -13,7 +13,7 @@
# @INVARIANT Batch processing is independent — one batch failure does not affect others.
# @RATIONALE Extracted from monolithic executor.py (1974 lines) into thin orchestrator
# to comply with INV_7. Sub-services in _run_service, _batch_proc, _llm_call, _batch_sizer.
# Module-level helpers moved to _utils.py.
# Module-level helpers moved to [EXT:internal:_utils].py.
# @REJECTED Keeping monolithic executor.py at 1974 lines — violates INV_7 by +1574 lines.
# Single monolithic LLM call — would lose all progress on any failure.
@@ -29,7 +29,7 @@ from ...models.translate import TranslationJob, TranslationRun, TranslationRunLa
from ...services.llm_provider import LLMProviderService
from ._run_service import RunExecutionService
from ._token_budget import estimate_token_budget
from ._utils import _check_translation_cache, _compute_source_hash, _enforce_dictionary, estimate_row_tokens
from .[EXT:internal:_utils] import _check_translation_cache, _compute_source_hash, _enforce_dictionary, estimate_row_tokens
__all__ = [
"TranslationExecutor", "estimate_row_tokens",

8
backend/src/plugins/translate/metrics.py
View File

@@ -1,10 +1,10 @@
# #region TranslationMetrics [C:3] [TYPE Module] [SEMANTICS sqlalchemy, translate, metrics, job, statistics]
# @BRIEF Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [TranslationSchedule]
# @RELATION DEPENDS_ON -> [TranslationSchedule]
# @RELATION DEPENDS_ON -> [TranslationSchedule]
# @RELATION DEPENDS_ON -> [TranslationSchedule]
# @PRE Database session is open.
# @POST Metrics are aggregated and returned; no side effects.
# @RATIONALE Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture.

16
backend/src/plugins/translate/orchestrator.py
View File

@@ -61,9 +61,9 @@ class TranslationOrchestrator:
# region start_run [TYPE Function]
# @PURPOSE: Start a new translation run for a job.
# @PRE: job_id exists. For manual runs, an accepted preview session must exist.
# @POST: TranslationRun is created in PENDING status with hash fields and config snapshot.
# @SIDE_EFFECT: DB writes.
# @PRE job_id exists. For manual runs, an accepted preview session must exist.
# @POST TranslationRun is created in PENDING status with hash fields and config snapshot.
# @SIDE_EFFECT DB writes.
def start_run(
self,
job_id: str,
@@ -82,9 +82,9 @@ class TranslationOrchestrator:
# region execute_run [TYPE Function]
# @PURPOSE: Execute a translation run: dispatch executor, generate SQL, submit to Superset.
# @PRE: run is in PENDING status.
# @POST: Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT: LLM calls, DB writes, Superset API calls.
# @PRE run is in PENDING status.
# @POST Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
def execute_run(
self,
run: TranslationRun,
@@ -119,7 +119,7 @@ class TranslationOrchestrator:
# region _generate_and_insert_sql [TYPE Function] [SEMANTICS backward-compat wrapper]
# @PURPOSE: Backward-compatible delegating wrapper for SQL generation and insert.
# @SIDE_EFFECT: Delegates to SQLInsertService. May call Superset API.
# @SIDE_EFFECT Delegates to SQLInsertService. May call Superset API.
def _generate_and_insert_sql(
self,
job: Any,
@@ -133,7 +133,7 @@ class TranslationOrchestrator:
# region _update_language_stats [TYPE Function] [SEMANTICS backward-compat wrapper]
# @PURPOSE: Backward-compatible delegating wrapper for language stats update.
# @SIDE_EFFECT: Delegates to TranslationResultAggregator.update_language_stats.
# @SIDE_EFFECT Delegates to TranslationResultAggregator.update_language_stats.
def _update_language_stats(
self,
run_id: str,

4
backend/src/plugins/translate/orchestrator_aggregator.py
View File

@@ -7,7 +7,7 @@
# @RELATION DEPENDS_ON -> [TranslationRecord]
# @RELATION DEPENDS_ON -> [TranslationRunLanguageStats]
# @RELATION DEPENDS_ON -> [TranslationEventLog]
# @RELATION DEPENDS_ON -> [orchestrator_lang_stats]
# @RELATION DEPENDS_ON -> [EXT:frontend:orchestrator_lang_stats]
# @RELATION DEPENDS_ON -> [orchestrator_query]
# @RATIONALE Language stats aggregation extracted to orchestrator_lang_stats; query methods to orchestrator_query.
@@ -119,7 +119,7 @@ class TranslationResultAggregator:
# region update_language_stats [TYPE Function]
# @PURPOSE: Aggregate TranslationLanguage entries and update TranslationRunLanguageStats.
# @SIDE_EFFECT: DB writes on language_stats objects.
# @SIDE_EFFECT DB writes on language_stats objects.
def update_language_stats(
self,
run_id: str,

6
backend/src/plugins/translate/orchestrator_exec.py
View File

@@ -52,9 +52,9 @@ class TranslationExecutionEngine:
# region execute_run [TYPE Function]
# @PURPOSE: Execute a translation run: dispatch executor, handle outcomes.
# @PRE: run is in PENDING status.
# @POST: Run executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT: LLM calls, DB writes, Superset API calls.
# @PRE run is in PENDING status.
# @POST Run executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
def execute_run(
self,
run: TranslationRun,

6
backend/src/plugins/translate/orchestrator_planner.py
View File

@@ -37,9 +37,9 @@ class TranslationPlanner:
# region plan_run [TYPE Function]
# @PURPOSE: Validate, compute hashes, and create a new TranslationRun.
# @PRE: job_id exists. For manual runs, an accepted preview session exists.
# @POST: TranslationRun is created in PENDING status with hash fields and config snapshot.
# @SIDE_EFFECT: DB writes; event recorded.
# @PRE job_id exists. For manual runs, an accepted preview session exists.
# @POST TranslationRun is created in PENDING status with hash fields and config snapshot.
# @SIDE_EFFECT DB writes; event recorded.
def plan_run(
self,
job_id: str,

6
backend/src/plugins/translate/orchestrator_retry.py
View File

@@ -44,7 +44,7 @@ class TranslationRunRetryManager:
# region retry_failed_batches [TYPE Function]
# @PURPOSE: Retry failed batches in a run.
# @SIDE_EFFECT: Re-executes batch translations; DB writes.
# @SIDE_EFFECT Re-executes batch translations; DB writes.
def retry_failed_batches(self, run_id: str) -> TranslationRun:
with belief_scope("TranslationRunRetryManager.retry_failed_batches"):
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
@@ -121,14 +121,14 @@ class TranslationRunRetryManager:
# region retry_insert [TYPE Function]
# @PURPOSE: Retry the SQL insert phase for a completed run.
# @SIDE_EFFECT: Superset API call; DB writes.
# @SIDE_EFFECT Superset API call; DB writes.
def retry_insert(self, run_id: str) -> TranslationRun:
return _retry_insert(self.db, self.config_manager, self.event_log, self.current_user, run_id)
# endregion retry_insert
# region cancel_run [TYPE Function]
# @PURPOSE: Cancel a running translation.
# @SIDE_EFFECT: DB writes; event log.
# @SIDE_EFFECT DB writes; event log.
def cancel_run(self, run_id: str) -> TranslationRun:
return _cancel_run(self.db, self.event_log, self.current_user, run_id)
# endregion cancel_run

6
backend/src/plugins/translate/orchestrator_run_completion.py
View File

@@ -15,7 +15,7 @@ from ...models.translate import TranslationJob, TranslationRun, TranslationRunLa
# region handle_executor_failure [TYPE Function]
# @PURPOSE: Handle executor failure — rollback, mark run as FAILED, log event.
# @SIDE_EFFECT: DB writes; event log.
# @SIDE_EFFECT DB writes; event log.
def handle_executor_failure(
db,
event_log,
@@ -47,7 +47,7 @@ def handle_executor_failure(
# region complete_cancelled [TYPE Function]
# @PURPOSE: Finalize a cancelled run — update language stats and log.
# @SIDE_EFFECT: DB writes; event log.
# @SIDE_EFFECT DB writes; event log.
def complete_cancelled(
db,
event_log,
@@ -69,7 +69,7 @@ def complete_cancelled(
# region complete_success [TYPE Function]
# @PURPOSE: Finalize a successful run — update stats, optionally insert SQL, commit.
# @SIDE_EFFECT: DB writes; event log; Superset API call if skip_insert is False.
# @SIDE_EFFECT DB writes; event log; Superset API call if skip_insert is False.
def complete_success(
db,
event_log,

6
backend/src/plugins/translate/orchestrator_runner.py
View File

@@ -38,9 +38,9 @@ class TranslationStageRunner:
# region execute_run [TYPE Function]
# @PURPOSE: Execute a translation run: dispatch executor, generate SQL, submit to Superset.
# @PRE: run is in PENDING status.
# @POST: Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT: LLM calls, DB writes, Superset API calls.
# @PRE run is in PENDING status.
# @POST Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
def execute_run(
self,
run: TranslationRun,

2
backend/src/plugins/translate/orchestrator_sql_rows.py
View File

@@ -50,7 +50,7 @@ def build_context_keys(job: TranslationJob, effective_target: str | None) -> lis
# #region build_rows [C:2] [TYPE Function] [SEMANTICS sql, rows, build]
# @BRIEF Build row data for SQL INSERT with per-language expansion.
# @SIDE_EFFECT: Reads translation record language entries.
# @SIDE_EFFECT Reads translation record language entries.
def build_rows(
records: list[TranslationRecord],
job: TranslationJob,

2
backend/src/plugins/translate/plugin.py
View File

@@ -12,7 +12,7 @@ from ...core.plugin_base import PluginBase
# #region TranslatePlugin [TYPE Class]
# @BRIEF Plugin for translating SQL queries and dashboard definitions across database dialects.
# @RELATION IMPLEMENTS -> backend.src.core.plugin_base.PluginBase
# @RELATION IMPLEMENTS -> [EXT:path:backend.src.core.plugin_base.PluginBase]
class TranslatePlugin(PluginBase):
@property
def id(self) -> str:

2
backend/src/plugins/translate/preview.py
View File

@@ -65,7 +65,7 @@ class TranslationPreview:
# region preview_rows [TYPE Function]
# @PURPOSE: Fetch sample rows, send to LLM, create preview session with per-language records.
# @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates DB rows.
# @SIDE_EFFECT Fetches data from Superset; calls LLM; creates DB rows.
def preview_rows(
self,
job_id: str,

4
backend/src/plugins/translate/preview_executor.py
View File

@@ -37,7 +37,7 @@ class PreviewExecutor:
# region fetch_sample_rows [TYPE Function]
# @PURPOSE: Fetch sample rows from Superset dataset for preview.
# @SIDE_EFFECT: Calls Superset chart data endpoint.
# @SIDE_EFFECT Calls Superset chart data endpoint.
def fetch_sample_rows(
self,
job: TranslationJob,
@@ -84,7 +84,7 @@ class PreviewExecutor:
# region call_llm [TYPE Function]
# @PURPOSE: Call the configured LLM provider with a prompt.
# @SIDE_EFFECT: Makes HTTP call to LLM provider.
# @SIDE_EFFECT Makes HTTP call to LLM provider.
def call_llm(self, job: TranslationJob, prompt: str, max_tokens: int = 8192) -> str:
with belief_scope("PreviewExecutor.call_llm"):
if not job.provider_id:

4
backend/src/plugins/translate/preview_prompt_builder.py
View File

@@ -29,8 +29,8 @@ class PreviewPromptBuilder:
# region build_prompt_from_rows [TYPE Function]
# @PURPOSE: Build the complete LLM prompt from source rows, dictionary, and job config.
# @PRE: job has valid configuration. source_rows is non-empty.
# @POST: Returns prompt string and token budget metadata.
# @PRE job has valid configuration. source_rows is non-empty.
# @POST Returns prompt string and token budget metadata.
def build_prompt_from_rows(
self,
job: TranslationJob,

2
backend/src/plugins/translate/preview_session_ops.py
View File

@@ -22,7 +22,7 @@ from .preview_session_serializer import (
# #region accept_preview_session [C:2] [TYPE Function] [SEMANTICS preview, session, accept]
# @BRIEF Mark a preview session as accepted, which gates full execution.
# @SIDE_EFFECT: DB writes on session status.
# @SIDE_EFFECT DB writes on session status.
def accept_preview_session(db: Session, job_id: str, _current_user: str | None = None) -> dict[str, Any]:
"""Mark a preview session as accepted and return the session data with records."""
session = (

2
backend/src/plugins/translate/prompt_builder.py
View File

@@ -1,7 +1,7 @@
# #region ContextAwarePromptBuilder [C:2] [TYPE Module] [SEMANTICS translate, prompt, context, dictionary]
# @BRIEF Pure-function prompt builder that enhances dictionary entries with context annotations.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [DictionaryEntry:Class]
# @RELATION DEPENDS_ON -> [DictionaryEntry]
# @RATIONALE Pure functions only — no I/O, no DB access. Separated from executor for testability.
# @REJECTED Embedding context inline in the executor would make it untestable without mocking DB.

48
backend/src/plugins/translate/scheduler.py
View File

@@ -1,16 +1,16 @@
# #region TranslationScheduler [C:4] [TYPE Module] [SEMANTICS sqlalchemy, translate, schedule, cron, job]
# @BRIEF Manage TranslationSchedule rows and register them with core SchedulerService.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [SchedulerService:Class]
# @RELATION DEPENDS_ON -> [TranslationOrchestrator:Class]
# @RELATION DEPENDS_ON -> [TranslationEventLog:Class]
# @PRE: Database session and SchedulerService are available.
# @POST: TranslationSchedule CRUD persisted; APScheduler jobs registered/updated/removed.
# @SIDE_EFFECT: Registers APScheduler jobs; runs translations on trigger; creates events.
# @RATIONALE: Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance.
# @REJECTED: Separate scheduler instance would create resource contention.
# @REJECTED: Polling-based approach — event-driven APScheduler is more precise.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationSchedule]
# @RELATION DEPENDS_ON -> [SchedulerService]
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
# @RELATION DEPENDS_ON -> [TranslationEventLog]
# @PRE Database session and SchedulerService are available.
# @POST TranslationSchedule CRUD persisted; APScheduler jobs registered/updated/removed.
# @SIDE_EFFECT Registers APScheduler jobs; runs translations on trigger; creates events.
# @RATIONALE Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance.
# @REJECTED Separate scheduler instance would create resource contention.
# @REJECTED Polling-based approach — event-driven APScheduler is more precise.
from datetime import UTC, datetime, timedelta
import uuid
@@ -37,8 +37,8 @@ class TranslationScheduler:
# region create_schedule [TYPE Function]
# @PURPOSE: Create a new schedule for a job.
# @PRE: job_id exists. cron_expression is valid.
# @POST: TranslationSchedule row created.
# @PRE job_id exists. cron_expression is valid.
# @POST TranslationSchedule row created.
def create_schedule(
self,
job_id: str,
@@ -83,8 +83,8 @@ class TranslationScheduler:
# region update_schedule [TYPE Function]
# @PURPOSE: Update an existing schedule.
# @PRE: job_id has an existing schedule.
# @POST: Schedule updated.
# @PRE job_id has an existing schedule.
# @POST Schedule updated.
def update_schedule(
self,
job_id: str,
@@ -130,8 +130,8 @@ class TranslationScheduler:
# region delete_schedule [TYPE Function]
# @PURPOSE: Delete a schedule for a job.
# @PRE: job_id has an existing schedule.
# @POST: Schedule deleted.
# @PRE job_id has an existing schedule.
# @POST Schedule deleted.
def delete_schedule(self, job_id: str) -> None:
with belief_scope("TranslationScheduler.delete_schedule"):
schedule = self.db.query(TranslationSchedule).filter(
@@ -156,8 +156,8 @@ class TranslationScheduler:
# region enable_disable_schedule [TYPE Function]
# @PURPOSE: Enable or disable a schedule.
# @PRE: job_id has an existing schedule.
# @POST: Schedule is_active updated.
# @PRE job_id has an existing schedule.
# @POST Schedule is_active updated.
def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule:
with belief_scope("TranslationScheduler.set_schedule_active"):
schedule = self.db.query(TranslationSchedule).filter(
@@ -181,8 +181,8 @@ class TranslationScheduler:
# region get_schedule [TYPE Function]
# @PURPOSE: Get schedule for a job.
# @PRE: job_id exists.
# @POST: Returns TranslationSchedule or raises ValueError.
# @PRE job_id exists.
# @POST Returns TranslationSchedule or raises ValueError.
def get_schedule(self, job_id: str) -> TranslationSchedule:
with belief_scope("TranslationScheduler.get_schedule"):
schedule = self.db.query(TranslationSchedule).filter(
@@ -195,7 +195,7 @@ class TranslationScheduler:
# region list_active_schedules [TYPE Function]
# @PURPOSE: List all active schedules.
# @POST: Returns list of active TranslationSchedule rows.
# @POST Returns list of active TranslationSchedule rows.
@staticmethod
def list_active_schedules(db: Session) -> list[TranslationSchedule]:
return (
@@ -207,8 +207,8 @@ class TranslationScheduler:
# region get_next_executions [TYPE Function]
# @PURPOSE: Compute next N execution times from cron expression.
# @PRE: cron_expression is valid.
# @POST: Returns list of ISO datetime strings.
# @PRE cron_expression is valid.
# @POST Returns list of ISO datetime strings.
@staticmethod
def get_next_executions(cron_expression: str, timezone_str: str = "UTC", n: int = 3) -> list[str]:
from zoneinfo import ZoneInfo

10
backend/src/plugins/translate/service.py
View File

@@ -9,7 +9,7 @@
# @SIDE_EFFECT Queries Superset for column metadata and database dialect at save time.
# @RATIONALE Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity.
# @REJECTED Monolithic service.py — violated INV_7. Decomposed into service_datasource, service_inline_correction, service_bulk_replace, service_utils.
# @REJECTED Monolithic service.py — violated INV_7. Decomposed into service_datasource, service_inline_correction, service_bulk_replace, service[EXT:internal:_utils].
from datetime import UTC, datetime
from typing import Any
@@ -23,7 +23,7 @@ from ...models.translate import TranslationJob, TranslationJobDictionary
from ...schemas.translate import TranslateJobCreate, TranslateJobUpdate
from .dictionary import _validate_bcp47
from .service_datasource import fetch_datasource_metadata
from .service_utils import _extract_dialect, job_to_response
from .service[EXT:internal:_utils] import _extract_dialect, job_to_response
# #region TranslateJobService [TYPE Class]
@@ -64,7 +64,7 @@ class TranslateJobService:
# region create_job [TYPE Function]
# @PURPOSE: Create a new translation job with column validation.
# @SIDE_EFFECT: Validates columns via SupersetClient; caches database_dialect.
# @SIDE_EFFECT Validates columns via SupersetClient; caches database_dialect.
def create_job(self, payload: TranslateJobCreate) -> TranslationJob:
logger.info(f"[TranslateJobService] Creating job '{payload.name}'")
if payload.source_datasource_id and not payload.translation_column:
@@ -132,7 +132,7 @@ class TranslateJobService:
# region update_job [TYPE Function]
# @PURPOSE: Update an existing translation job.
# @SIDE_EFFECT: Re-detects database_dialect if source_datasource_id changed.
# @SIDE_EFFECT Re-detects database_dialect if source_datasource_id changed.
def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob:
logger.info(f"[TranslateJobService] Updating job '{job_id}'")
job = self.get_job(job_id)
@@ -272,5 +272,5 @@ from .service_datasource import ( # noqa: E402, F401
get_dialect_from_database,
)
from .service_inline_correction import InlineCorrectionService # noqa: E402, F401
from .service_utils import _extract_dialect, job_to_response # noqa: E402, F401
from .service[EXT:internal:_utils] import _extract_dialect, job_to_response # noqa: E402, F401
# #endregion TranslateJobService

2
backend/src/plugins/translate/service_inline_correction.py
View File

@@ -14,7 +14,7 @@ from sqlalchemy.orm import Session
from ...core.logger import belief_scope, logger
from ...models.translate import DictionaryEntry, TranslationLanguage, TranslationRecord
from ._utils import _normalize_term
from .[EXT:internal:_utils] import _normalize_term
class InlineCorrectionService:

4
backend/src/plugins/translate/service_target_schema.py
View File

@@ -3,8 +3,8 @@
# сравнение с ожидаемыми (из build_columns), возврат diff.
# @LAYER Service
# @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor]
# @RELATION DEPENDS_ON -> [schemas.translate.TargetSchemaValidationRequest]
# @RELATION DEPENDS_ON -> [schemas.translate.TargetSchemaValidationResponse]
# @RELATION DEPENDS_ON -> [EXT:method:schemas.translate.TargetSchemaValidationRequest]
# @RELATION DEPENDS_ON -> [EXT:method:schemas.translate.TargetSchemaValidationResponse]
# @PRE Superset окружение доступно, target_database_id валиден.
# @POST Возвращает актуальные, ожидаемые, отсутствующие и лишние колонки.
# @SIDE_EFFECT Выполняет SQL-запрос через Superset SQL Lab.

32
backend/src/plugins/translate/sql_generator.py
View File

@@ -1,14 +1,14 @@
# #region SQLGenerator [C:3] [TYPE Module] [SEMANTICS clickhouse, translate, sql, insert, generate]
# @BRIEF Dialect-aware safe SQL generation for INSERT/UPSERT operations.
# @LAYER: Domain
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION DEPENDS_ON -> [TranslationRun]
# @PRE: Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS.
# @POST: Returns safe SQL strings for the target dialect.
# @SIDE_EFFECT: None — pure code generation.
# @RATIONALE: Dialect-aware SQL uses ON CONFLICT for PostgreSQL; plain INSERT for ClickHouse with documented limitations.
# @REJECTED: UPDATE statements — source is append-only; UPSERT covers overwrite case.
# @REJECTED: ORM-based insert bypasses Superset's SQL Lab audit trail.
# @PRE Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS.
# @POST Returns safe SQL strings for the target dialect.
# @SIDE_EFFECT None — pure code generation.
# @RATIONALE Dialect-aware SQL uses ON CONFLICT for PostgreSQL; plain INSERT for ClickHouse with documented limitations.
# @REJECTED UPDATE statements — source is append-only; UPSERT covers overwrite case.
# @REJECTED ORM-based insert bypasses Superset's SQL Lab audit trail.
from datetime import UTC, datetime
from typing import Any
@@ -60,8 +60,8 @@ def _normalize_timestamp_value(value: Any) -> str | None:
# #region _quote_identifier [C:4] [TYPE Function]
# @BRIEF Quote an identifier per dialect rules. PostgreSQL uses double quotes; ClickHouse uses backticks.
# @PRE: identifier is a non-empty string.
# @POST: Returns safely quoted identifier.
# @PRE identifier is a non-empty string.
# @POST Returns safely quoted identifier.
def _quote_identifier(identifier: str, dialect: str) -> str:
"""Quote a SQL identifier per dialect rules."""
if not identifier:
@@ -212,15 +212,15 @@ def generate_upsert_sql(
# #region SQLGenerator [C:3] [TYPE Class]
# @BRIEF Generate safe, dialect-appropriate SQL INSERT/UPSERT statements.
# @PRE: Job has target_schema, target_table, key columns configured.
# @POST: Returns generated SQL string for the target dialect.
# @PRE Job has target_schema, target_table, key columns configured.
# @POST Returns generated SQL string for the target dialect.
class SQLGenerator:
# region SQLGenerator.generate [TYPE Function]
# @PURPOSE: Generate SQL for a set of rows, detecting dialect from the job configuration.
# @PRE: dialect is a supported database dialect. columns list is non-empty. rows is non-empty.
# @POST: Returns tuple of (sql_string, statement_count).
# @SIDE_EFFECT: None — pure SQL generation.
# @PRE dialect is a supported database dialect. columns list is non-empty. rows is non-empty.
# @POST Returns tuple of (sql_string, statement_count).
# @SIDE_EFFECT None — pure SQL generation.
@staticmethod
def generate(
dialect: str,
@@ -334,8 +334,8 @@ class SQLGenerator:
# region SQLGenerator.generate_batch [TYPE Function]
# @PURPOSE: Generate separate INSERT statements for each row (batch-safe version).
# @PRE: Same as generate().
# @POST: Returns list of (sql_string, row_index) tuples.
# @PRE Same as generate().
# @POST Returns list of (sql_string, row_index) tuples.
@staticmethod
def generate_batch(
dialect: str,