busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(llm): add fetch-models endpoint, fix SQL Lab INSERT (client_id truncation, sync mode, target_column, timestamp normalization)

Browse Source 
- Add POST /api/llm/providers/fetch-models route with LLMClient.fetch_models()
- Add target_column to TranslationJob model/schema/service/orchestrator
- Fix SQL Lab execute: truncate client_id to 11 chars (varchar(11))
- Switch SQL Lab to sync mode (runAsync: false) — no Celery workers
- Fix polling: unwrap nested result from Superset query API
- Fix ClickHouse timestamp: normalize float timestamps to YYYY-MM-DD
This commit is contained in:
busya
2026-05-13 20:06:15 +03:00
parent 39ab647851
commit a3c7c402b7
276 changed files with 1923 additions and 1822 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/__init__.py
View File

@@ -1,3 +1,3 @@
# #region SrcRoot [TYPE Module]
# #region SrcRoot [TYPE Module] [SEMANTICS root, package]
# @BRIEF Canonical backend package root for application, scripts, and tests.
# #endregion SrcRoot

63
backend/src/api/routes/llm.py
View File

@@ -75,6 +75,69 @@ async def get_providers(
# #endregion get_providers
# #region fetch_models [TYPE Function]
# @BRIEF Fetch available models from an LLM provider by base_url+provider_type, or by provider_id.
# @PRE: User is authenticated. Either provider_id or base_url+provider_type must be provided.
# @POST: Returns a list of available model IDs.
# @RELATION CALLS -> [LLMProviderService]
# @RELATION CALLS -> [LLMClient]
@router.post("/providers/fetch-models")
async def fetch_models(
payload: dict,
current_user: User = Depends(get_current_active_user),
db: Session = Depends(get_db),
):
from ...plugins.llm_analysis.service import LLMClient
from ...plugins.llm_analysis.models import LLMProviderType
base_url = (payload.get("base_url") or "").strip()
provider_type_str = (payload.get("provider_type") or "").strip()
api_key = payload.get("api_key") or ""
provider_id = payload.get("provider_id") or ""
# Resolve provider_id to stored credentials if no direct api_key given
if not api_key and provider_id:
service = LLMProviderService(db)
db_provider = service.get_provider(provider_id)
if not db_provider:
raise HTTPException(status_code=404, detail="Provider not found")
base_url = base_url or db_provider.base_url
provider_type_str = provider_type_str or db_provider.provider_type
stored_key = service.get_decrypted_api_key(provider_id)
if stored_key:
api_key = stored_key
if not base_url:
raise HTTPException(status_code=400, detail="base_url is required")
if not provider_type_str:
raise HTTPException(status_code=400, detail="provider_type is required")
try:
provider_type = LLMProviderType(provider_type_str)
except ValueError:
raise HTTPException(status_code=400, detail=f"Invalid provider_type: {provider_type_str}")
client = LLMClient(
provider_type=provider_type,
api_key=api_key or "sk-placeholder",
base_url=base_url,
default_model="",
)
try:
models = await client.fetch_models()
return {"models": models}
except Exception as e:
logger.warning(
f"[llm_routes.fetch_models] Failed to fetch models: {e}",
extra={"src": "llm_routes.fetch_models"},
)
raise HTTPException(status_code=502, detail=str(e))
# #endregion fetch_models
# #region get_llm_status [TYPE Function]
# @BRIEF Returns whether LLM runtime is configured for dashboard validation.
# @PRE: User is authenticated.

105
backend/src/api/routes/translate/_run_routes.py
View File

@@ -2,11 +2,12 @@
# @BRIEF Translation Run execution, history, status, records and batches routes.
# @LAYER: API
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, status, Query
from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from ....core.database import get_db
from ....core.database import get_db, SessionLocal
from ....core.logger import logger, belief_scope
from ....schemas.auth import User
from ....dependencies import get_current_user, has_permission, get_config_manager
@@ -39,11 +40,107 @@ async def run_translation(
try:
orch = TranslationOrchestrator(db, config_manager, current_user.username)
run = orch.start_run(job_id=job_id, is_scheduled=False)
# Execute asynchronously in background
# The request-scoped db session will be closed after this handler returns.
# The background thread must use its OWN session to avoid operating on a
# closed or expunged session.
import threading
def _background_execute():
bg_db = SessionLocal()
try:
bg_orch = TranslationOrchestrator(
bg_db, config_manager,
current_user.username if current_user else None,
)
# Re-fetch the run within the background session to get a fresh,
# attached object
from ....models.translate import TranslationRun as TRModel
bg_run = bg_db.query(TRModel).filter(TRModel.id == run.id).first()
if bg_run is None:
logger.explore(
"Background execute: run not found",
extra={"src": "translate_routes", "run_id": run.id},
)
return
# execute_run internally calls self.db.commit()
bg_orch.execute_run(bg_run)
# ----- VERIFICATION -----
# After execute_run the run object from bg_db is detached after
# commit. Re-query to verify the status was persisted in the
# database and is a terminal state.
check_run = bg_db.query(TRModel).filter(TRModel.id == run.id).first()
if check_run and check_run.status in ("COMPLETED", "FAILED", "CANCELLED"):
logger.reason(
"Background execute verified",
extra={
"src": "translate_routes",
"run_id": run.id,
"status": check_run.status,
},
)
else:
# execute_run appeared to succeed but the run is still in a
# non-terminal state — manually fail it so the frontend
# polling can detect the terminal state and stop spinning.
actual_status = check_run.status if check_run else "NOT_FOUND"
logger.explore(
"Background execute: run not in terminal state after commit",
extra={
"src": "translate_routes",
"run_id": run.id,
"status": actual_status,
},
)
if check_run:
check_run.status = "FAILED"
check_run.error_message = (
"Background execution did not reach terminal state "
f"(was: {actual_status})"
)
bg_db.commit()
except Exception as bg_err:
logger.explore(
"Background execute failed",
extra={
"src": "translate_routes",
"run_id": run.id,
"error": str(bg_err),
},
)
# Mark the run as FAILED so the frontend polling can detect a
# terminal state and stop spinning.
try:
fb_db = SessionLocal()
try:
fb_run = fb_db.query(TRModel).filter(TRModel.id == run.id).first()
if fb_run:
fb_run.status = "FAILED"
fb_run.error_message = f"Background execute error: {bg_err}"
fb_run.completed_at = datetime.now(timezone.utc)
fb_db.commit()
logger.reason(
"Background execute: run marked FAILED",
extra={"src": "translate_routes", "run_id": run.id},
)
finally:
fb_db.close()
except Exception as fb_err:
logger.explore(
"Background execute: unable to mark run FAILED",
extra={
"src": "translate_routes",
"run_id": run.id,
"error": str(fb_err),
},
)
finally:
bg_db.close()
threading.Thread(
target=orch.execute_run,
args=(run,),
target=_background_execute,
daemon=True,
).start()
return _run_to_response(run)

18
backend/src/core/database.py
View File

@@ -484,6 +484,24 @@ def _ensure_translation_jobs_columns(bind_engine):
)
raise
if "target_database_id" not in existing_columns:
try:
with bind_engine.begin() as connection:
connection.execute(
text(
"ALTER TABLE translation_jobs "
"ADD COLUMN target_database_id VARCHAR"
)
)
logger.reflect(
"Added target_database_id column to translation_jobs",
)
except Exception as migration_error:
logger.explore(
"Failed to add target_database_id to translation_jobs",
extra={"error": str(migration_error)},
)
def _ensure_dataset_review_session_columns(bind_engine):
with belief_scope("_ensure_dataset_review_session_columns"):

2
backend/src/core/superset_client/_databases.py
View File

@@ -47,7 +47,7 @@ class SupersetDatabasesMixin:
# @RELATION: CALLS -> [SupersetClientGetDatabases]
def get_databases_summary(self) -> List[Dict]:
with belief_scope("SupersetClient.get_databases_summary"):
query = {"columns": ["uuid", "database_name", "backend"]}
query = {"columns": ["id", "uuid", "database_name", "backend"]}
_, databases = self.get_databases(query=query)
# Map 'backend' to 'engine' for consistency with contracts
for db in databases:

44
backend/src/core/utils/network.py
View File

@@ -243,11 +243,45 @@ class APIClient:
with belief_scope("authenticate"):
app_logger.info("[authenticate][Enter] Authenticating to %s", self.base_url)
cached_tokens = SupersetAuthCache.get(self._auth_cache_key)
if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"):
self._tokens = cached_tokens
self._authenticated = True
app_logger.info("[authenticate][CacheHit] Reusing cached Superset auth tokens for %s", self.base_url)
return self._tokens
if cached_tokens and cached_tokens.get("access_token"):
# Cache hit — we have a valid access_token, but we need to
# re-establish the session cookie in this new requests.Session.
# Superset CSRF protection pairs the X-CSRFToken header with a
# Flask session cookie set during login. A fresh requests.Session
# lacks that cookie, so even a valid cached CSRF token would fail
# with "CSRF session token is missing". We re-fetch the CSRF
# token to both establish the session cookie AND get a fresh
# CSRF token for the current session.
try:
csrf_url = f"{self.api_base_url}/security/csrf_token/"
csrf_response = self.session.get(
csrf_url,
headers={"Authorization": f"Bearer {cached_tokens['access_token']}"},
timeout=self.request_settings["timeout"],
)
csrf_response.raise_for_status()
csrf_token = csrf_response.json()["result"]
self._tokens = {
"access_token": cached_tokens["access_token"],
"csrf_token": csrf_token,
}
self._authenticated = True
# Update cache with fresh CSRF token for this session
SupersetAuthCache.set(self._auth_cache_key, self._tokens)
app_logger.info(
"[authenticate][CacheHit] Reusing cached Superset auth tokens for %s "
"(CSRF token refreshed for new session)",
self.base_url,
)
return self._tokens
except Exception as refresh_err:
app_logger.warning(
"[authenticate][CacheRefreshFailed] CSRF token refresh failed, "
"falling back to full re-authentication: %s",
refresh_err,
)
SupersetAuthCache.invalidate(self._auth_cache_key)
# Fall through to full authentication below
try:
login_url = f"{self.api_base_url}/security/login"
# Log the payload keys and values (masking password)

6
backend/src/models/translate.py
View File

@@ -42,7 +42,8 @@ class TranslationJob(Base):
# Column mapping
source_key_cols = Column(JSON, nullable=True, comment="Source key column names for composite key")
target_key_cols = Column(JSON, nullable=True, comment="Target key column names for composite key")
translation_column = Column(String, nullable=True, comment="The column whose values will be translated")
translation_column = Column(String, nullable=True, comment="Source column whose values will be translated")
target_column = Column(String, nullable=True, comment="Target column for translated output (defaults to translation_column)")
context_columns = Column(JSON, nullable=True, comment="Context column names included in LLM prompt")
# LLM & processing settings
@@ -53,6 +54,7 @@ class TranslationJob(Base):
# Environment association
environment_id = Column(String, nullable=True, comment="Superset environment ID for datasource access")
target_database_id = Column(String, nullable=True, comment="Superset database ID for SQL Lab insert target")
created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
@@ -119,6 +121,7 @@ class TranslationRecord(Base):
source_object_type = Column(String, nullable=True) # query, dashboard, chart, dataset
source_object_id = Column(String, nullable=True)
source_object_name = Column(String, nullable=True)
source_data = Column(JSON, nullable=True, comment="Original source row key columns for upsert matching")
status = Column(String, nullable=False, default="PENDING") # PENDING, SUCCESS, FAILED, SKIPPED
error_message = Column(Text, nullable=True)
token_count_input = Column(Integer, nullable=True)
@@ -174,6 +177,7 @@ class TranslationPreviewRecord(Base):
source_object_type = Column(String, nullable=True)
source_object_id = Column(String, nullable=True)
source_object_name = Column(String, nullable=True)
source_data = Column(JSON, nullable=True, comment="Original source row key columns for upsert matching")
status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED
feedback = Column(Text, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))

1
backend/src/plugins/git/llm_extension.py
View File

@@ -1,5 +1,4 @@
# #region GitLLMExtensionModule [C:3] [TYPE Module] [SEMANTICS git, llm, commit, message, generation]
# @COMPLEXITY: 3
# @BRIEF LLM-based extensions for the Git plugin, specifically for commit message generation.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [LLMClient]

1
backend/src/plugins/llm_analysis/__tests__/test_client_headers.py
View File

@@ -1,6 +1,5 @@
# region TestClientHeaders [TYPE Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @COMPLEXITY: 3
# @SEMANTICS: tests, llm-client, openrouter, headers
# @PURPOSE: Verify OpenRouter client initialization includes provider-specific headers.

1
backend/src/plugins/llm_analysis/__tests__/test_screenshot_service.py
View File

@@ -1,6 +1,5 @@
# region TestScreenshotService [TYPE Module]
# @RELATION: VERIFIES ->[src.plugins.llm_analysis.service.ScreenshotService]
# @COMPLEXITY: 3
# @SEMANTICS: tests, screenshot-service, navigation, timeout-regression
# @PURPOSE: Protect dashboard screenshot navigation from brittle networkidle waits.

1
backend/src/plugins/llm_analysis/__tests__/test_service.py
View File

@@ -1,6 +1,5 @@
# region TestService [TYPE Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @COMPLEXITY: 3
# @SEMANTICS: tests, llm-analysis, fallback, provider-error, unknown-status
# @PURPOSE: Verify LLM analysis transport/provider failures do not masquerade as dashboard FAIL results.

1
backend/src/plugins/llm_analysis/plugin.py
View File

@@ -1,5 +1,4 @@
# #region LLMAnalysisPlugin [C:3] [TYPE Module] [SEMANTICS llm, analysis, dashboard, validation, documentation]
# @COMPLEXITY: 3
# @BRIEF Implements DashboardValidationPlugin and DocumentationPlugin.
# @LAYER: Domain
# @RELATION INHERITS -> [PluginBase]

1
backend/src/plugins/llm_analysis/scheduler.py
View File

@@ -1,5 +1,4 @@
# #region LLMAnalysisScheduler [C:3] [TYPE Module] [SEMANTICS scheduler, llm, validation, task, cron]
# @COMPLEXITY: 3
# @BRIEF Provides helper functions to schedule LLM-based validation tasks.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [SchedulerService]

24
backend/src/plugins/llm_analysis/service.py
View File

@@ -1,5 +1,4 @@
# #region LLMAnalysisService [C:3] [TYPE Module] [SEMANTICS llm, screenshot, playwright, openai, tenacity]
# @COMPLEXITY: 3
# @BRIEF Services for LLM interaction and dashboard screenshots.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> playwright
@@ -876,6 +875,29 @@ class LLMClient:
return await self.get_json_completion(messages)
# endregion LLMClient.test_runtime_connection
# region LLMClient.fetch_models [TYPE Function]
# @PURPOSE: Fetch available models from the provider's API.
# @PRE: Client is initialized with provider credentials.
# @POST: Returns a list of model ID strings.
# @SIDE_EFFECT: Calls external LLM API /v1/models endpoint.
async def fetch_models(self) -> List[str]:
with belief_scope("LLMClient.fetch_models"):
try:
response = await self.client.models.list()
model_ids = [m.id for m in response.data]
model_ids.sort()
logger.reason(
f"[LLMClient.fetch_models] Fetched {len(model_ids)} models from {self.base_url}",
extra={"src": "LLMClient.fetch_models"},
)
return model_ids
except Exception as e:
logger.warning(
f"[LLMClient.fetch_models] Failed to fetch models: {e}",
)
raise
# endregion LLMClient.fetch_models
# region LLMClient.analyze_dashboard [TYPE Function]
# @PURPOSE: Sends dashboard data (screenshot + logs) to LLM for health analysis.
# @PRE: screenshot_path exists, logs is a list of strings.

3
backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py
View File

@@ -1,5 +1,4 @@
# region ClickHouseInsertIntegration [TYPE Module]
# @COMPLEXITY: 3
# @SEMANTICS: test, clickhouse, integration, insert, join
# @PURPOSE: Integration tests for ClickHouse INSERT with timestamp normalization and JOIN verification.
# @LAYER: Test
@@ -369,6 +368,8 @@ class TestOrchestratorInsertFlow:
) as MockExecutor:
mock_executor = MagicMock()
MockExecutor.return_value = mock_executor
mock_executor.resolve_database_id.return_value = None
mock_executor.get_database_backend.return_value = None
mock_executor.execute_and_poll.return_value = {
"status": "success",
"query_id": "q-123",

2
backend/src/plugins/translate/__tests__/test_dictionary.py
View File

@@ -1,5 +1,4 @@
# region DictionaryTests [TYPE Module]
# @COMPLEXITY: 3
# @SEMANTICS: tests, dictionary, crud, import, filter
# @PURPOSE: Validate DictionaryManager CRUD, import, deletion guards, and batch filtering.
# @RELATION: BINDS_TO -> [DictionaryManager:Class]
@@ -38,7 +37,6 @@ from src.plugins.translate._utils import _normalize_term, _detect_delimiter
# region _FakeJob [TYPE Class]
# @COMPLEXITY: 1
# @PURPOSE: Helper to create inline TranslationJob records.
class _FakeJob:
pass

208
backend/src/plugins/translate/__tests__/test_orchestrator.py
View File

@@ -1,5 +1,4 @@
# region OrchestratorTests [TYPE Module]
# @COMPLEXITY: 3
# @SEMANTICS: test, translate, orchestrator, events
# @PURPOSE: Tests for TranslationOrchestrator: run lifecycle, partial failure, batch retry, event invariants, NULL handling.
# @LAYER: Test
@@ -10,9 +9,10 @@
# @TEST_FIXTURE: mock_config_manager -> MagicMock ConfigManager
# @TEST_EDGE: missing_preview -> raises ValueError
# @TEST_EDGE: invalid_run_status -> raises ValueError
# @TEST_EDGE: executor_failure -> run is marked FAILED
import pytest
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
from datetime import datetime, timezone
from src.plugins.translate.orchestrator import TranslationOrchestrator
@@ -234,6 +234,210 @@ class TestTranslationOrchestrator:
with pytest.raises(ValueError, match="PENDING"):
orch.execute_run(run)
# region test_execute_run_success [TYPE Function]
# @BRIEF Happy path: executor completes, SQL generated, Superset submits.
def test_execute_run_success(self, mock_job: MagicMock) -> None:
db = MagicMock()
config_manager = MagicMock()
mock_job.target_table = "target_tbl"
# Mock job query — first call returns mock_job, final re-query returns completed_run
completed_run_response = MagicMock()
completed_run_response.id = "run-1"
completed_run_response.job_id = "job-123"
completed_run_response.status = "COMPLETED"
completed_run_response.total_records = 10
completed_run_response.successful_records = 10
completed_run_response.failed_records = 0
completed_run_response.skipped_records = 0
completed_run_response.completed_at = datetime.now(timezone.utc)
completed_run_response.error_message = None
completed_run_response.insert_status = "success"
completed_run_response.superset_execution_id = "q-1"
completed_run_response.superset_execution_log = {}
# query job -> mock_job, query TranslationRun -> completed_run_response
db.query.return_value.filter.return_value.first.side_effect = [
mock_job,
completed_run_response,
]
# Create a PENDING input run
run = MagicMock()
run.id = "run-1"
run.job_id = "job-123"
run.status = "PENDING"
with patch(
"src.plugins.translate.orchestrator.TranslationExecutor"
) as MockExecutor:
mock_executor_instance = MagicMock()
mock_executor_instance.execute_run.return_value = completed_run_response
MockExecutor.return_value = mock_executor_instance
orch = TranslationOrchestrator(db, config_manager, "test-user")
with patch.object(orch, "event_log"):
with patch.object(
orch, "_generate_and_insert_sql",
return_value={"status": "success", "query_id": "q-1", "rows_affected": 10},
):
result = orch.execute_run(run)
assert result.status == "COMPLETED"
assert result.insert_status == "success"
assert result.superset_execution_id == "q-1"
assert result.total_records == 10
assert result.successful_records == 10
# endregion test_execute_run_success
# region test_execute_run_executor_failure [TYPE Function]
# @BRIEF Executor raises exception, run is marked FAILED.
def test_execute_run_executor_failure(self, mock_job: MagicMock) -> None:
db = MagicMock()
config_manager = MagicMock()
# Mock job query
db.query.return_value.filter.return_value.first.return_value = mock_job
run = MagicMock()
run.id = "run-1"
run.job_id = "job-123"
run.status = "PENDING"
run.error_message = None
with patch(
"src.plugins.translate.orchestrator.TranslationExecutor"
) as MockExecutor:
mock_executor_instance = MagicMock()
mock_executor_instance.execute_run.side_effect = ValueError(
"LLM provider unavailable"
)
MockExecutor.return_value = mock_executor_instance
orch = TranslationOrchestrator(db, config_manager, "test-user")
with patch.object(orch, "event_log"):
result = orch.execute_run(run)
assert result.status == "FAILED"
assert "LLM provider unavailable" in (result.error_message or "")
# endregion test_execute_run_executor_failure
# region test_execute_run_skip_insert [TYPE Function]
# @BRIEF skip_insert=True completes run without SQL generation or Superset submission.
def test_execute_run_skip_insert(self, mock_job: MagicMock) -> None:
db = MagicMock()
config_manager = MagicMock()
db.query.return_value.filter.return_value.first.return_value = mock_job
run = MagicMock()
run.id = "run-1"
run.job_id = "job-123"
run.status = "PENDING"
run.completed_at = None
completed_run = MagicMock()
completed_run.id = "run-1"
completed_run.job_id = "job-123"
completed_run.status = "COMPLETED"
completed_run.total_records = 5
completed_run.successful_records = 5
completed_run.failed_records = 0
completed_run.skipped_records = 0
completed_run.completed_at = datetime.now(timezone.utc)
completed_run.error_message = None
with patch(
"src.plugins.translate.orchestrator.TranslationExecutor"
) as MockExecutor:
mock_executor_instance = MagicMock()
mock_executor_instance.execute_run.return_value = completed_run
MockExecutor.return_value = mock_executor_instance
orch = TranslationOrchestrator(db, config_manager, "test-user")
with patch.object(orch, "_generate_and_insert_sql") as mock_gen_sql:
with patch.object(orch, "event_log"):
result = orch.execute_run(run, skip_insert=True)
assert result.status == "COMPLETED"
# _generate_and_insert_sql should NOT be called in skip_insert mode
mock_gen_sql.assert_not_called()
assert result.total_records == 5
# endregion test_execute_run_skip_insert
# region test_execute_run_no_job [TYPE Function]
# @BRIEF Job not found raises ValueError.
def test_execute_run_no_job(self) -> None:
db = MagicMock()
config_manager = MagicMock()
# Mock job query returns None (job not found)
db.query.return_value.filter.return_value.first.return_value = None
run = MagicMock()
run.id = "run-1"
run.job_id = "job-missing"
run.status = "PENDING"
orch = TranslationOrchestrator(db, config_manager, "test-user")
with pytest.raises(ValueError, match="not found"):
orch.execute_run(run)
# endregion test_execute_run_no_job
# region test_execute_run_insert_failure [TYPE Function]
# @BRIEF SQL generation/insert returns failure, run still completes but with error logged.
def test_execute_run_insert_failure(self, mock_job: MagicMock) -> None:
db = MagicMock()
config_manager = MagicMock()
# First query returns mock_job, re-query returns completed_run
completed_run_response = MagicMock()
completed_run_response.id = "run-1"
completed_run_response.job_id = "job-123"
completed_run_response.status = "COMPLETED"
completed_run_response.total_records = 3
completed_run_response.successful_records = 3
completed_run_response.failed_records = 0
completed_run_response.skipped_records = 0
completed_run_response.completed_at = datetime.now(timezone.utc)
completed_run_response.error_message = None
completed_run_response.insert_status = "failed"
completed_run_response.superset_execution_id = ""
completed_run_response.superset_execution_log = {}
db.query.return_value.filter.return_value.first.side_effect = [
mock_job,
completed_run_response,
]
run = MagicMock()
run.id = "run-1"
run.job_id = "job-123"
run.status = "PENDING"
with patch(
"src.plugins.translate.orchestrator.TranslationExecutor"
) as MockExecutor:
mock_executor_instance = MagicMock()
mock_executor_instance.execute_run.return_value = completed_run_response
MockExecutor.return_value = mock_executor_instance
orch = TranslationOrchestrator(db, config_manager, "test-user")
with patch.object(
orch, "_generate_and_insert_sql",
return_value={"status": "failed", "error_message": "timeout", "query_id": None},
):
with patch.object(orch, "event_log"):
result = orch.execute_run(run)
assert result.status == "COMPLETED"
assert result.insert_status == "failed"
assert "timeout" in (result.error_message or "")
# endregion test_execute_run_insert_failure
# region test_cancel_run [TYPE Function]
# @PURPOSE: Cancel a run changes status to CANCELLED.
def test_cancel_run(self) -> None:

1
backend/src/plugins/translate/__tests__/test_preview.py
View File

@@ -1,5 +1,4 @@
# region TranslationPreviewTests [TYPE Module]
# @COMPLEXITY: 3
# @SEMANTICS: test, translate, preview, session
# @PURPOSE: Test TranslationPreview service: preview, approve/edit/reject state transitions, cost estimation, acceptance gate.
# @LAYER: Test

1
backend/src/plugins/translate/__tests__/test_sql_generator.py
View File

@@ -1,5 +1,4 @@
# region SQLGeneratorTests [TYPE Module]
# @COMPLEXITY: 3
# @SEMANTICS: test, translate, sql_generator
# @PURPOSE: Tests for SQLGenerator: PostgreSQL INSERT + UPSERT, ClickHouse INSERT, dialect quoting, NULL handling, injection safety.
# @LAYER: Test

14
backend/src/plugins/translate/dictionary.py
View File

@@ -32,7 +32,6 @@ from ._utils import _normalize_term, _detect_delimiter
# @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
class DictionaryManager:
# region DictionaryManager.create_dictionary [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Create a new terminology dictionary.
# @PRE: payload contains name, source_dialect, target_dialect.
# @POST: New TerminologyDictionary row is created and returned.
@@ -60,7 +59,6 @@ class DictionaryManager:
# endregion DictionaryManager.create_dictionary
# region DictionaryManager.update_dictionary [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Update an existing terminology dictionary.
# @PRE: dict_id exists in terminology_dictionaries table.
# @POST: Dictionary metadata is updated and returned.
@@ -92,7 +90,6 @@ class DictionaryManager:
# endregion DictionaryManager.update_dictionary
# region DictionaryManager.delete_dictionary [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Delete a dictionary, blocked if attached to active/scheduled jobs.
# @PRE: dict_id exists.
# @POST: Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.
@@ -136,7 +133,6 @@ class DictionaryManager:
# endregion DictionaryManager.delete_dictionary
# region DictionaryManager.get_dictionary [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Get a single dictionary by ID with entry count.
# @PRE: dict_id exists.
# @POST: Returns dict with dictionary + entry_count or raises ValueError.
@@ -149,7 +145,6 @@ class DictionaryManager:
# endregion DictionaryManager.get_dictionary
# region DictionaryManager.list_dictionaries [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: List dictionaries with pagination and entry counts.
# @PRE: page >= 1, page_size between 1 and 100.
# @POST: Returns (list of dicts, total_count).
@@ -169,7 +164,6 @@ class DictionaryManager:
# endregion DictionaryManager.list_dictionaries
# region DictionaryManager.add_entry [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Add an entry to a dictionary, enforcing unique source_term_normalized.
# @PRE: dict_id exists. source_term and target_term are non-empty.
# @POST: New DictionaryEntry row is created or raises on duplicate.
@@ -210,7 +204,6 @@ class DictionaryManager:
# endregion DictionaryManager.add_entry
# region DictionaryManager.edit_entry [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Edit an existing dictionary entry with duplicate-aware normalization.
# @PRE: entry_id exists.
# @POST: Entry fields are updated.
@@ -255,7 +248,6 @@ class DictionaryManager:
# endregion DictionaryManager.edit_entry
# region DictionaryManager.delete_entry [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Delete a single dictionary entry.
# @PRE: entry_id exists.
# @POST: Entry is deleted.
@@ -272,7 +264,6 @@ class DictionaryManager:
# endregion DictionaryManager.delete_entry
# region DictionaryManager.clear_entries [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Delete all entries for a dictionary.
# @PRE: dict_id exists.
# @POST: All entries for the dictionary are deleted.
@@ -290,7 +281,6 @@ class DictionaryManager:
# endregion DictionaryManager.clear_entries
# region DictionaryManager.list_entries [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: List entries for a dictionary with pagination.
# @PRE: dict_id exists.
# @POST: Returns (list of entries, total_count).
@@ -316,7 +306,6 @@ class DictionaryManager:
# endregion DictionaryManager.list_entries
# region DictionaryManager.import_entries [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Import entries from CSV/TSV content with duplicate detection and conflict resolution.
# @PRE: content is valid CSV or TSV. dict_id exists.
# @POST: Entries are created/updated/skipped per conflict mode. Returns result summary.
@@ -456,7 +445,6 @@ class DictionaryManager:
# endregion DictionaryManager.import_entries
# region DictionaryManager.filter_for_batch [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.
# @PRE: job_id exists and source_texts is a list of strings.
# @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries.
@@ -555,7 +543,6 @@ class DictionaryManager:
# region DictionaryManager.submit_correction [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Submit a term correction from a run result. Validates language match, detects conflicts.
# @PRE: source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.
# @POST: Entry created or updated; origin tracking populated. Returns action + conflict info.
@@ -653,7 +640,6 @@ class DictionaryManager:
# endregion DictionaryManager.submit_correction
# region DictionaryManager.submit_bulk_corrections [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Submit multiple term corrections atomically — all succeed or all fail.
# @PRE: corrections list is non-empty. dict_id exists.
# @POST: All corrections applied or none applied with conflict list.

1
backend/src/plugins/translate/events.py
View File

@@ -48,7 +48,6 @@ class TranslationEventLog:
self.db = db
# region log_event [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Write an immutable event. Enforces terminal event invariant for non-null run_id.
# @PRE: event_type must be a known type. If run_id is not None, enforce terminal invariant.
# @POST: TranslationEvent row is created.

134
backend/src/plugins/translate/executor.py
View File

@@ -164,12 +164,102 @@ class TranslationExecutor:
# endregion execute_run
# region _fetch_source_rows [TYPE Function]
# @PURPOSE: Fetch source rows from the accepted preview session for this job.
# @PRE: job_id exists.
# @POST: Returns list of dicts with source data.
# @PURPOSE: Fetch full source dataset from Superset (via datasource) for full translation.
# @PRE: job_id exists. Job may have source_datasource_id for full fetch.
# @POST: Returns list of dicts with source data (all rows from the source datasource).
# @SIDE_EFFECT: Makes HTTP call to Superset chart data API when datasource is configured.
def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]:
with belief_scope("TranslationExecutor._fetch_source_rows"):
# Get the latest APPLIED preview session
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
# If source_datasource_id is configured, fetch ALL rows from the Superset chart data API
if job and job.source_datasource_id:
try:
logger.reason("Fetching full dataset from Superset datasource", {
"run_id": run_id,
"datasource_id": job.source_datasource_id,
"environment_id": job.environment_id,
})
# Determine environment
environments = self.config_manager.get_environments()
target_env_id = job.environment_id or job.source_dialect or ""
env_config = next(
(e for e in environments if e.id == target_env_id),
None,
)
if not env_config and environments:
env_config = environments[0]
if env_config:
from ...core.superset_client import SupersetClient
client = SupersetClient(env_config)
# Fetch dataset detail to build proper query context
dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))
# Build query context (same approach as preview but without row_limit)
query_context = client.build_dataset_preview_query_context(
dataset_id=int(job.source_datasource_id),
dataset_record=dataset_detail,
template_params={},
effective_filters=[],
)
# Remove row_limit to get ALL rows; use result_type="samples"
queries = query_context.get("queries", [])
if queries:
queries[0].pop("row_limit", None)
queries[0].pop("result_type", None)
queries[0]["metrics"] = []
query_context["result_type"] = "samples"
form_data = query_context.get("form_data", {})
form_data.pop("query_mode", None)
response = client.network.request(
method="POST",
endpoint="/api/v1/chart/data",
data=json.dumps(query_context),
headers={"Content-Type": "application/json"},
)
# Extract rows
rows = self._extract_chart_data_rows(response)
if rows:
logger.reason(f"Fetched {len(rows)} rows from Superset datasource", {
"run_id": run_id,
})
# Map rows to source_rows format
source_rows = []
for idx, row in enumerate(rows):
source_data_dict = dict(row) if row else None
source_text = str(row.get(job.translation_column, "")) if job.translation_column else json.dumps(row)
source_rows.append({
"row_index": str(idx),
"source_text": source_text,
"approved_translation": None,
"source_object_name": f"Row {idx}",
"source_data": source_data_dict,
})
return source_rows
else:
logger.explore("Superset datasource returned no rows", {
"run_id": run_id,
"datasource_id": job.source_datasource_id,
})
else:
logger.explore("No environment config found for datasource fetch", {
"env_id": target_env_id,
})
except Exception as e:
logger.explore("Failed to fetch full dataset from Superset, falling back to preview", {
"run_id": run_id,
"error": str(e),
})
# Fall through to preview-based fetch
# Fallback: get the latest APPLIED preview session
session = (
self.db.query(TranslationPreviewSession)
.filter(
@@ -195,20 +285,48 @@ class TranslationExecutor:
source_rows = []
for rec in records:
source_data_dict = None
if hasattr(rec, "source_data") and rec.source_data:
source_data_dict = dict(rec.source_data)
source_rows.append({
"row_index": rec.source_object_id or "0",
"source_text": rec.source_sql or "",
"approved_translation": rec.target_sql if rec.status == "APPROVED" else None,
"source_object_name": rec.source_object_name or "",
"source_data": source_data_dict,
})
logger.reason(f"Fetched {len(source_rows)} source rows from preview", {
logger.reason(f"Fetched {len(source_rows)} source rows from preview fallback", {
"run_id": run_id,
"session_id": session.id,
})
return source_rows
# endregion _fetch_source_rows
# region _extract_chart_data_rows [TYPE Function]
# @PURPOSE: Extract data rows from Superset chart data API response.
# @POST: Returns list of dicts with column-value pairs.
@staticmethod
def _extract_chart_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]:
result = response.get("result")
if isinstance(result, list):
for item in result:
if isinstance(item, dict):
data = item.get("data")
if isinstance(data, list) and data:
return data
if isinstance(result, dict):
data = result.get("data")
if isinstance(data, list) and data:
return data
data = response.get("data")
if isinstance(data, list) and data:
return data
if isinstance(result, list):
return result
return []
# endregion _extract_chart_data_rows
# region _process_batch [TYPE Function]
# @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records.
# @PRE: job and batch_rows are valid.
@@ -272,6 +390,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data"),
status="SUCCESS",
)
self.db.add(record)
@@ -398,6 +517,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data"),
status="FAILED",
error_message=f"LLM call failed after {retries} retries: {last_error}",
)
@@ -425,6 +545,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data"),
status="SKIPPED",
error_message=f"LLM parse failure: {e}",
)
@@ -456,6 +577,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data"),
status="SKIPPED",
error_message="NULL translation returned by LLM",
)
@@ -474,6 +596,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data"),
status="SKIPPED",
error_message="Empty translation returned by LLM",
)
@@ -490,6 +613,7 @@ class TranslationExecutor:
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data"),
status="SUCCESS",
)
self.db.add(record)

78
backend/src/plugins/translate/orchestrator.py
View File

@@ -63,7 +63,6 @@ class TranslationOrchestrator:
self._job: Optional[TranslationJob] = None
# region start_run [TYPE Function]
# @COMPLEXITY: 5
# @PURPOSE: Start a new translation run for a job.
# @PRE: job_id exists. For manual runs, there must be an accepted preview session.
# @POST: TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.
@@ -105,6 +104,7 @@ class TranslationOrchestrator:
"source_key_cols": job.source_key_cols,
"target_key_cols": job.target_key_cols,
"translation_column": job.translation_column,
"target_column": job.target_column,
"context_columns": job.context_columns,
"target_language": job.target_language,
"provider_id": job.provider_id,
@@ -263,7 +263,11 @@ class TranslationOrchestrator:
run.insert_status = insert_result.get("status")
run.superset_execution_id = str(insert_result.get("query_id") or "")
run.superset_execution_log = insert_result
run.status = "COMPLETED" if insert_result.get("status") == "success" else "COMPLETED"
# Preserve the translation-phase status. If the executor already
# marked the run FAILED (all LLM calls failed) we do NOT upgrade it
# to COMPLETED just because the insert phase ran.
if run.status != "FAILED":
run.status = "COMPLETED"
if insert_result.get("error_message"):
run.error_message = insert_result["error_message"]
run.completed_at = datetime.now(timezone.utc)
@@ -288,7 +292,9 @@ class TranslationOrchestrator:
)
self.db.commit()
self.db.refresh(run)
# Re-query run after commit — refresh may fail if the object was
# created in a different session or became detached during commit.
run = self.db.query(TranslationRun).filter(TranslationRun.id == run.id).first()
logger.reflect("Run execution complete", {
"run_id": run.id,
@@ -329,11 +335,20 @@ class TranslationOrchestrator:
"dialect": job.database_dialect or job.target_dialect,
})
# Build rows for SQL generation
# Determine effective target column for INSERT (defaults to translation_column)
effective_target = job.target_column or job.translation_column
# Build columns for SQL generation
columns = job.context_columns or []
# Always include translation_column (original text) if it's different from target
if job.translation_column and job.translation_column not in columns:
columns.append(job.translation_column)
# Add target_column separately if it differs from translation_column
if effective_target and effective_target != job.translation_column and effective_target not in columns:
columns.append(effective_target)
# Also include key columns if used for upsert
if job.target_key_cols:
for k in job.target_key_cols:
@@ -343,27 +358,57 @@ class TranslationOrchestrator:
rows_for_sql = []
for rec in records:
row_data = {}
source_data = rec.source_data or {}
# Context columns from source data
if job.context_columns:
for col in job.context_columns:
row_data[col] = ""
if job.translation_column:
row_data[job.translation_column] = rec.target_sql or ""
row_data[col] = source_data.get(col, "")
# Original text column (translation_column)
if job.translation_column and job.translation_column not in (job.target_key_cols or []):
# If target_column differs, keep the original value from source_data;
# otherwise use the translated value
if effective_target and effective_target != job.translation_column:
row_data[job.translation_column] = source_data.get(job.translation_column, "")
else:
row_data[job.translation_column] = rec.target_sql or ""
# Translated text goes into target_column (may be same as translation_column)
if effective_target:
row_data[effective_target] = rec.target_sql or ""
# Key columns from source data
if job.target_key_cols:
source_data = rec.source_data or {}
for k in job.target_key_cols:
# Use source_data value if available, fall back to empty string
row_data[k] = source_data.get(k, "")
rows_for_sql.append(row_data)
if not columns:
# Use target_sql as the sole column
columns = [job.translation_column or "translated_text"]
columns = [effective_target or "translated_text"]
rows_for_sql = [{columns[0]: rec.target_sql or ""} for rec in records]
# Resolve the real database backend engine from Superset
try:
env_id = job.environment_id or job.source_dialect or ""
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
executor.resolve_database_id(
target_database_id=job.target_database_id,
)
real_backend = executor.get_database_backend()
except Exception as e:
logger.explore("Failed to resolve database backend, falling back to job dialet", {
"error": str(e),
})
real_backend = None
dialect = real_backend or job.database_dialect or job.target_dialect or "postgresql"
# Generate SQL
try:
sql, row_count = SQLGenerator.generate(
dialect=job.database_dialect or job.target_dialect or "postgresql",
dialect=dialect,
target_schema=job.target_schema,
target_table=job.target_table or "translated_data",
columns=columns,
@@ -375,19 +420,24 @@ class TranslationOrchestrator:
logger.explore("SQL generation failed", {"error": str(e)})
return {"status": "failed", "error_message": str(e), "query_id": None}
logger.reason("SQL generated with dialect", {
"dialect": dialect,
"real_backend": real_backend,
"job_database_dialect": job.database_dialect,
"job_target_dialect": job.target_dialect,
})
# Log insert phase start
self.event_log.log_event(
job_id=job.id,
run_id=run.id,
event_type="INSERT_PHASE_STARTED",
payload={"sql_length": len(sql), "row_count": row_count},
payload={"sql_length": len(sql), "row_count": row_count, "dialect": dialect},
created_by=self.current_user,
)
# Submit to Superset
try:
env_id = job.environment_id or job.source_dialect or ""
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
result = executor.execute_and_poll(
sql=sql,
max_polls=30,

17
backend/src/plugins/translate/preview.py
View File

@@ -134,7 +134,6 @@ class TranslationPreview:
self.current_user = current_user
# region preview_rows [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.
# @PRE: job_id exists and job has source_datasource_id, translation_column configured.
# @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session.
@@ -290,6 +289,19 @@ class TranslationPreview:
status = "PENDING"
feedback = None
# Extract source_data: store original row key columns for upsert matching
source_row = meta.get("source_row", {})
source_data = None
if job.target_key_cols:
source_data = {
k: source_row.get(k)
for k in job.target_key_cols
if k in source_row
}
elif source_row:
# No key columns configured — store the full row as fallback
source_data = dict(source_row)
record = TranslationPreviewRecord(
id=str(uuid.uuid4()),
session_id=session.id,
@@ -298,6 +310,7 @@ class TranslationPreview:
source_object_type="table_row",
source_object_id=str(idx),
source_object_name=f"Row {idx + 1}",
source_data=source_data,
status=status,
feedback=feedback,
created_at=datetime.now(timezone.utc),
@@ -516,7 +529,6 @@ class TranslationPreview:
# endregion get_preview_session
# region _fetch_sample_rows [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Fetch sample rows from the Superset dataset for preview.
# @PRE: job has source_datasource_id and translation_column.
# @POST: Returns list of dicts with row data.
@@ -636,7 +648,6 @@ class TranslationPreview:
# endregion _extract_data_rows
# region _call_llm [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Call the configured LLM provider with the preview prompt.
# @PRE: job has a valid provider_id.
# @POST: Returns raw LLM response string.

5
backend/src/plugins/translate/service.py
View File

@@ -241,12 +241,14 @@ class TranslateJobService:
source_key_cols=payload.source_key_cols or [],
target_key_cols=payload.target_key_cols or [],
translation_column=payload.translation_column,
target_column=payload.target_column,
context_columns=payload.context_columns or [],
target_language=payload.target_language,
provider_id=payload.provider_id,
batch_size=payload.batch_size,
upsert_strategy=payload.upsert_strategy,
environment_id=payload.environment_id,
target_database_id=payload.target_database_id,
status="DRAFT",
created_by=self.current_user,
)
@@ -360,6 +362,7 @@ class TranslateJobService:
source_key_cols=source.source_key_cols,
target_key_cols=source.target_key_cols,
translation_column=source.translation_column,
target_column=source.target_column,
context_columns=source.context_columns,
target_language=source.target_language,
provider_id=source.provider_id,
@@ -461,6 +464,7 @@ def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -
source_key_cols=job.source_key_cols or [],
target_key_cols=job.target_key_cols or [],
translation_column=job.translation_column,
target_column=job.target_column,
context_columns=job.context_columns or [],
target_language=job.target_language,
provider_id=job.provider_id,
@@ -472,6 +476,7 @@ def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -
updated_at=job.updated_at,
dictionary_ids=dict_ids or [],
environment_id=job.environment_id,
target_database_id=job.target_database_id,
)
# #endregion job_to_response

15
backend/src/plugins/translate/sql_generator.py
View File

@@ -90,11 +90,16 @@ def _encode_sql_value(value: Any, dialect: Optional[str] = None) -> str:
if isinstance(value, bool):
return "TRUE" if value else "FALSE"
# For ClickHouse: try to normalize timestamp-like string values
if dialect in CLICKHOUSE_DIALECTS and isinstance(value, str) and value:
normalized = _normalize_timestamp_value(value)
if normalized:
return f"'{normalized}'"
# For ClickHouse: try to normalize timestamp-like values (both string and numeric)
if dialect in CLICKHOUSE_DIALECTS:
if isinstance(value, str) and value:
normalized = _normalize_timestamp_value(value)
if normalized:
return f"'{normalized}'"
elif isinstance(value, (int, float)):
normalized = _normalize_timestamp_value(value)
if normalized:
return f"'{normalized}'"
if isinstance(value, (int, float)):
return str(value)

146
backend/src/plugins/translate/superset_executor.py
View File

@@ -32,6 +32,7 @@ class SupersetSqlLabExecutor:
self.env_id = env_id
self._client: Optional[SupersetClient] = None
self._database_id: Optional[int] = None
self._database_backend: Optional[str] = None
# region _get_client [TYPE Function]
# @PURPOSE: Lazy-initialize SupersetClient for the configured environment.
@@ -53,15 +54,43 @@ class SupersetSqlLabExecutor:
# endregion _get_client
# region resolve_database_id [TYPE Function]
# @PURPOSE: Resolve the target database ID from the environment.
# @PURPOSE: Resolve the target database ID from the environment and fetch its backend engine.
# @PRE: database_name or database_id should be known.
# @POST: Returns database_id integer or raises ValueError.
# @SIDE_EFFECT: Fetches databases list from Superset.
def resolve_database_id(self, database_name: Optional[str] = None) -> int:
# @POST: Returns database_id integer or raises ValueError. Also sets self._database_backend.
# @SIDE_EFFECT: Fetches databases list from Superset; optionally fetches single DB for backend.
def resolve_database_id(
self,
database_name: Optional[str] = None,
target_database_id: Optional[str] = None,
) -> int:
with belief_scope("SupersetSqlLabExecutor.resolve_database_id"):
client = self._get_client()
# If a specific target_database_id is provided, use it directly
if target_database_id:
try:
db_id = int(target_database_id)
# Fetch full DB info to get backend
db_info = client.get_database(db_id)
result = db_info.get("result", db_info)
self._database_id = result.get("id", db_id)
self._database_backend = result.get("backend") or result.get("engine")
logger.reason("Resolved database ID by target_database_id", {
"target_database_id": target_database_id,
"database_id": self._database_id,
"backend": self._database_backend,
})
return self._database_id
except Exception as e:
logger.explore("Failed to resolve by target_database_id, falling back", {
"target_database_id": target_database_id,
"error": str(e),
})
# Fall through to default resolution
# Fetch full database list with backend info
_, databases = client.get_databases(
query={"columns": ["id", "database_name"]}
query={"columns": ["id", "database_name", "backend"]}
)
if not databases:
raise ValueError("No databases found in Superset environment")
@@ -70,9 +99,11 @@ class SupersetSqlLabExecutor:
for db in databases:
if db.get("database_name", "").lower() == database_name.lower():
self._database_id = db["id"]
self._database_backend = db.get("backend")
logger.reason("Resolved database ID by name", {
"database_name": database_name,
"database_id": self._database_id,
"backend": self._database_backend,
})
return self._database_id
raise ValueError(
@@ -81,13 +112,22 @@ class SupersetSqlLabExecutor:
# Default: use first database
self._database_id = databases[0]["id"]
self._database_backend = databases[0].get("backend")
logger.reason("Using default database", {
"database_name": databases[0].get("database_name"),
"database_id": self._database_id,
"backend": self._database_backend,
})
return self._database_id
# endregion resolve_database_id
# region get_database_backend [TYPE Function]
# @PURPOSE: Return the cached database backend/engine string.
# @POST: Returns backend string or None if not yet resolved.
def get_database_backend(self) -> Optional[str]:
return self._database_backend
# endregion get_database_backend
# region execute_sql [TYPE Function]
# @PURPOSE: Submit SQL to Superset SQL Lab and return execution tracking info.
# @PRE: sql is valid SQL string. database_id is a valid Superset DB ID.
@@ -117,27 +157,75 @@ class SupersetSqlLabExecutor:
"runAsync": run_async,
"schema": None,
"tab": "translation-insert",
"client_id": f"trl-{uuid.uuid4().hex[:7]}", # max 11 chars for Superset varchar(11)
}
try:
response = client.network.request(
method="POST",
endpoint="/api/v1/sqllab/execute/",
data=json.dumps(payload),
headers={"Content-Type": "application/json"},
)
except Exception as e:
logger.explore("SQL Lab execute failed", {"error": str(e)})
raise ValueError(f"Superset SQL Lab execute failed: {e}")
# Try multiple SQL Lab endpoints — some Superset versions use /sqllab/execute/,
# others use /sql_lab/execute/ (with underscore)
candidate_endpoints = ["/api/v1/sqllab/execute/", "/api/v1/sql_lab/execute/"]
response = None
last_error = None
for endpoint in candidate_endpoints:
try:
response = client.network.request(
method="POST",
endpoint=endpoint,
data=json.dumps(payload),
headers={"Content-Type": "application/json"},
)
if isinstance(response, dict) and response:
logger.reason(f"SQL Lab endpoint succeeded", extra={
"endpoint": endpoint,
})
break
except Exception as ep_err:
last_error = ep_err
logger.explore(f"SQL Lab endpoint failed, trying next", extra={
"error": str(ep_err),
"endpoint": endpoint,
})
response = None
# Parse response
if response is None:
error_msg = f"All SQL Lab endpoints failed. Last error: {last_error}"
logger.explore("SQL Lab execute failed", extra={"error": error_msg})
raise ValueError(f"Superset SQL Lab execute failed: {error_msg}")
# Parse response — try multiple known Superset response formats
result = response if isinstance(response, dict) else {}
query_id = result.get("query_id") or result.get("id")
# Superset may return query_id at top level, nested in "result", as "id", or in "query" block
query_id = (
result.get("query_id")
or result.get("id")
or (result.get("result") or {}).get("query_id")
or (result.get("result") or {}).get("id")
or (result.get("query") or {}).get("query_id")
or (result.get("query") or {}).get("id")
)
status = result.get("status", "unknown")
# Log full response for debugging if query_id is missing
if not query_id:
logger.explore("No query_id from SQL Lab execute", extra={
"database_id": db_id,
"status": status,
"raw_response_keys": list(result.keys()),
"raw_response_preview": json.dumps(result)[:2000],
})
logger.reason("SQL Lab execute response (no query_id)", extra={
"database_id": db_id,
"status": status,
"response_keys": list(result.keys()),
"has_result": "result" in result,
"has_query": "query" in result,
"raw_preview": json.dumps(result)[:2000],
})
logger.reason("SQL Lab execute response", {
"query_id": query_id,
"status": status,
"response_keys": list(result.keys()),
})
return {
@@ -174,7 +262,9 @@ class SupersetSqlLabExecutor:
method="GET",
endpoint=f"/api/v1/query/{query_id}",
)
result = response if isinstance(response, dict) else {}
raw = response if isinstance(response, dict) else {}
# Superset wraps query data in a nested "result" field
result = raw.get("result", raw)
status = result.get("status", "unknown")
state = result.get("state", result.get("status", ""))
@@ -255,9 +345,27 @@ class SupersetSqlLabExecutor:
exec_result = self.execute_sql(
sql=sql,
database_id=database_id,
run_async=True,
run_async=False, # Sync mode — Superset returns result directly
)
status = exec_result.get("status", "unknown")
# Sync mode: response already has the final status and data
if status == "success":
logger.reason("SQL execution completed (sync)", {
"query_id": exec_result.get("query_id"),
})
return {
"query_id": exec_result.get("query_id"),
"status": "success",
"state": "success",
"rows_affected": exec_result.get("raw_response", {}).get("query", {}).get("rows"),
"error_message": None,
"results": exec_result.get("raw_response", {}).get("data"),
"completed_on": exec_result.get("raw_response", {}).get("query", {}).get("endDttm"),
}
# For async mode (fallback): poll for completion
query_id = exec_result.get("query_id")
if not query_id:
logger.explore("No query_id from SQL Lab execute", {

8
backend/src/schemas/translate.py
View File

@@ -23,7 +23,8 @@ class TranslateJobCreate(BaseModel):
target_table: Optional[str] = Field(None, description="Target table name")
source_key_cols: Optional[List[str]] = Field(default_factory=list, description="Source key column names")
target_key_cols: Optional[List[str]] = Field(default_factory=list, description="Target key column names")
translation_column: Optional[str] = Field(None, description="Column to translate")
translation_column: Optional[str] = Field(None, description="Source column to translate")
target_column: Optional[str] = Field(None, description="Target column for translated output (defaults to translation_column)")
context_columns: Optional[List[str]] = Field(default_factory=list, description="Context column names")
target_language: Optional[str] = Field(None, description="Target language code")
provider_id: Optional[str] = Field(None, description="LLM provider ID")
@@ -31,6 +32,7 @@ class TranslateJobCreate(BaseModel):
upsert_strategy: str = Field("MERGE", description="UPSERT strategy: MERGE, INSERT, UPDATE")
dictionary_ids: Optional[List[str]] = Field(default_factory=list, description="Associated terminology dictionary IDs")
environment_id: Optional[str] = Field(None, description="Superset environment ID")
target_database_id: Optional[str] = Field(None, description="Superset database ID for SQL Lab insert target")
# #endregion TranslateJobCreate
@@ -49,6 +51,7 @@ class TranslateJobUpdate(BaseModel):
source_key_cols: Optional[List[str]] = None
target_key_cols: Optional[List[str]] = None
translation_column: Optional[str] = None
target_column: Optional[str] = None
context_columns: Optional[List[str]] = None
target_language: Optional[str] = None
provider_id: Optional[str] = None
@@ -57,6 +60,7 @@ class TranslateJobUpdate(BaseModel):
status: Optional[str] = None
dictionary_ids: Optional[List[str]] = None
environment_id: Optional[str] = None
target_database_id: Optional[str] = None
# #endregion TranslateJobUpdate
@@ -76,6 +80,7 @@ class TranslateJobResponse(BaseModel):
source_key_cols: Optional[List[str]] = None
target_key_cols: Optional[List[str]] = None
translation_column: Optional[str] = None
target_column: Optional[str] = None
context_columns: Optional[List[str]] = None
target_language: Optional[str] = None
provider_id: Optional[str] = None
@@ -87,6 +92,7 @@ class TranslateJobResponse(BaseModel):
updated_at: Optional[datetime] = None
dictionary_ids: Optional[List[str]] = None
environment_id: Optional[str] = None
target_database_id: Optional[str] = None
class Config:
from_attributes = True

1
backend/src/services/__tests__/test_encryption_manager.py
View File

@@ -1,6 +1,5 @@
# region test_encryption_manager [TYPE Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @COMPLEXITY: 3
# @SEMANTICS: encryption, security, fernet, api-keys, tests
# @PURPOSE: Unit tests for EncryptionManager encrypt/decrypt functionality.
# @LAYER: Domain

1
backend/src/services/__tests__/test_health_service.py
View File

@@ -5,7 +5,6 @@ from src.services.health_service import HealthService
from src.models.llm import ValidationRecord
# region test_health_service [TYPE Module]
# @COMPLEXITY: 3
# @PURPOSE: Unit tests for HealthService aggregation logic.
# @RELATION: VERIFIES ->[src.services.health_service.HealthService]

10
backend/src/services/__tests__/test_llm_plugin_persistence.py
View File

@@ -1,6 +1,5 @@
# region test_llm_plugin_persistence [TYPE Module]
# @RELATION: VERIFIES -> [DashboardValidationPlugin:Class]
# @COMPLEXITY: 3
# @PURPOSE: Regression test for ValidationRecord persistence fields populated from task context.
import types
@@ -11,7 +10,6 @@ from src.plugins.llm_analysis import plugin as plugin_module
# region _DummyLogger [TYPE Class]
# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module]
# @COMPLEXITY: 1
# @PURPOSE: Minimal logger shim for TaskContext-like objects used in tests.
# @INVARIANT: Logging methods are no-ops and must not mutate test state.
class _DummyLogger:
@@ -36,7 +34,6 @@ class _DummyLogger:
# region _FakeDBSession [TYPE Class]
# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module]
# @COMPLEXITY: 2
# @PURPOSE: Captures persisted records for assertion and mimics SQLAlchemy session methods used by plugin.
# @INVARIANT: add/commit/close provide only persistence signals asserted by this test.
class _FakeDBSession:
@@ -61,7 +58,6 @@ class _FakeDBSession:
# region test_dashboard_validation_plugin_persists_task_and_environment_ids [TYPE Function]
# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module]
# @RELATION: VERIFIES -> [DashboardValidationPlugin:Class]
# @COMPLEXITY: 2
# @PURPOSE: Ensure db ValidationRecord includes context.task_id and params.environment_id.
# @INVARIANT: Assertions remain restricted to persisted task/environment identity fields and session lifecycle signals.
@pytest.mark.asyncio
@@ -82,7 +78,6 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
# region _FakeProviderService [TYPE Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: LLM provider service stub returning deterministic provider and decrypted API key for plugin tests.
# @INVARIANT: Returns same provider and key regardless of provider_id argument; no lookup logic.
class _FakeProviderService:
@@ -99,7 +94,6 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
# region _FakeScreenshotService [TYPE Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: Screenshot service stub that accepts capture_dashboard calls without side effects.
# @INVARIANT: capture_dashboard is intentionally permissive for this persistence-focused test and does not validate argument values.
class _FakeScreenshotService:
@@ -113,7 +107,6 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
# region _FakeLLMClient [TYPE Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 2
# @PURPOSE: Deterministic LLM client double returning canonical analysis payload for persistence-path assertions.
# @INVARIANT: analyze_dashboard is side-effect free and returns schema-compatible PASS result.
class _FakeLLMClient:
@@ -137,7 +130,6 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
# region _FakeNotificationService [TYPE Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: Notification service stub that accepts plugin dispatch_report payload without introducing side effects.
# @INVARIANT: dispatch_report accepts arbitrary keyword payloads because this test verifies persistence fields, not notification payload schema.
class _FakeNotificationService:
@@ -151,7 +143,6 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
# region _FakeConfigManager [TYPE Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: Config manager stub providing storage root path and minimal settings for plugin execution path.
# @INVARIANT: Only storage.root_path and llm fields are safe to access; all other settings fields are absent.
class _FakeConfigManager:
@@ -170,7 +161,6 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
# region _FakeSupersetClient [TYPE Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: Superset client stub exposing network.request as a lambda that returns empty result list.
# @INVARIANT: network.request intentionally accepts arbitrary keyword payloads because response shape, not request signature, is the persistence-path dependency.
class _FakeSupersetClient:

7
backend/src/services/__tests__/test_llm_prompt_templates.py
View File

@@ -1,5 +1,4 @@
# region test_llm_prompt_templates [TYPE Module]
# @COMPLEXITY: 3
# @SEMANTICS: tests, llm, prompts, templates, settings
# @PURPOSE: Validate normalization and rendering behavior for configurable LLM prompt templates.
# @LAYER: Domain Tests
@@ -19,7 +18,6 @@ from src.services.llm_prompt_templates import (
# region test_normalize_llm_settings_adds_default_prompts [TYPE Function]
# @RELATION: BINDS_TO -> test_llm_prompt_templates
# @COMPLEXITY: 3
# @PURPOSE: Ensure legacy/partial llm settings are expanded with all prompt defaults.
# @PRE: Input llm settings do not contain complete prompts object.
# @POST: Returned structure includes required prompt templates with fallback defaults.
@@ -43,7 +41,6 @@ def test_normalize_llm_settings_adds_default_prompts():
# region test_normalize_llm_settings_keeps_custom_prompt_values [TYPE Function]
# @RELATION: BINDS_TO -> test_llm_prompt_templates
# @COMPLEXITY: 3
# @PURPOSE: Ensure user-customized prompt values are preserved during normalization.
# @PRE: Input llm settings contain custom prompt override.
# @POST: Custom prompt value remains unchanged in normalized output.
@@ -59,7 +56,6 @@ def test_normalize_llm_settings_keeps_custom_prompt_values():
# region test_render_prompt_replaces_known_placeholders [TYPE Function]
# @RELATION: BINDS_TO -> test_llm_prompt_templates
# @COMPLEXITY: 3
# @PURPOSE: Ensure template placeholders are deterministically replaced.
# @PRE: Template contains placeholders matching provided variables.
# @POST: Rendered prompt string contains substituted values.
@@ -77,7 +73,6 @@ def test_render_prompt_replaces_known_placeholders():
# region test_is_multimodal_model_detects_known_vision_models [TYPE Function]
# @RELATION: BINDS_TO -> test_llm_prompt_templates
# @COMPLEXITY: 2
# @PURPOSE: Ensure multimodal model detection recognizes common vision-capable model names.
def test_is_multimodal_model_detects_known_vision_models():
assert is_multimodal_model("gpt-4o") is True
@@ -91,7 +86,6 @@ def test_is_multimodal_model_detects_known_vision_models():
# region test_resolve_bound_provider_id_prefers_binding_then_default [TYPE Function]
# @RELATION: BINDS_TO -> test_llm_prompt_templates
# @COMPLEXITY: 2
# @PURPOSE: Verify provider binding resolution priority.
def test_resolve_bound_provider_id_prefers_binding_then_default():
settings = {
@@ -107,7 +101,6 @@ def test_resolve_bound_provider_id_prefers_binding_then_default():
# region test_normalize_llm_settings_keeps_assistant_planner_settings [TYPE Function]
# @RELATION: BINDS_TO -> test_llm_prompt_templates
# @COMPLEXITY: 2
# @PURPOSE: Ensure assistant planner provider/model fields are preserved and normalized.
def test_normalize_llm_settings_keeps_assistant_planner_settings():
normalized = normalize_llm_settings(

3
backend/src/services/__tests__/test_llm_provider.py
View File

@@ -1,6 +1,5 @@
# region test_llm_provider [TYPE Module]
# @RELATION: VERIFIES -> [src.services.llm_provider:Module]
# @COMPLEXITY: 3
# @SEMANTICS: tests, llm-provider, encryption, contract
# @PURPOSE: Contract testing for LLMProviderService and EncryptionManager
# endregion test_llm_provider
@@ -68,7 +67,6 @@ def test_decrypt_invalid_data():
# region mock_db [TYPE Fixture]
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @COMPLEXITY: 1
# @PURPOSE: MagicMock(spec=Session) fixture providing a constrained DB session double for LLMProviderService tests.
# @INVARIANT: Chained calls beyond Session spec create unconstrained intermediate mocks; only top-level query/add/commit are spec-enforced.
@pytest.fixture
@@ -82,7 +80,6 @@ def mock_db():
# region service [TYPE Fixture]
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @COMPLEXITY: 1
# @PURPOSE: LLMProviderService fixture wired to mock_db for provider CRUD tests.
@pytest.fixture
def service(mock_db):

1
backend/src/services/__tests__/test_rbac_permission_catalog.py
View File

@@ -1,6 +1,5 @@
# region test_rbac_permission_catalog [TYPE Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @COMPLEXITY: 3
# @SEMANTICS: tests, rbac, permissions, catalog, discovery, sync
# @PURPOSE: Verifies RBAC permission catalog discovery and idempotent synchronization behavior.
# @LAYER: Service Tests

1
backend/src/services/__tests__/test_resource_service.py
View File

@@ -1,5 +1,4 @@
# region TestResourceService [TYPE Module]
# @COMPLEXITY: 3
# @SEMANTICS: resource-service, tests, dashboards, datasets, activity
# @PURPOSE: Unit tests for ResourceService
# @LAYER: Service

4
backend/src/services/auth_service.py
View File

@@ -31,7 +31,6 @@ from ..core.logger import belief_scope
# @RELATION DEPENDS_ON -> [Role]
class AuthService:
# region AuthService_init [TYPE Function]
# @COMPLEXITY: 1
# @PURPOSE: Initializes the authentication service with repository access over an active DB session.
# @PRE: db is a valid SQLAlchemy Session instance bound to the auth persistence context.
# @POST: self.repo is initialized and ready for auth user/role CRUD operations.
@@ -45,7 +44,6 @@ class AuthService:
# endregion AuthService_init
# region AuthService.authenticate_user [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Validates credentials and account state for local username/password authentication.
# @PRE: username and password are non-empty credential inputs.
# @POST: Returns User only when user exists, is active, and password hash verification succeeds; otherwise returns None.
@@ -76,7 +74,6 @@ class AuthService:
# endregion AuthService.authenticate_user
# region AuthService.create_session [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Issues an access token payload for an already authenticated user.
# @PRE: user is a valid User entity containing username and iterable roles with role.name values.
# @POST: Returns session dict with non-empty access_token and token_type='bearer'.
@@ -98,7 +95,6 @@ class AuthService:
# endregion AuthService.create_session
# region AuthService.provision_adfs_user [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Performs ADFS Just-In-Time provisioning and role synchronization from AD group mappings.
# @PRE: user_info contains identity claims where at least one of 'upn' or 'email' is present; 'groups' may be absent.
# @POST: Returns persisted user entity with roles synchronized to mapped AD groups and refreshed state.

1
backend/src/services/clean_release/__tests__/test_audit_service.py
View File

@@ -1,6 +1,5 @@
# region TestAuditService [TYPE Module]
# @RELATION: [DEPENDS_ON] ->[AuditService]
# @COMPLEXITY: 3
# @SEMANTICS: tests, clean-release, audit, logging
# @PURPOSE: Validate audit hooks emit expected log patterns for clean release lifecycle.
# @LAYER: Infra

1
backend/src/services/clean_release/__tests__/test_compliance_orchestrator.py
View File

@@ -1,6 +1,5 @@
# region TestComplianceOrchestrator [TYPE Module]
# @RELATION: [DEPENDS_ON] ->[ComplianceOrchestrator]
# @COMPLEXITY: 3
# @SEMANTICS: tests, clean-release, orchestrator, stage-state-machine
# @PURPOSE: Validate compliance orchestrator stage transitions and final status derivation.
# @LAYER: Domain

1
backend/src/services/clean_release/__tests__/test_manifest_builder.py
View File

@@ -1,5 +1,4 @@
# region TestManifestBuilder [TYPE Module]
# @COMPLEXITY: 5
# @SEMANTICS: tests, clean-release, manifest, deterministic
# @PURPOSE: Validate deterministic manifest generation behavior for US1.
# @LAYER: Domain

1
backend/src/services/clean_release/__tests__/test_preparation_service.py
View File

@@ -1,5 +1,4 @@
# region TestPreparationService [TYPE Module]
# @COMPLEXITY: 3
# @SEMANTICS: tests, clean-release, preparation, flow
# @PURPOSE: Validate release candidate preparation flow, including policy evaluation and manifest persisting.
# @LAYER: Domain

1
backend/src/services/clean_release/__tests__/test_report_builder.py
View File

@@ -1,6 +1,5 @@
# region TestReportBuilder [TYPE Module]
# @RELATION: [DEPENDS_ON] ->[ReportBuilder]
# @COMPLEXITY: 3
# @SEMANTICS: tests, clean-release, report-builder, counters
# @PURPOSE: Validate compliance report builder counter integrity and blocked-run constraints.
# @LAYER: Domain

1
backend/src/services/clean_release/__tests__/test_source_isolation.py
View File

@@ -1,6 +1,5 @@
# region TestSourceIsolation [TYPE Module]
# @RELATION: [DEPENDS_ON] ->[SourceIsolation]
# @COMPLEXITY: 3
# @SEMANTICS: tests, clean-release, source-isolation, internal-only
# @PURPOSE: Verify internal source registry validation behavior.
# @LAYER: Domain

1
backend/src/services/clean_release/__tests__/test_stages.py
View File

@@ -1,6 +1,5 @@
# region TestStages [TYPE Module]
# @RELATION: [DEPENDS_ON] ->[ComplianceStages]
# @COMPLEXITY: 3
# @SEMANTICS: tests, clean-release, compliance, stages
# @PURPOSE: Validate final status derivation logic from stage results.
# @LAYER: Domain

5
backend/src/services/clean_release/repositories/manifest_repository.py
View File

@@ -19,7 +19,6 @@ class ManifestRepository:
"""Repository for distribution manifest persistence."""
# region ManifestRepository.__init__ [TYPE Function]
# @COMPLEXITY: 1
# @PURPOSE: Initialize repository with an active SQLAlchemy session.
# @PRE: db is a valid SQLAlchemy Session instance.
# @POST: Repository is ready for database operations.
@@ -28,7 +27,6 @@ class ManifestRepository:
# endregion ManifestRepository.__init__
# region ManifestRepository.save [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Persist a DistributionManifest to the database.
# @PRE: manifest is a valid DistributionManifest instance with required fields populated.
# @POST: Manifest is committed to database and refreshed with generated ID.
@@ -43,7 +41,6 @@ class ManifestRepository:
# endregion ManifestRepository.save
# region ManifestRepository.get_by_id [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Retrieve a single DistributionManifest by its primary key.
# @PRE: manifest_id is a valid string identifier.
# @POST: Returns DistributionManifest if found, None otherwise.
@@ -56,7 +53,6 @@ class ManifestRepository:
# endregion ManifestRepository.get_by_id
# region ManifestRepository.get_latest_for_candidate [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Retrieve the most recent manifest version for a given candidate.
# @PRE: candidate_id is a valid string identifier.
# @POST: Returns the highest manifest_version manifest for the candidate, or None.
@@ -72,7 +68,6 @@ class ManifestRepository:
# endregion ManifestRepository.get_latest_for_candidate
# region ManifestRepository.list_by_candidate [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: List all manifests for a specific candidate, ordered by version.
# @PRE: candidate_id is a valid string identifier.
# @POST: Returns a list of DistributionManifest instances (may be empty).

6
backend/src/services/dataset_review/clarification_engine.py
View File

@@ -105,7 +105,6 @@ class ClarificationAnswerCommand:
# @SIDE_EFFECT: Mutates clarification answers, session flags, and related clarification findings.
class ClarificationEngine:
# region ClarificationEngine_init [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Bind repository dependency for clarification persistence operations.
def __init__(self, repository: DatasetReviewSessionRepository) -> None:
self.repository = repository
@@ -113,7 +112,6 @@ class ClarificationEngine:
# endregion ClarificationEngine_init
# region build_question_payload [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Return the one active highest-priority clarification question payload.
# @PRE: Session contains unresolved clarification state or a resumable clarification session.
# @POST: Returns exactly one active/open question payload or None when no unresolved question remains.
@@ -173,7 +171,6 @@ class ClarificationEngine:
# endregion build_question_payload
# region record_answer [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Persist one clarification answer before any pointer/readiness mutation.
# @PRE: Target question belongs to the session's active clarification session and is still open.
# @POST: Answer row is persisted before current-question pointer advances.
@@ -260,7 +257,6 @@ class ClarificationEngine:
# endregion record_answer
# region summarize_progress [TYPE Function]
# @COMPLEXITY: 1
# @PURPOSE: Produce a compact progress summary for pause/resume and completion UX.
def summarize_progress(self, clarification_session: ClarificationSession) -> str:
resolved = count_resolved_questions(clarification_session)
@@ -270,7 +266,6 @@ class ClarificationEngine:
# endregion summarize_progress
# region _get_latest_clarification_session [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Select the latest clarification session for the current dataset review aggregate.
def _get_latest_clarification_session(self, session: DatasetReviewSession) -> Optional[ClarificationSession]:
if not session.clarification_sessions:
@@ -281,7 +276,6 @@ class ClarificationEngine:
# endregion _get_latest_clarification_session
# region _find_question [TYPE Function]
# @COMPLEXITY: 1
# @PURPOSE: Resolve a clarification question from the active clarification aggregate.
def _find_question(self, clarification_session: ClarificationSession, question_id: str) -> Optional[ClarificationQuestion]:
for q in clarification_session.questions:

3
backend/src/services/dataset_review/event_logger.py
View File

@@ -45,14 +45,12 @@ class SessionEventPayload:
# @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent]
class SessionEventLogger:
# region SessionEventLogger_init [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Bind a live SQLAlchemy session to the session-event logger.
def __init__(self, db: Session) -> None:
self.db = db
# endregion SessionEventLogger_init
# region log_event [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Persist one explicit session event row for an owned dataset-review mutation.
# @RELATION: [DEPENDS_ON] ->[SessionEvent]
# @PRE: session_id, actor_user_id, event_type, and event_summary are non-empty.
@@ -125,7 +123,6 @@ class SessionEventLogger:
# endregion log_event
# region log_for_session [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Convenience wrapper for logging an event directly from a session aggregate root.
# @RELATION: [CALLS] ->[SessionEventLogger.log_event]
def log_for_session(

6
backend/src/services/dataset_review/orchestrator.py
View File

@@ -105,7 +105,6 @@ logger = cast(Any, logger)
# @INVARIANT: session ownership is preserved on every mutation and recovery remains explicit when partial.
class DatasetReviewOrchestrator:
# region DatasetReviewOrchestrator_init [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Bind repository, config, and task dependencies required by the orchestration boundary.
# @PRE: repository/config_manager are valid collaborators for the current request scope.
# @POST: Instance holds collaborator references used by start/preview/launch orchestration methods.
@@ -124,7 +123,6 @@ class DatasetReviewOrchestrator:
# endregion DatasetReviewOrchestrator_init
# region start_session [TYPE Function]
# @COMPLEXITY: 5
# @PURPOSE: Initialize a new session from a Superset link or dataset selection and trigger context recovery.
# @RELATION: CALLS -> [SupersetContextExtractor.parse_superset_link]
# @RELATION: CALLS -> [TaskManager.create_task]
@@ -279,7 +277,6 @@ class DatasetReviewOrchestrator:
# endregion start_session
# region prepare_launch_preview [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Assemble effective execution inputs and trigger Superset-side preview compilation.
# @RELATION: CALLS -> [SupersetCompilationAdapter.compile_preview]
# @PRE: all required variables have candidate values or explicitly accepted defaults.
@@ -359,7 +356,6 @@ class DatasetReviewOrchestrator:
# endregion prepare_launch_preview
# region launch_dataset [TYPE Function]
# @COMPLEXITY: 5
# @PURPOSE: Start the approved dataset execution through SQL Lab and persist run context for audit/replay.
# @RELATION: CALLS -> [SupersetCompilationAdapter.create_sql_lab_session]
# @PRE: session is run-ready and compiled preview is current.
@@ -459,7 +455,6 @@ class DatasetReviewOrchestrator:
# endregion launch_dataset
# region _build_recovery_bootstrap [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Recover and materialize initial imported filters, template variables, and draft execution mappings after session creation.
# @PRE: session belongs to the just-created review aggregate and parsed_context was produced for the same environment scope.
# @POST: Returns bootstrap imported filters, template variables, execution mappings, and updated findings without persisting them directly.
@@ -559,7 +554,6 @@ class DatasetReviewOrchestrator:
# endregion _build_recovery_bootstrap
# region _enqueue_recovery_task [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Link session start to observable async recovery when task infrastructure is available.
# @PRE: session is already persisted.
# @POST: returns task identifier when a task could be enqueued, otherwise None.

2
backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py
View File

@@ -28,14 +28,12 @@ from src.services.dataset_review.repositories.session_repository import (
# region SessionRepositoryTests [TYPE Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @COMPLEXITY: 2
# @PURPOSE: Unit tests for DatasetReviewSessionRepository.
@pytest.fixture
def db_session():
# region db_session [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Build isolated in-memory SQLAlchemy session seeded with baseline user/environment rows.
# @RELATION: BINDS_TO -> [SessionRepositoryTests]
engine = create_engine("sqlite:///:memory:")

12
backend/src/services/dataset_review/repositories/session_repository.py
View File

@@ -64,7 +64,6 @@ class DatasetReviewSessionVersionConflictError(ValueError):
# @SIDE_EFFECT: mutates and queries the persistence layer through the injected database session.
class DatasetReviewSessionRepository:
# region init_repo [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Bind one live SQLAlchemy session to the repository instance.
# @PRE: db_session is not None
# @POST: Repository instance initialized with valid session
@@ -75,7 +74,6 @@ class DatasetReviewSessionRepository:
# endregion init_repo
# region get_owned_session [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Resolve one owner-scoped dataset review session for mutation paths.
# @PRE: session_id and user_id are non-empty identifiers from the authenticated ownership scope.
# @POST: returns the owned session or raises a deterministic access error.
@@ -96,7 +94,6 @@ class DatasetReviewSessionRepository:
# endregion get_owned_session
# region create_sess [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Persist an initial dataset review session shell.
# @POST: session is committed, refreshed, and returned with persisted identifiers.
def create_session(self, session: DatasetReviewSession) -> DatasetReviewSession:
@@ -111,7 +108,6 @@ class DatasetReviewSessionRepository:
# endregion create_sess
# region require_session_version [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Enforce optimistic-lock version matching before a session mutation is persisted.
# @POST: returns the same session when versions match; otherwise raises deterministic conflict error.
def require_session_version(self, session: DatasetReviewSession, expected_version: int) -> DatasetReviewSession:
@@ -127,7 +123,6 @@ class DatasetReviewSessionRepository:
# endregion require_session_version
# region bump_session_version [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Increment optimistic-lock version after a successful session mutation is assembled.
# @POST: session version increments monotonically.
def bump_session_version(self, session: DatasetReviewSession) -> int:
@@ -141,7 +136,6 @@ class DatasetReviewSessionRepository:
# endregion bump_session_version
# region commit_session_mutation [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Commit one prepared session mutation and translate stale writes into deterministic conflicts.
# @POST: session mutation is committed with one version increment or a deterministic conflict error is raised.
def commit_session_mutation(
@@ -168,7 +162,6 @@ class DatasetReviewSessionRepository:
# endregion commit_session_mutation
# region load_detail [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Return the full session aggregate for API and frontend resume flows.
# @POST: Returns SessionDetail with all fields populated or None.
def load_session_detail(self, session_id: str, user_id: str) -> Optional[DatasetReviewSession]:
@@ -202,7 +195,6 @@ class DatasetReviewSessionRepository:
# endregion load_detail
# region save_profile_and_findings [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Persist profile state and replace validation findings for an owned session.
# @POST: stored profile matches the current session and findings are replaced.
def save_profile_and_findings(
@@ -217,7 +209,6 @@ class DatasetReviewSessionRepository:
# endregion save_profile_and_findings
# region save_recovery_state [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Persist imported filters, template variables, and initial execution mappings.
def save_recovery_state(
self, session_id: str, user_id: str, imported_filters: List[ImportedFilter],
@@ -234,7 +225,6 @@ class DatasetReviewSessionRepository:
# endregion save_recovery_state
# region save_preview [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Persist a preview snapshot and mark prior session previews stale.
def save_preview(
self, session_id: str, user_id: str, preview: CompiledPreview, expected_version: Optional[int] = None,
@@ -248,7 +238,6 @@ class DatasetReviewSessionRepository:
# endregion save_preview
# region save_run_context [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Persist an immutable launch audit snapshot for an owned session.
def save_run_context(
self, session_id: str, user_id: str, run_context: DatasetRunContext, expected_version: Optional[int] = None,
@@ -262,7 +251,6 @@ class DatasetReviewSessionRepository:
# endregion save_run_context
# region list_user_sess [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: List review sessions owned by a specific user ordered by most recent update.
def list_sessions_for_user(self, user_id: str) -> List[DatasetReviewSession]:
with belief_scope("DatasetReviewSessionRepository.list_sessions_for_user"):

12
backend/src/services/dataset_review/semantic_resolver.py
View File

@@ -47,14 +47,12 @@ class DictionaryResolutionResult:
# @SIDE_EFFECT: emits semantic trace logs for ranking and fallback decisions.
class SemanticSourceResolver:
# region resolve_from_file [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize uploaded semantic file records into field-level candidates.
def resolve_from_file(self, source_payload: Mapping[str, Any], fields: Iterable[Mapping[str, Any]]) -> DictionaryResolutionResult:
return DictionaryResolutionResult(source_ref=str(source_payload.get("source_ref") or "uploaded_file"))
# endregion resolve_from_file
# region resolve_from_dictionary [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Resolve candidates from connected tabular dictionary sources.
# @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]
# @RELATION: [DEPENDS_ON] ->[SemanticCandidate]
@@ -212,7 +210,6 @@ class SemanticSourceResolver:
# endregion resolve_from_dictionary
# region resolve_from_reference_dataset [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Reuse semantic metadata from trusted Superset datasets.
def resolve_from_reference_dataset(
self,
@@ -223,7 +220,6 @@ class SemanticSourceResolver:
# endregion resolve_from_reference_dataset
# region rank_candidates [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Apply confidence ordering and determine best candidate per field.
# @RELATION: [DEPENDS_ON] ->[SemanticCandidate]
def rank_candidates(self, candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
@@ -241,14 +237,12 @@ class SemanticSourceResolver:
# endregion rank_candidates
# region detect_conflicts [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Mark competing candidate sets that require explicit user review.
def detect_conflicts(self, candidates: List[Dict[str, Any]]) -> bool:
return len(candidates) > 1
# endregion detect_conflicts
# region apply_field_decision [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Accept, reject, or manually override a field-level semantic value.
def apply_field_decision(self, field_state: Mapping[str, Any], decision: Mapping[str, Any]) -> Dict[str, Any]:
merged = dict(field_state)
@@ -257,7 +251,6 @@ class SemanticSourceResolver:
# endregion apply_field_decision
# region propagate_source_version_update [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Propagate a semantic source version change to unlocked field entries without silently overwriting manual or locked values.
# @RELATION: [DEPENDS_ON] ->[SemanticSource]
# @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]
@@ -314,7 +307,6 @@ class SemanticSourceResolver:
# endregion propagate_source_version_update
# region _normalize_dictionary_row [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize one dictionary row into a consistent lookup structure.
def _normalize_dictionary_row(self, row: Mapping[str, Any]) -> Dict[str, Any]:
field_name = (
@@ -334,7 +326,6 @@ class SemanticSourceResolver:
# endregion _normalize_dictionary_row
# region _find_fuzzy_matches [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Produce confidence-scored fuzzy matches while keeping them reviewable.
def _find_fuzzy_matches(self, field_name: str, rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
normalized_target = self._normalize_key(field_name)
@@ -352,7 +343,6 @@ class SemanticSourceResolver:
# endregion _find_fuzzy_matches
# region _build_candidate_payload [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Project normalized dictionary rows into semantic candidate payloads.
def _build_candidate_payload(
self,
@@ -373,7 +363,6 @@ class SemanticSourceResolver:
# endregion _build_candidate_payload
# region _match_priority [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Encode trusted-confidence ordering so exact dictionary reuse beats fuzzy invention.
def _match_priority(self, match_type: Optional[str]) -> int:
priority = {
@@ -386,7 +375,6 @@ class SemanticSourceResolver:
# endregion _match_priority
# region _normalize_key [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize field identifiers for stable exact/fuzzy comparisons.
def _normalize_key(self, value: str) -> str:
return "".join(ch for ch in str(value or "").strip().lower() if ch.isalnum() or ch == "_")

1
backend/src/services/git/_status.py
View File

@@ -13,7 +13,6 @@ from src.core.logger import logger, belief_scope
# @BRIEF Mixin providing repository status, diff, and commit history for GitService.
class GitServiceStatusMixin:
# region _parse_status_porcelain [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Parse git status --porcelain output into staged, modified, and untracked file lists.
# @PRE: `repo` is an open GitPython Repo instance.
# @POST: Returns (staged, modified, untracked) tuple of file path lists.

1
backend/src/services/git_service.py
View File

@@ -1,5 +1,4 @@
# #region git_service [C:1] [TYPE Module:Tombstone] [SEMANTICS git, service, shim, re-export, decomissioned]
# @COMPLEXITY: 1
# @BRIEF Re-export shim — GitService has been decomposed into services/git/ package.
# All consumers continue to import from this same path without changes.
# @RELATION REDIRECTS_TO -> [GitServiceModule]

5
backend/src/services/health_service.py
View File

@@ -46,7 +46,6 @@ class HealthService:
"""
# region HealthService_init [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Initialize health service with DB session and optional config access for dashboard metadata resolution.
# @PRE: db is a valid SQLAlchemy session.
# @POST: Service is ready to aggregate summaries and delete health reports.
@@ -61,7 +60,6 @@ class HealthService:
# endregion HealthService_init
# region _prime_dashboard_meta_cache [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Warm dashboard slug/title cache with one Superset list fetch per environment.
# @PRE: records may contain mixed numeric and slug dashboard identifiers.
# @POST: Numeric dashboard ids for known environments are cached when discoverable.
@@ -153,7 +151,6 @@ class HealthService:
# endregion _prime_dashboard_meta_cache
# region _resolve_dashboard_meta [TYPE Function]
# @COMPLEXITY: 1
# @PURPOSE: Resolve slug/title for a dashboard referenced by persisted validation record.
# @PRE: dashboard_id may be numeric or slug-like; environment_id may be empty.
# @POST: Returns dict with `slug` and `title` keys, using cache when possible.
@@ -184,7 +181,6 @@ class HealthService:
# endregion _resolve_dashboard_meta
# region get_health_summary [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Aggregate latest validation status per dashboard and enrich rows with dashboard slug/title.
# @PRE: environment_id may be omitted to aggregate across all environments.
# @POST: Returns HealthSummaryResponse with counts and latest record row per dashboard.
@@ -288,7 +284,6 @@ class HealthService:
# endregion get_health_summary
# region delete_validation_report [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Delete one persisted health report and optionally clean linked task/log artifacts.
# @PRE: record_id is a validation record identifier.
# @POST: Returns True only when a matching record was deleted.

6
backend/src/services/llm_provider.py
View File

@@ -123,7 +123,6 @@ class LLMProviderService:
# endregion LLMProviderService_init
# region get_all_providers [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Returns all configured LLM providers.
# @PRE: Database connection must be active.
# @POST: Returns list of all LLMProvider records.
@@ -135,7 +134,6 @@ class LLMProviderService:
# endregion get_all_providers
# region get_provider [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Returns a single LLM provider by ID.
# @PRE: provider_id must be a valid string.
# @POST: Returns LLMProvider or None if not found.
@@ -149,7 +147,6 @@ class LLMProviderService:
# endregion get_provider
# region create_provider [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Creates a new LLM provider with encrypted API key.
# @PRE: config must contain valid provider configuration.
# @POST: New provider created and persisted to database.
@@ -175,7 +172,6 @@ class LLMProviderService:
# endregion create_provider
# region update_provider [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Updates an existing LLM provider.
# @PRE: provider_id must exist, config must be valid.
# @POST: Provider updated and persisted to database.
@@ -210,7 +206,6 @@ class LLMProviderService:
# endregion update_provider
# region delete_provider [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Deletes an LLM provider.
# @PRE: provider_id must exist.
# @POST: Provider removed from database.
@@ -227,7 +222,6 @@ class LLMProviderService:
# endregion delete_provider
# region get_decrypted_api_key [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Returns the decrypted API key for a provider.
# @PRE: provider_id must exist with valid encrypted key.
# @POST: Returns decrypted API key or None on failure.

3
backend/src/services/mapping_service.py
View File

@@ -28,7 +28,6 @@ from ..core.utils.matching import suggest_mappings
class MappingService:
# region init [TYPE Function]
# @PURPOSE: Initializes the mapping service with a config manager.
# @COMPLEXITY: 3
# @PRE: config_manager is provided.
# @PARAM: config_manager (ConfigManager) - The configuration manager.
# @POST: Service is initialized.
@@ -41,7 +40,6 @@ class MappingService:
# region _get_client [TYPE Function]
# @PURPOSE: Helper to get an initialized SupersetClient for an environment.
# @COMPLEXITY: 3
# @PARAM: env_id (str) - The ID of the environment.
# @PRE: environment must exist in config.
# @POST: Returns an initialized SupersetClient.
@@ -60,7 +58,6 @@ class MappingService:
# region get_suggestions [TYPE Function]
# @PURPOSE: Fetches databases from both environments and returns fuzzy matching suggestions.
# @COMPLEXITY: 3
# @PARAM: source_env_id (str) - Source environment ID.
# @PARAM: target_env_id (str) - Target environment ID.
# @PRE: Both environments must be accessible.

1
backend/src/services/notifications/__tests__/test_notification_service.py
View File

@@ -1,5 +1,4 @@
# region test_notification_service [TYPE Module]
# @COMPLEXITY: 2
# @PURPOSE: Unit tests for NotificationService routing and dispatch logic.
# @RELATION: TESTS ->[NotificationService:Class]

7
backend/src/services/notifications/service.py
View File

@@ -44,7 +44,6 @@ from .providers import (
# @DATA_CONTRACT: Input[ValidationRecord, Optional[ValidationPolicy], Optional[BackgroundTasks]] -> Output[None]
class NotificationService:
# region NotificationService_init [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Bind DB and configuration collaborators used for provider initialization and routing.
# @RELATION: [BINDS_TO] ->[NotificationService]
# @RELATION: [DEPENDS_ON] ->[ValidationPolicy]
@@ -57,7 +56,6 @@ class NotificationService:
# endregion NotificationService_init
# region _initialize_providers [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Materialize configured notification channel adapters once per service lifetime.
# @RELATION: [DEPENDS_ON] ->[SMTPProvider]
# @RELATION: [DEPENDS_ON] ->[TelegramProvider]
@@ -83,7 +81,6 @@ class NotificationService:
# endregion _initialize_providers
# region dispatch_report [TYPE Function]
# @COMPLEXITY: 4
# @PURPOSE: Route one validation record to resolved owners and configured custom channels.
# @RELATION: [CALLS] ->[_initialize_providers]
# @RELATION: [CALLS] ->[_should_notify]
@@ -143,7 +140,6 @@ class NotificationService:
# endregion dispatch_report
# region _should_notify [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Evaluate record status against effective alert policy.
# @RELATION: [DEPENDS_ON] ->[ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[ValidationPolicy]
@@ -161,7 +157,6 @@ class NotificationService:
# endregion _should_notify
# region _resolve_targets [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Resolve owner and policy-defined delivery targets for one validation record.
# @RELATION: [CALLS] ->[_find_dashboard_owners]
# @RELATION: [DEPENDS_ON] ->[ValidationRecord]
@@ -198,7 +193,6 @@ class NotificationService:
# endregion _resolve_targets
# region _find_dashboard_owners [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Load candidate dashboard owners from persisted profile preferences.
# @RELATION: [DEPENDS_ON] ->[ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[UserDashboardPreference]
@@ -220,7 +214,6 @@ class NotificationService:
# endregion _find_dashboard_owners
# region _build_body [TYPE Function]
# @COMPLEXITY: 2
# @PURPOSE: Format one validation record into provider-ready body text.
# @RELATION: [DEPENDS_ON] ->[ValidationRecord]
def _build_body(self, record: ValidationRecord) -> str:

1
backend/src/services/reports/__tests__/test_report_normalizer.py
View File

@@ -1,5 +1,4 @@
# region test_report_normalizer [TYPE Module]
# @COMPLEXITY: 2
# @SEMANTICS: tests, reports, normalizer, fallback
# @PURPOSE: Validate unknown task type fallback and partial payload normalization behavior.
# @RELATION: TESTS ->[normalize_report:Function]

1
backend/src/services/reports/__tests__/test_report_service.py
View File

@@ -1,5 +1,4 @@
# region test_report_service [TYPE Module]
# @COMPLEXITY: 2
# @PURPOSE: Unit tests for ReportsService list/detail operations
# @RELATION: TESTS ->[ReportsService:Class]
# @LAYER: Domain

1
backend/src/services/reports/report_service.py
View File

@@ -57,7 +57,6 @@ from .normalizer import normalize_task_report
# @TEST_INVARIANT: consistent_pagination -> verifies: [valid_service]
class ReportsService:
# region init [TYPE Function]
# @COMPLEXITY: 5
# @PURPOSE: Initialize service with TaskManager dependency.
# @PRE: task_manager is a live TaskManager instance.
# @POST: self.task_manager is assigned and ready for read operations.

13
backend/src/services/resource_service.py
View File

@@ -21,7 +21,6 @@ from ..core.logger import logger, belief_scope
class ResourceService:
# region ResourceService_init [TYPE Function]
# @COMPLEXITY: 1
# @PURPOSE: Initialize the resource service with dependencies
# @PRE: None
# @POST: ResourceService is ready to fetch resources
@@ -32,7 +31,6 @@ class ResourceService:
# endregion ResourceService_init
# region get_dashboards_with_status [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetch dashboards from environment with Git status and last task status
# @PRE: env is a valid Environment object
# @POST: Returns list of dashboards with enhanced metadata
@@ -82,7 +80,6 @@ class ResourceService:
# endregion get_dashboards_with_status
# region get_dashboards_page_with_status [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetch one dashboard page from environment and enrich only that page with status metadata.
# @PRE: env is valid; page >= 1; page_size > 0.
# @POST: Returns page items plus total counters without scanning all pages locally.
@@ -149,7 +146,6 @@ class ResourceService:
# endregion get_dashboards_page_with_status
# region _get_last_llm_task_for_dashboard [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Get most recent LLM validation task for a dashboard in an environment
# @PRE: dashboard_id is a valid integer identifier
# @POST: Returns the newest llm_dashboard_validation task summary or None
@@ -233,7 +229,6 @@ class ResourceService:
# endregion _get_last_llm_task_for_dashboard
# region _normalize_task_status [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Normalize task status to stable uppercase values for UI/API projections
# @PRE: raw_status can be enum or string
# @POST: Returns uppercase status without enum class prefix
@@ -251,7 +246,6 @@ class ResourceService:
# endregion _normalize_task_status
# region _normalize_validation_status [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Normalize LLM validation status to PASS/FAIL/WARN/UNKNOWN
# @PRE: raw_status can be any scalar type
# @POST: Returns normalized validation status token or None
@@ -268,7 +262,6 @@ class ResourceService:
# endregion _normalize_validation_status
# region _normalize_datetime_for_compare [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Normalize datetime values to UTC-aware values for safe comparisons.
# @PRE: value may be datetime or any scalar.
# @POST: Returns UTC-aware datetime; non-datetime values map to minimal UTC datetime.
@@ -285,7 +278,6 @@ class ResourceService:
# endregion _normalize_datetime_for_compare
# region get_datasets_with_status [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetch datasets from environment with mapping progress and last task status
# @PRE: env is a valid Environment object
# @POST: Returns list of datasets with enhanced metadata
@@ -324,7 +316,6 @@ class ResourceService:
# endregion get_datasets_with_status
# region get_activity_summary [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Get summary of active and recent tasks for the activity indicator
# @PRE: tasks is a list of Task objects
# @POST: Returns summary with active_count and recent_tasks
@@ -366,7 +357,6 @@ class ResourceService:
# endregion get_activity_summary
# region _get_git_status_for_dashboard [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Get Git sync status for a dashboard
# @PRE: dashboard_id is a valid integer
# @POST: Returns git status or None if no repo exists
@@ -426,7 +416,6 @@ class ResourceService:
# endregion _get_git_status_for_dashboard
# region _get_last_task_for_resource [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Get the most recent task for a specific resource
# @PRE: resource_id is a valid string
# @POST: Returns task summary or None if no tasks found
@@ -465,7 +454,6 @@ class ResourceService:
# endregion _get_last_task_for_resource
# region _extract_resource_name_from_task [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Extract resource name from task params
# @PRE: task is a valid Task object
# @POST: Returns resource name or task ID
@@ -478,7 +466,6 @@ class ResourceService:
# endregion _extract_resource_name_from_task
# region _extract_resource_type_from_task [TYPE Function]
# @COMPLEXITY: 3
# @PURPOSE: Extract resource type from task params
# @PRE: task is a valid Task object
# @POST: Returns resource type or 'unknown'