From a3c7c402b74300f00dd531febc79f4ac2de1dadc Mon Sep 17 00:00:00 2001 From: busya Date: Wed, 13 May 2026 20:06:15 +0300 Subject: [PATCH] fix(llm): add fetch-models endpoint, fix SQL Lab INSERT (client_id truncation, sync mode, target_column, timestamp normalization) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add POST /api/llm/providers/fetch-models route with LLMClient.fetch_models() - Add target_column to TranslationJob model/schema/service/orchestrator - Fix SQL Lab execute: truncate client_id to 11 chars (varchar(11)) - Switch SQL Lab to sync mode (runAsync: false) — no Celery workers - Fix polling: unwrap nested result from Superset query API - Fix ClickHouse timestamp: normalize float timestamps to YYYY-MM-DD --- backend/src/__init__.py | 2 +- backend/src/api/routes/llm.py | 63 +++++ .../src/api/routes/translate/_run_routes.py | 105 +++++++- backend/src/core/database.py | 18 ++ .../src/core/superset_client/_databases.py | 2 +- backend/src/core/utils/network.py | 44 +++- backend/src/models/translate.py | 6 +- backend/src/plugins/git/llm_extension.py | 1 - .../__tests__/test_client_headers.py | 1 - .../__tests__/test_screenshot_service.py | 1 - .../llm_analysis/__tests__/test_service.py | 1 - backend/src/plugins/llm_analysis/plugin.py | 1 - backend/src/plugins/llm_analysis/scheduler.py | 1 - backend/src/plugins/llm_analysis/service.py | 24 +- .../test_clickhouse_insert_integration.py | 3 +- .../translate/__tests__/test_dictionary.py | 2 - .../translate/__tests__/test_orchestrator.py | 208 ++++++++++++++- .../translate/__tests__/test_preview.py | 1 - .../translate/__tests__/test_sql_generator.py | 1 - backend/src/plugins/translate/dictionary.py | 14 - backend/src/plugins/translate/events.py | 1 - backend/src/plugins/translate/executor.py | 134 +++++++++- backend/src/plugins/translate/orchestrator.py | 78 +++++- backend/src/plugins/translate/preview.py | 17 +- backend/src/plugins/translate/service.py | 5 + .../src/plugins/translate/sql_generator.py | 15 +- .../plugins/translate/superset_executor.py | 146 +++++++++-- backend/src/schemas/translate.py | 8 +- .../__tests__/test_encryption_manager.py | 1 - .../services/__tests__/test_health_service.py | 1 - .../__tests__/test_llm_plugin_persistence.py | 10 - .../__tests__/test_llm_prompt_templates.py | 7 - .../services/__tests__/test_llm_provider.py | 3 - .../__tests__/test_rbac_permission_catalog.py | 1 - .../__tests__/test_resource_service.py | 1 - backend/src/services/auth_service.py | 4 - .../__tests__/test_audit_service.py | 1 - .../__tests__/test_compliance_orchestrator.py | 1 - .../__tests__/test_manifest_builder.py | 1 - .../__tests__/test_preparation_service.py | 1 - .../__tests__/test_report_builder.py | 1 - .../__tests__/test_source_isolation.py | 1 - .../clean_release/__tests__/test_stages.py | 1 - .../repositories/manifest_repository.py | 5 - .../dataset_review/clarification_engine.py | 6 - .../services/dataset_review/event_logger.py | 3 - .../services/dataset_review/orchestrator.py | 6 - .../__tests__/test_session_repository.py | 2 - .../repositories/session_repository.py | 12 - .../dataset_review/semantic_resolver.py | 12 - backend/src/services/git/_status.py | 1 - backend/src/services/git_service.py | 1 - backend/src/services/health_service.py | 5 - backend/src/services/llm_provider.py | 6 - backend/src/services/mapping_service.py | 3 - .../__tests__/test_notification_service.py | 1 - backend/src/services/notifications/service.py | 7 - .../__tests__/test_report_normalizer.py | 1 - .../reports/__tests__/test_report_service.py | 1 - .../src/services/reports/report_service.py | 1 - backend/src/services/resource_service.py | 13 - .../core/migration/test_archive_parser.py | 1 - .../migration/test_dry_run_orchestrator.py | 1 - .../tests/core/test_git_service_gitea_pr.py | 1 - backend/tests/core/test_mapping_service.py | 1 - backend/tests/core/test_migration_engine.py | 2 - .../tests/scripts/test_clean_release_cli.py | 1 - .../tests/scripts/test_clean_release_tui.py | 1 - .../scripts/test_clean_release_tui_v2.py | 1 - .../clean_release/test_approval_service.py | 1 - .../test_candidate_manifest_services.py | 1 - .../test_compliance_execution_service.py | 1 - .../test_compliance_task_integration.py | 2 - .../clean_release/test_demo_mode_isolation.py | 1 - .../test_policy_resolution_service.py | 2 - .../clean_release/test_publication_service.py | 1 - .../test_report_audit_immutability.py | 1 - .../dataset_review/test_superset_matrix.py | 5 - backend/tests/test_auth.py | 1 - backend/tests/test_dashboards_api.py | 3 +- backend/tests/test_log_persistence.py | 2 - backend/tests/test_logger.py | 1 - backend/tests/test_resource_hubs.py | 1 - backend/tests/test_smoke_app.py | 1 - backend/tests/test_task_manager.py | 1 - backend/tests/test_task_persistence.py | 3 - backend/tests/test_translate_corrections.py | 1 - backend/tests/test_translate_history.py | 1 - backend/tests/test_translate_jobs.py | 1 - backend/tests/test_translate_scheduler.py | 1 - frontend/src/components/DashboardGrid.svelte | 25 +- frontend/src/components/DynamicForm.svelte | 8 +- frontend/src/components/EnvSelector.svelte | 4 +- frontend/src/components/Footer.svelte | 4 +- frontend/src/components/MappingTable.svelte | 8 +- .../src/components/MissingMappingModal.svelte | 8 +- frontend/src/components/Navbar.svelte | 1 - frontend/src/components/PasswordPrompt.svelte | 8 +- .../components/RepositoryDashboardGrid.svelte | 89 ++++--- .../StartupEnvironmentWizard.svelte | 1 - frontend/src/components/TaskHistory.svelte | 24 +- frontend/src/components/TaskList.svelte | 12 +- frontend/src/components/TaskLogViewer.svelte | 17 +- frontend/src/components/TaskRunner.svelte | 37 ++- frontend/src/components/Toast.svelte | 1 - .../__tests__/task_log_viewer.test.js | 5 +- .../src/components/auth/ProtectedRoute.svelte | 5 +- .../components/backups/BackupManager.svelte | 12 +- .../src/components/git/BranchSelector.svelte | 20 +- .../src/components/git/CommitHistory.svelte | 8 +- .../src/components/git/CommitModal.svelte | 13 +- .../components/git/ConflictResolver.svelte | 8 +- .../src/components/git/DeploymentModal.svelte | 20 +- frontend/src/components/git/GitManager.svelte | 96 +++---- ...nager.unfinished_merge.integration.test.js | 5 +- frontend/src/components/llm/DocPreview.svelte | 1 - .../src/components/llm/ProviderConfig.svelte | 1 - .../components/llm/ValidationReport.svelte | 5 +- .../provider_config.integration.test.js | 10 +- .../src/components/storage/FileList.svelte | 17 +- .../src/components/storage/FileUpload.svelte | 9 +- .../src/components/tasks/LogEntryRow.svelte | 5 +- .../src/components/tasks/LogFilterBar.svelte | 1 - .../src/components/tasks/TaskLogPanel.svelte | 1 - .../components/tools/ConnectionForm.svelte | 8 +- .../components/tools/ConnectionList.svelte | 8 +- .../src/components/tools/DebugTool.svelte | 12 +- .../src/components/tools/MapperTool.svelte | 12 +- frontend/src/lib/Counter.svelte | 1 - frontend/src/lib/api.js | 49 ++-- .../src/lib/api/__tests__/reports_api.test.js | 17 +- frontend/src/lib/api/assistant.js | 33 ++- frontend/src/lib/api/datasetReview.js | 29 +-- frontend/src/lib/api/reports.js | 21 +- frontend/src/lib/api/translate.js | 143 ++++++----- .../lib/auth/__tests__/permissions.test.js | 5 +- frontend/src/lib/auth/permissions.js | 17 +- frontend/src/lib/auth/store.ts | 29 +-- .../assistant/AssistantChatPanel.svelte | 1 - .../AssistantClarificationCard.svelte | 1 - .../assistant_chat.integration.test.js | 14 +- ...ssistant_clarification.integration.test.js | 5 +- ...ssistant_first_message.integration.test.js | 10 +- .../dataset-review/CompiledSQLPreview.svelte | 1 - .../ExecutionMappingReview.svelte | 1 - .../LaunchConfirmationPanel.svelte | 1 - .../dataset-review/SemanticLayerReview.svelte | 1 - .../dataset-review/SourceIntakePanel.svelte | 1 - .../ValidationFindingsPanel.svelte | 1 - .../__tests__/source_intake_panel.ux.test.js | 5 +- .../us2_semantic_workspace.ux.test.js | 5 +- .../__tests__/us3_execution_batch.ux.test.js | 5 +- .../validation_findings_panel.ux.test.js | 5 +- .../lib/components/health/HealthMatrix.svelte | 1 - .../lib/components/health/PolicyForm.svelte | 1 - .../lib/components/layout/Breadcrumbs.svelte | 1 - .../src/lib/components/layout/Sidebar.svelte | 1 - .../lib/components/layout/TaskDrawer.svelte | 19 +- .../lib/components/layout/TopNavbar.svelte | 1 - .../__tests__/sidebarNavigation.test.js | 5 +- .../__tests__/test_breadcrumbs.svelte.js | 6 +- .../layout/__tests__/test_sidebar.svelte.js | 5 +- .../__tests__/test_taskDrawer.svelte.js | 5 +- .../layout/__tests__/test_topNavbar.svelte.js | 5 +- .../components/layout/sidebarNavigation.js | 13 +- .../lib/components/reports/ReportCard.svelte | 1 - .../reports/ReportDetailPanel.svelte | 1 - .../lib/components/reports/ReportsList.svelte | 1 - .../__tests__/fixtures/reports.fixtures.js | 5 +- .../reports/__tests__/report_card.ux.test.js | 5 +- .../report_detail.integration.test.js | 5 +- .../__tests__/report_detail.ux.test.js | 5 +- .../__tests__/report_type_profiles.test.js | 5 +- .../reports_filter_performance.test.js | 5 +- .../reports/__tests__/reports_list.ux.test.js | 5 +- .../reports_page.integration.test.js | 5 +- .../components/reports/reportTypeProfiles.js | 9 +- .../translate/BulkCorrectionSidebar.svelte | 1 - .../translate/ScheduleConfig.svelte | 1 - .../translate/TermCorrectionPopup.svelte | 1 - .../TranslationMetricsDashboard.svelte | 1 - .../translate/TranslationPreview.svelte | 1 - .../translate/TranslationRunProgress.svelte | 35 ++- .../translate/TranslationRunResult.svelte | 1 - .../test_translation_preview.svelte.js | 44 ++-- frontend/src/lib/i18n/index.ts | 17 +- frontend/src/lib/stores.js | 37 ++- .../stores/__tests__/assistantChat.test.js | 10 +- .../lib/stores/__tests__/mocks/env_public.js | 5 +- .../lib/stores/__tests__/mocks/environment.js | 5 +- .../lib/stores/__tests__/mocks/navigation.js | 5 +- .../src/lib/stores/__tests__/mocks/state.js | 5 +- .../src/lib/stores/__tests__/mocks/stores.js | 5 +- .../src/lib/stores/__tests__/setupTests.js | 5 +- .../src/lib/stores/__tests__/sidebar.test.js | 21 +- .../src/lib/stores/__tests__/test_activity.js | 5 +- .../__tests__/test_datasetReviewSession.js | 5 +- .../src/lib/stores/__tests__/test_sidebar.js | 5 +- .../lib/stores/__tests__/test_taskDrawer.js | 5 +- frontend/src/lib/stores/activity.js | 5 +- frontend/src/lib/stores/assistantChat.js | 37 ++- .../src/lib/stores/datasetReviewSession.js | 5 +- frontend/src/lib/stores/environmentContext.js | 5 +- frontend/src/lib/stores/health.js | 5 +- frontend/src/lib/stores/sidebar.js | 5 +- frontend/src/lib/stores/taskDrawer.js | 5 +- frontend/src/lib/toasts.js | 16 +- frontend/src/lib/ui/Button.svelte | 1 - frontend/src/lib/ui/Card.svelte | 1 - frontend/src/lib/ui/Icon.svelte | 1 - frontend/src/lib/ui/Input.svelte | 1 - frontend/src/lib/ui/LanguageSwitcher.svelte | 1 - frontend/src/lib/ui/PageHeader.svelte | 1 - frontend/src/lib/ui/Select.svelte | 1 - frontend/src/lib/ui/index.ts | 5 +- frontend/src/lib/utils.js | 5 +- frontend/src/lib/utils/debounce.js | 5 +- frontend/src/pages/Dashboard.svelte | 9 +- frontend/src/pages/Settings.svelte | 29 +-- frontend/src/routes/+layout.ts | 5 +- frontend/src/routes/+page.ts | 4 +- frontend/src/routes/admin/roles/+page.svelte | 21 +- .../src/routes/admin/settings/+page.svelte | 17 +- .../routes/admin/settings/llm/+page.svelte | 1 - frontend/src/routes/admin/users/+page.svelte | 21 +- frontend/src/routes/dashboards/+page.svelte | 101 ++++---- .../src/routes/dashboards/[id]/+page.svelte | 1 - .../components/DashboardGitManager.svelte | 5 +- .../DashboardLinkedResources.svelte | 5 +- .../components/DashboardTaskHistory.svelte | 5 +- ...board-profile-override.integration.test.js | 5 +- .../src/routes/dashboards/health/+page.svelte | 12 +- .../__tests__/health_page.integration.test.js | 5 +- frontend/src/routes/datasets/+page.svelte | 1 - .../src/routes/datasets/[id]/+page.svelte | 1 - .../routes/datasets/review/[id]/+page.svelte | 3 + .../dataset_review_workspace.ux.test.js | 3 +- .../__tests__/dataset_review_entry.test.js | 40 ++- .../__tests__/dataset_review_entry.ux.test.js | 39 ++- frontend/src/routes/migration/+page.svelte | 94 +++---- .../routes/migration/mappings/+page.svelte | 32 +-- .../__tests__/fixtures/profile.fixtures.js | 5 +- .../profile-preferences.integration.test.js | 227 ++--------------- ...profile-settings-state.integration.test.js | 241 +++--------------- .../routes/reports/llm/[taskId]/+page.svelte | 1 - .../llm/[taskId]/report_page.contract.test.js | 10 +- frontend/src/routes/settings/+page.svelte | 1 - frontend/src/routes/settings/+page.ts | 4 +- .../routes/settings/EnvironmentsTab.svelte | 1 - .../settings_page.integration.test.js | 38 +-- .../__tests__/settings_page.ux.test.js | 5 +- .../routes/settings/automation/+page.svelte | 1 - .../routes/settings/connections/+page.svelte | 5 +- frontend/src/routes/settings/git/+page.svelte | 37 ++- .../__tests__/git_settings_page.ux.test.js | 5 +- .../settings/notifications/+page.svelte | 1 - .../src/routes/storage/backups/+page.svelte | 1 - .../src/routes/storage/repos/+page.svelte | 13 +- .../src/routes/tools/backups/+page.svelte | 1 - frontend/src/routes/tools/debug/+page.svelte | 1 - frontend/src/routes/tools/mapper/+page.svelte | 1 - .../src/routes/tools/storage/+page.svelte | 25 +- frontend/src/routes/translate/+page.svelte | 1 - .../src/routes/translate/[id]/+page.svelte | 4 +- .../translate/dictionaries/+page.svelte | 1 - .../translate/dictionaries/[id]/+page.svelte | 1 - .../src/routes/translate/history/+page.svelte | 1 - .../src/services/__tests__/gitService.test.js | 5 +- frontend/src/services/adminService.js | 57 ++--- frontend/src/services/connectionService.js | 12 +- frontend/src/services/gitService.js | 71 +++--- frontend/src/services/storageService.js | 33 ++- frontend/src/services/taskService.js | 24 +- frontend/src/services/toolsService.js | 8 +- frontend/src/types/backup.ts | 4 +- frontend/src/types/dashboard.ts | 5 +- 276 files changed, 1923 insertions(+), 1822 deletions(-) diff --git a/backend/src/__init__.py b/backend/src/__init__.py index 2de99616..94e38492 100644 --- a/backend/src/__init__.py +++ b/backend/src/__init__.py @@ -1,3 +1,3 @@ -# #region SrcRoot [TYPE Module] +# #region SrcRoot [TYPE Module] [SEMANTICS root, package] # @BRIEF Canonical backend package root for application, scripts, and tests. # #endregion SrcRoot diff --git a/backend/src/api/routes/llm.py b/backend/src/api/routes/llm.py index 9e72170d..46ec517c 100644 --- a/backend/src/api/routes/llm.py +++ b/backend/src/api/routes/llm.py @@ -75,6 +75,69 @@ async def get_providers( # #endregion get_providers +# #region fetch_models [TYPE Function] +# @BRIEF Fetch available models from an LLM provider by base_url+provider_type, or by provider_id. +# @PRE: User is authenticated. Either provider_id or base_url+provider_type must be provided. +# @POST: Returns a list of available model IDs. +# @RELATION CALLS -> [LLMProviderService] +# @RELATION CALLS -> [LLMClient] +@router.post("/providers/fetch-models") +async def fetch_models( + payload: dict, + current_user: User = Depends(get_current_active_user), + db: Session = Depends(get_db), +): + from ...plugins.llm_analysis.service import LLMClient + from ...plugins.llm_analysis.models import LLMProviderType + + base_url = (payload.get("base_url") or "").strip() + provider_type_str = (payload.get("provider_type") or "").strip() + api_key = payload.get("api_key") or "" + provider_id = payload.get("provider_id") or "" + + # Resolve provider_id to stored credentials if no direct api_key given + if not api_key and provider_id: + service = LLMProviderService(db) + db_provider = service.get_provider(provider_id) + if not db_provider: + raise HTTPException(status_code=404, detail="Provider not found") + base_url = base_url or db_provider.base_url + provider_type_str = provider_type_str or db_provider.provider_type + stored_key = service.get_decrypted_api_key(provider_id) + if stored_key: + api_key = stored_key + + if not base_url: + raise HTTPException(status_code=400, detail="base_url is required") + if not provider_type_str: + raise HTTPException(status_code=400, detail="provider_type is required") + + try: + provider_type = LLMProviderType(provider_type_str) + except ValueError: + raise HTTPException(status_code=400, detail=f"Invalid provider_type: {provider_type_str}") + + client = LLMClient( + provider_type=provider_type, + api_key=api_key or "sk-placeholder", + base_url=base_url, + default_model="", + ) + + try: + models = await client.fetch_models() + return {"models": models} + except Exception as e: + logger.warning( + f"[llm_routes.fetch_models] Failed to fetch models: {e}", + extra={"src": "llm_routes.fetch_models"}, + ) + raise HTTPException(status_code=502, detail=str(e)) + + +# #endregion fetch_models + + # #region get_llm_status [TYPE Function] # @BRIEF Returns whether LLM runtime is configured for dashboard validation. # @PRE: User is authenticated. diff --git a/backend/src/api/routes/translate/_run_routes.py b/backend/src/api/routes/translate/_run_routes.py index 7fccecd6..93d97f93 100644 --- a/backend/src/api/routes/translate/_run_routes.py +++ b/backend/src/api/routes/translate/_run_routes.py @@ -2,11 +2,12 @@ # @BRIEF Translation Run execution, history, status, records and batches routes. # @LAYER: API +from datetime import datetime, timezone from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session -from ....core.database import get_db +from ....core.database import get_db, SessionLocal from ....core.logger import logger, belief_scope from ....schemas.auth import User from ....dependencies import get_current_user, has_permission, get_config_manager @@ -39,11 +40,107 @@ async def run_translation( try: orch = TranslationOrchestrator(db, config_manager, current_user.username) run = orch.start_run(job_id=job_id, is_scheduled=False) - # Execute asynchronously in background + # The request-scoped db session will be closed after this handler returns. + # The background thread must use its OWN session to avoid operating on a + # closed or expunged session. import threading + + def _background_execute(): + bg_db = SessionLocal() + try: + bg_orch = TranslationOrchestrator( + bg_db, config_manager, + current_user.username if current_user else None, + ) + # Re-fetch the run within the background session to get a fresh, + # attached object + from ....models.translate import TranslationRun as TRModel + bg_run = bg_db.query(TRModel).filter(TRModel.id == run.id).first() + if bg_run is None: + logger.explore( + "Background execute: run not found", + extra={"src": "translate_routes", "run_id": run.id}, + ) + return + + # execute_run internally calls self.db.commit() + bg_orch.execute_run(bg_run) + + # ----- VERIFICATION ----- + # After execute_run the run object from bg_db is detached after + # commit. Re-query to verify the status was persisted in the + # database and is a terminal state. + check_run = bg_db.query(TRModel).filter(TRModel.id == run.id).first() + if check_run and check_run.status in ("COMPLETED", "FAILED", "CANCELLED"): + logger.reason( + "Background execute verified", + extra={ + "src": "translate_routes", + "run_id": run.id, + "status": check_run.status, + }, + ) + else: + # execute_run appeared to succeed but the run is still in a + # non-terminal state — manually fail it so the frontend + # polling can detect the terminal state and stop spinning. + actual_status = check_run.status if check_run else "NOT_FOUND" + logger.explore( + "Background execute: run not in terminal state after commit", + extra={ + "src": "translate_routes", + "run_id": run.id, + "status": actual_status, + }, + ) + if check_run: + check_run.status = "FAILED" + check_run.error_message = ( + "Background execution did not reach terminal state " + f"(was: {actual_status})" + ) + bg_db.commit() + + except Exception as bg_err: + logger.explore( + "Background execute failed", + extra={ + "src": "translate_routes", + "run_id": run.id, + "error": str(bg_err), + }, + ) + # Mark the run as FAILED so the frontend polling can detect a + # terminal state and stop spinning. + try: + fb_db = SessionLocal() + try: + fb_run = fb_db.query(TRModel).filter(TRModel.id == run.id).first() + if fb_run: + fb_run.status = "FAILED" + fb_run.error_message = f"Background execute error: {bg_err}" + fb_run.completed_at = datetime.now(timezone.utc) + fb_db.commit() + logger.reason( + "Background execute: run marked FAILED", + extra={"src": "translate_routes", "run_id": run.id}, + ) + finally: + fb_db.close() + except Exception as fb_err: + logger.explore( + "Background execute: unable to mark run FAILED", + extra={ + "src": "translate_routes", + "run_id": run.id, + "error": str(fb_err), + }, + ) + finally: + bg_db.close() + threading.Thread( - target=orch.execute_run, - args=(run,), + target=_background_execute, daemon=True, ).start() return _run_to_response(run) diff --git a/backend/src/core/database.py b/backend/src/core/database.py index ca1752e4..4f4d7014 100644 --- a/backend/src/core/database.py +++ b/backend/src/core/database.py @@ -484,6 +484,24 @@ def _ensure_translation_jobs_columns(bind_engine): ) raise + if "target_database_id" not in existing_columns: + try: + with bind_engine.begin() as connection: + connection.execute( + text( + "ALTER TABLE translation_jobs " + "ADD COLUMN target_database_id VARCHAR" + ) + ) + logger.reflect( + "Added target_database_id column to translation_jobs", + ) + except Exception as migration_error: + logger.explore( + "Failed to add target_database_id to translation_jobs", + extra={"error": str(migration_error)}, + ) + def _ensure_dataset_review_session_columns(bind_engine): with belief_scope("_ensure_dataset_review_session_columns"): diff --git a/backend/src/core/superset_client/_databases.py b/backend/src/core/superset_client/_databases.py index 43e03af7..80465877 100644 --- a/backend/src/core/superset_client/_databases.py +++ b/backend/src/core/superset_client/_databases.py @@ -47,7 +47,7 @@ class SupersetDatabasesMixin: # @RELATION: CALLS -> [SupersetClientGetDatabases] def get_databases_summary(self) -> List[Dict]: with belief_scope("SupersetClient.get_databases_summary"): - query = {"columns": ["uuid", "database_name", "backend"]} + query = {"columns": ["id", "uuid", "database_name", "backend"]} _, databases = self.get_databases(query=query) # Map 'backend' to 'engine' for consistency with contracts for db in databases: diff --git a/backend/src/core/utils/network.py b/backend/src/core/utils/network.py index 9b1d9fc4..a4671a44 100644 --- a/backend/src/core/utils/network.py +++ b/backend/src/core/utils/network.py @@ -243,11 +243,45 @@ class APIClient: with belief_scope("authenticate"): app_logger.info("[authenticate][Enter] Authenticating to %s", self.base_url) cached_tokens = SupersetAuthCache.get(self._auth_cache_key) - if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"): - self._tokens = cached_tokens - self._authenticated = True - app_logger.info("[authenticate][CacheHit] Reusing cached Superset auth tokens for %s", self.base_url) - return self._tokens + if cached_tokens and cached_tokens.get("access_token"): + # Cache hit — we have a valid access_token, but we need to + # re-establish the session cookie in this new requests.Session. + # Superset CSRF protection pairs the X-CSRFToken header with a + # Flask session cookie set during login. A fresh requests.Session + # lacks that cookie, so even a valid cached CSRF token would fail + # with "CSRF session token is missing". We re-fetch the CSRF + # token to both establish the session cookie AND get a fresh + # CSRF token for the current session. + try: + csrf_url = f"{self.api_base_url}/security/csrf_token/" + csrf_response = self.session.get( + csrf_url, + headers={"Authorization": f"Bearer {cached_tokens['access_token']}"}, + timeout=self.request_settings["timeout"], + ) + csrf_response.raise_for_status() + csrf_token = csrf_response.json()["result"] + self._tokens = { + "access_token": cached_tokens["access_token"], + "csrf_token": csrf_token, + } + self._authenticated = True + # Update cache with fresh CSRF token for this session + SupersetAuthCache.set(self._auth_cache_key, self._tokens) + app_logger.info( + "[authenticate][CacheHit] Reusing cached Superset auth tokens for %s " + "(CSRF token refreshed for new session)", + self.base_url, + ) + return self._tokens + except Exception as refresh_err: + app_logger.warning( + "[authenticate][CacheRefreshFailed] CSRF token refresh failed, " + "falling back to full re-authentication: %s", + refresh_err, + ) + SupersetAuthCache.invalidate(self._auth_cache_key) + # Fall through to full authentication below try: login_url = f"{self.api_base_url}/security/login" # Log the payload keys and values (masking password) diff --git a/backend/src/models/translate.py b/backend/src/models/translate.py index 53af069b..25c20232 100644 --- a/backend/src/models/translate.py +++ b/backend/src/models/translate.py @@ -42,7 +42,8 @@ class TranslationJob(Base): # Column mapping source_key_cols = Column(JSON, nullable=True, comment="Source key column names for composite key") target_key_cols = Column(JSON, nullable=True, comment="Target key column names for composite key") - translation_column = Column(String, nullable=True, comment="The column whose values will be translated") + translation_column = Column(String, nullable=True, comment="Source column whose values will be translated") + target_column = Column(String, nullable=True, comment="Target column for translated output (defaults to translation_column)") context_columns = Column(JSON, nullable=True, comment="Context column names included in LLM prompt") # LLM & processing settings @@ -53,6 +54,7 @@ class TranslationJob(Base): # Environment association environment_id = Column(String, nullable=True, comment="Superset environment ID for datasource access") + target_database_id = Column(String, nullable=True, comment="Superset database ID for SQL Lab insert target") created_by = Column(String, nullable=True) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) @@ -119,6 +121,7 @@ class TranslationRecord(Base): source_object_type = Column(String, nullable=True) # query, dashboard, chart, dataset source_object_id = Column(String, nullable=True) source_object_name = Column(String, nullable=True) + source_data = Column(JSON, nullable=True, comment="Original source row key columns for upsert matching") status = Column(String, nullable=False, default="PENDING") # PENDING, SUCCESS, FAILED, SKIPPED error_message = Column(Text, nullable=True) token_count_input = Column(Integer, nullable=True) @@ -174,6 +177,7 @@ class TranslationPreviewRecord(Base): source_object_type = Column(String, nullable=True) source_object_id = Column(String, nullable=True) source_object_name = Column(String, nullable=True) + source_data = Column(JSON, nullable=True, comment="Original source row key columns for upsert matching") status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED feedback = Column(Text, nullable=True) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) diff --git a/backend/src/plugins/git/llm_extension.py b/backend/src/plugins/git/llm_extension.py index 5e538ec3..868a2758 100644 --- a/backend/src/plugins/git/llm_extension.py +++ b/backend/src/plugins/git/llm_extension.py @@ -1,5 +1,4 @@ # #region GitLLMExtensionModule [C:3] [TYPE Module] [SEMANTICS git, llm, commit, message, generation] -# @COMPLEXITY: 3 # @BRIEF LLM-based extensions for the Git plugin, specifically for commit message generation. # @LAYER: Domain # @RELATION DEPENDS_ON -> [LLMClient] diff --git a/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py b/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py index 5389ee7d..58d203be 100644 --- a/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py +++ b/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py @@ -1,6 +1,5 @@ # region TestClientHeaders [TYPE Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 3 # @SEMANTICS: tests, llm-client, openrouter, headers # @PURPOSE: Verify OpenRouter client initialization includes provider-specific headers. diff --git a/backend/src/plugins/llm_analysis/__tests__/test_screenshot_service.py b/backend/src/plugins/llm_analysis/__tests__/test_screenshot_service.py index d76f690c..44ef64e6 100644 --- a/backend/src/plugins/llm_analysis/__tests__/test_screenshot_service.py +++ b/backend/src/plugins/llm_analysis/__tests__/test_screenshot_service.py @@ -1,6 +1,5 @@ # region TestScreenshotService [TYPE Module] # @RELATION: VERIFIES ->[src.plugins.llm_analysis.service.ScreenshotService] -# @COMPLEXITY: 3 # @SEMANTICS: tests, screenshot-service, navigation, timeout-regression # @PURPOSE: Protect dashboard screenshot navigation from brittle networkidle waits. diff --git a/backend/src/plugins/llm_analysis/__tests__/test_service.py b/backend/src/plugins/llm_analysis/__tests__/test_service.py index 3283fa36..1efdbba3 100644 --- a/backend/src/plugins/llm_analysis/__tests__/test_service.py +++ b/backend/src/plugins/llm_analysis/__tests__/test_service.py @@ -1,6 +1,5 @@ # region TestService [TYPE Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 3 # @SEMANTICS: tests, llm-analysis, fallback, provider-error, unknown-status # @PURPOSE: Verify LLM analysis transport/provider failures do not masquerade as dashboard FAIL results. diff --git a/backend/src/plugins/llm_analysis/plugin.py b/backend/src/plugins/llm_analysis/plugin.py index 2bbad8c7..06e11df9 100644 --- a/backend/src/plugins/llm_analysis/plugin.py +++ b/backend/src/plugins/llm_analysis/plugin.py @@ -1,5 +1,4 @@ # #region LLMAnalysisPlugin [C:3] [TYPE Module] [SEMANTICS llm, analysis, dashboard, validation, documentation] -# @COMPLEXITY: 3 # @BRIEF Implements DashboardValidationPlugin and DocumentationPlugin. # @LAYER: Domain # @RELATION INHERITS -> [PluginBase] diff --git a/backend/src/plugins/llm_analysis/scheduler.py b/backend/src/plugins/llm_analysis/scheduler.py index 4a7fce8c..924ff949 100644 --- a/backend/src/plugins/llm_analysis/scheduler.py +++ b/backend/src/plugins/llm_analysis/scheduler.py @@ -1,5 +1,4 @@ # #region LLMAnalysisScheduler [C:3] [TYPE Module] [SEMANTICS scheduler, llm, validation, task, cron] -# @COMPLEXITY: 3 # @BRIEF Provides helper functions to schedule LLM-based validation tasks. # @LAYER: Domain # @RELATION DEPENDS_ON -> [SchedulerService] diff --git a/backend/src/plugins/llm_analysis/service.py b/backend/src/plugins/llm_analysis/service.py index 65f8f47b..ee2fe28e 100644 --- a/backend/src/plugins/llm_analysis/service.py +++ b/backend/src/plugins/llm_analysis/service.py @@ -1,5 +1,4 @@ # #region LLMAnalysisService [C:3] [TYPE Module] [SEMANTICS llm, screenshot, playwright, openai, tenacity] -# @COMPLEXITY: 3 # @BRIEF Services for LLM interaction and dashboard screenshots. # @LAYER: Domain # @RELATION DEPENDS_ON -> playwright @@ -876,6 +875,29 @@ class LLMClient: return await self.get_json_completion(messages) # endregion LLMClient.test_runtime_connection + # region LLMClient.fetch_models [TYPE Function] + # @PURPOSE: Fetch available models from the provider's API. + # @PRE: Client is initialized with provider credentials. + # @POST: Returns a list of model ID strings. + # @SIDE_EFFECT: Calls external LLM API /v1/models endpoint. + async def fetch_models(self) -> List[str]: + with belief_scope("LLMClient.fetch_models"): + try: + response = await self.client.models.list() + model_ids = [m.id for m in response.data] + model_ids.sort() + logger.reason( + f"[LLMClient.fetch_models] Fetched {len(model_ids)} models from {self.base_url}", + extra={"src": "LLMClient.fetch_models"}, + ) + return model_ids + except Exception as e: + logger.warning( + f"[LLMClient.fetch_models] Failed to fetch models: {e}", + ) + raise + # endregion LLMClient.fetch_models + # region LLMClient.analyze_dashboard [TYPE Function] # @PURPOSE: Sends dashboard data (screenshot + logs) to LLM for health analysis. # @PRE: screenshot_path exists, logs is a list of strings. diff --git a/backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py b/backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py index 85cf16bf..d174dc50 100644 --- a/backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py +++ b/backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py @@ -1,5 +1,4 @@ # region ClickHouseInsertIntegration [TYPE Module] -# @COMPLEXITY: 3 # @SEMANTICS: test, clickhouse, integration, insert, join # @PURPOSE: Integration tests for ClickHouse INSERT with timestamp normalization and JOIN verification. # @LAYER: Test @@ -369,6 +368,8 @@ class TestOrchestratorInsertFlow: ) as MockExecutor: mock_executor = MagicMock() MockExecutor.return_value = mock_executor + mock_executor.resolve_database_id.return_value = None + mock_executor.get_database_backend.return_value = None mock_executor.execute_and_poll.return_value = { "status": "success", "query_id": "q-123", diff --git a/backend/src/plugins/translate/__tests__/test_dictionary.py b/backend/src/plugins/translate/__tests__/test_dictionary.py index ffe1aed7..9ab0277f 100644 --- a/backend/src/plugins/translate/__tests__/test_dictionary.py +++ b/backend/src/plugins/translate/__tests__/test_dictionary.py @@ -1,5 +1,4 @@ # region DictionaryTests [TYPE Module] -# @COMPLEXITY: 3 # @SEMANTICS: tests, dictionary, crud, import, filter # @PURPOSE: Validate DictionaryManager CRUD, import, deletion guards, and batch filtering. # @RELATION: BINDS_TO -> [DictionaryManager:Class] @@ -38,7 +37,6 @@ from src.plugins.translate._utils import _normalize_term, _detect_delimiter # region _FakeJob [TYPE Class] -# @COMPLEXITY: 1 # @PURPOSE: Helper to create inline TranslationJob records. class _FakeJob: pass diff --git a/backend/src/plugins/translate/__tests__/test_orchestrator.py b/backend/src/plugins/translate/__tests__/test_orchestrator.py index 00d02169..e87ca96e 100644 --- a/backend/src/plugins/translate/__tests__/test_orchestrator.py +++ b/backend/src/plugins/translate/__tests__/test_orchestrator.py @@ -1,5 +1,4 @@ # region OrchestratorTests [TYPE Module] -# @COMPLEXITY: 3 # @SEMANTICS: test, translate, orchestrator, events # @PURPOSE: Tests for TranslationOrchestrator: run lifecycle, partial failure, batch retry, event invariants, NULL handling. # @LAYER: Test @@ -10,9 +9,10 @@ # @TEST_FIXTURE: mock_config_manager -> MagicMock ConfigManager # @TEST_EDGE: missing_preview -> raises ValueError # @TEST_EDGE: invalid_run_status -> raises ValueError +# @TEST_EDGE: executor_failure -> run is marked FAILED import pytest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from datetime import datetime, timezone from src.plugins.translate.orchestrator import TranslationOrchestrator @@ -234,6 +234,210 @@ class TestTranslationOrchestrator: with pytest.raises(ValueError, match="PENDING"): orch.execute_run(run) + # region test_execute_run_success [TYPE Function] + # @BRIEF Happy path: executor completes, SQL generated, Superset submits. + def test_execute_run_success(self, mock_job: MagicMock) -> None: + db = MagicMock() + config_manager = MagicMock() + mock_job.target_table = "target_tbl" + + # Mock job query — first call returns mock_job, final re-query returns completed_run + completed_run_response = MagicMock() + completed_run_response.id = "run-1" + completed_run_response.job_id = "job-123" + completed_run_response.status = "COMPLETED" + completed_run_response.total_records = 10 + completed_run_response.successful_records = 10 + completed_run_response.failed_records = 0 + completed_run_response.skipped_records = 0 + completed_run_response.completed_at = datetime.now(timezone.utc) + completed_run_response.error_message = None + completed_run_response.insert_status = "success" + completed_run_response.superset_execution_id = "q-1" + completed_run_response.superset_execution_log = {} + # query job -> mock_job, query TranslationRun -> completed_run_response + db.query.return_value.filter.return_value.first.side_effect = [ + mock_job, + completed_run_response, + ] + + # Create a PENDING input run + run = MagicMock() + run.id = "run-1" + run.job_id = "job-123" + run.status = "PENDING" + + with patch( + "src.plugins.translate.orchestrator.TranslationExecutor" + ) as MockExecutor: + mock_executor_instance = MagicMock() + mock_executor_instance.execute_run.return_value = completed_run_response + MockExecutor.return_value = mock_executor_instance + + orch = TranslationOrchestrator(db, config_manager, "test-user") + with patch.object(orch, "event_log"): + with patch.object( + orch, "_generate_and_insert_sql", + return_value={"status": "success", "query_id": "q-1", "rows_affected": 10}, + ): + result = orch.execute_run(run) + + assert result.status == "COMPLETED" + assert result.insert_status == "success" + assert result.superset_execution_id == "q-1" + assert result.total_records == 10 + assert result.successful_records == 10 + + # endregion test_execute_run_success + + # region test_execute_run_executor_failure [TYPE Function] + # @BRIEF Executor raises exception, run is marked FAILED. + def test_execute_run_executor_failure(self, mock_job: MagicMock) -> None: + db = MagicMock() + config_manager = MagicMock() + + # Mock job query + db.query.return_value.filter.return_value.first.return_value = mock_job + + run = MagicMock() + run.id = "run-1" + run.job_id = "job-123" + run.status = "PENDING" + run.error_message = None + + with patch( + "src.plugins.translate.orchestrator.TranslationExecutor" + ) as MockExecutor: + mock_executor_instance = MagicMock() + mock_executor_instance.execute_run.side_effect = ValueError( + "LLM provider unavailable" + ) + MockExecutor.return_value = mock_executor_instance + + orch = TranslationOrchestrator(db, config_manager, "test-user") + with patch.object(orch, "event_log"): + result = orch.execute_run(run) + + assert result.status == "FAILED" + assert "LLM provider unavailable" in (result.error_message or "") + + # endregion test_execute_run_executor_failure + + # region test_execute_run_skip_insert [TYPE Function] + # @BRIEF skip_insert=True completes run without SQL generation or Superset submission. + def test_execute_run_skip_insert(self, mock_job: MagicMock) -> None: + db = MagicMock() + config_manager = MagicMock() + + db.query.return_value.filter.return_value.first.return_value = mock_job + + run = MagicMock() + run.id = "run-1" + run.job_id = "job-123" + run.status = "PENDING" + run.completed_at = None + + completed_run = MagicMock() + completed_run.id = "run-1" + completed_run.job_id = "job-123" + completed_run.status = "COMPLETED" + completed_run.total_records = 5 + completed_run.successful_records = 5 + completed_run.failed_records = 0 + completed_run.skipped_records = 0 + completed_run.completed_at = datetime.now(timezone.utc) + completed_run.error_message = None + + with patch( + "src.plugins.translate.orchestrator.TranslationExecutor" + ) as MockExecutor: + mock_executor_instance = MagicMock() + mock_executor_instance.execute_run.return_value = completed_run + MockExecutor.return_value = mock_executor_instance + + orch = TranslationOrchestrator(db, config_manager, "test-user") + with patch.object(orch, "_generate_and_insert_sql") as mock_gen_sql: + with patch.object(orch, "event_log"): + result = orch.execute_run(run, skip_insert=True) + + assert result.status == "COMPLETED" + # _generate_and_insert_sql should NOT be called in skip_insert mode + mock_gen_sql.assert_not_called() + assert result.total_records == 5 + + # endregion test_execute_run_skip_insert + + # region test_execute_run_no_job [TYPE Function] + # @BRIEF Job not found raises ValueError. + def test_execute_run_no_job(self) -> None: + db = MagicMock() + config_manager = MagicMock() + + # Mock job query returns None (job not found) + db.query.return_value.filter.return_value.first.return_value = None + + run = MagicMock() + run.id = "run-1" + run.job_id = "job-missing" + run.status = "PENDING" + + orch = TranslationOrchestrator(db, config_manager, "test-user") + with pytest.raises(ValueError, match="not found"): + orch.execute_run(run) + + # endregion test_execute_run_no_job + + # region test_execute_run_insert_failure [TYPE Function] + # @BRIEF SQL generation/insert returns failure, run still completes but with error logged. + def test_execute_run_insert_failure(self, mock_job: MagicMock) -> None: + db = MagicMock() + config_manager = MagicMock() + + # First query returns mock_job, re-query returns completed_run + completed_run_response = MagicMock() + completed_run_response.id = "run-1" + completed_run_response.job_id = "job-123" + completed_run_response.status = "COMPLETED" + completed_run_response.total_records = 3 + completed_run_response.successful_records = 3 + completed_run_response.failed_records = 0 + completed_run_response.skipped_records = 0 + completed_run_response.completed_at = datetime.now(timezone.utc) + completed_run_response.error_message = None + completed_run_response.insert_status = "failed" + completed_run_response.superset_execution_id = "" + completed_run_response.superset_execution_log = {} + db.query.return_value.filter.return_value.first.side_effect = [ + mock_job, + completed_run_response, + ] + + run = MagicMock() + run.id = "run-1" + run.job_id = "job-123" + run.status = "PENDING" + + with patch( + "src.plugins.translate.orchestrator.TranslationExecutor" + ) as MockExecutor: + mock_executor_instance = MagicMock() + mock_executor_instance.execute_run.return_value = completed_run_response + MockExecutor.return_value = mock_executor_instance + + orch = TranslationOrchestrator(db, config_manager, "test-user") + with patch.object( + orch, "_generate_and_insert_sql", + return_value={"status": "failed", "error_message": "timeout", "query_id": None}, + ): + with patch.object(orch, "event_log"): + result = orch.execute_run(run) + + assert result.status == "COMPLETED" + assert result.insert_status == "failed" + assert "timeout" in (result.error_message or "") + + # endregion test_execute_run_insert_failure + # region test_cancel_run [TYPE Function] # @PURPOSE: Cancel a run changes status to CANCELLED. def test_cancel_run(self) -> None: diff --git a/backend/src/plugins/translate/__tests__/test_preview.py b/backend/src/plugins/translate/__tests__/test_preview.py index 2530d199..c7b0e778 100644 --- a/backend/src/plugins/translate/__tests__/test_preview.py +++ b/backend/src/plugins/translate/__tests__/test_preview.py @@ -1,5 +1,4 @@ # region TranslationPreviewTests [TYPE Module] -# @COMPLEXITY: 3 # @SEMANTICS: test, translate, preview, session # @PURPOSE: Test TranslationPreview service: preview, approve/edit/reject state transitions, cost estimation, acceptance gate. # @LAYER: Test diff --git a/backend/src/plugins/translate/__tests__/test_sql_generator.py b/backend/src/plugins/translate/__tests__/test_sql_generator.py index 6405d5cf..6afc040f 100644 --- a/backend/src/plugins/translate/__tests__/test_sql_generator.py +++ b/backend/src/plugins/translate/__tests__/test_sql_generator.py @@ -1,5 +1,4 @@ # region SQLGeneratorTests [TYPE Module] -# @COMPLEXITY: 3 # @SEMANTICS: test, translate, sql_generator # @PURPOSE: Tests for SQLGenerator: PostgreSQL INSERT + UPSERT, ClickHouse INSERT, dialect quoting, NULL handling, injection safety. # @LAYER: Test diff --git a/backend/src/plugins/translate/dictionary.py b/backend/src/plugins/translate/dictionary.py index 50ef4fd3..8abe22c8 100644 --- a/backend/src/plugins/translate/dictionary.py +++ b/backend/src/plugins/translate/dictionary.py @@ -32,7 +32,6 @@ from ._utils import _normalize_term, _detect_delimiter # @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards. class DictionaryManager: # region DictionaryManager.create_dictionary [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Create a new terminology dictionary. # @PRE: payload contains name, source_dialect, target_dialect. # @POST: New TerminologyDictionary row is created and returned. @@ -60,7 +59,6 @@ class DictionaryManager: # endregion DictionaryManager.create_dictionary # region DictionaryManager.update_dictionary [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Update an existing terminology dictionary. # @PRE: dict_id exists in terminology_dictionaries table. # @POST: Dictionary metadata is updated and returned. @@ -92,7 +90,6 @@ class DictionaryManager: # endregion DictionaryManager.update_dictionary # region DictionaryManager.delete_dictionary [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Delete a dictionary, blocked if attached to active/scheduled jobs. # @PRE: dict_id exists. # @POST: Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs. @@ -136,7 +133,6 @@ class DictionaryManager: # endregion DictionaryManager.delete_dictionary # region DictionaryManager.get_dictionary [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Get a single dictionary by ID with entry count. # @PRE: dict_id exists. # @POST: Returns dict with dictionary + entry_count or raises ValueError. @@ -149,7 +145,6 @@ class DictionaryManager: # endregion DictionaryManager.get_dictionary # region DictionaryManager.list_dictionaries [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: List dictionaries with pagination and entry counts. # @PRE: page >= 1, page_size between 1 and 100. # @POST: Returns (list of dicts, total_count). @@ -169,7 +164,6 @@ class DictionaryManager: # endregion DictionaryManager.list_dictionaries # region DictionaryManager.add_entry [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Add an entry to a dictionary, enforcing unique source_term_normalized. # @PRE: dict_id exists. source_term and target_term are non-empty. # @POST: New DictionaryEntry row is created or raises on duplicate. @@ -210,7 +204,6 @@ class DictionaryManager: # endregion DictionaryManager.add_entry # region DictionaryManager.edit_entry [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Edit an existing dictionary entry with duplicate-aware normalization. # @PRE: entry_id exists. # @POST: Entry fields are updated. @@ -255,7 +248,6 @@ class DictionaryManager: # endregion DictionaryManager.edit_entry # region DictionaryManager.delete_entry [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Delete a single dictionary entry. # @PRE: entry_id exists. # @POST: Entry is deleted. @@ -272,7 +264,6 @@ class DictionaryManager: # endregion DictionaryManager.delete_entry # region DictionaryManager.clear_entries [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Delete all entries for a dictionary. # @PRE: dict_id exists. # @POST: All entries for the dictionary are deleted. @@ -290,7 +281,6 @@ class DictionaryManager: # endregion DictionaryManager.clear_entries # region DictionaryManager.list_entries [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: List entries for a dictionary with pagination. # @PRE: dict_id exists. # @POST: Returns (list of entries, total_count). @@ -316,7 +306,6 @@ class DictionaryManager: # endregion DictionaryManager.list_entries # region DictionaryManager.import_entries [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Import entries from CSV/TSV content with duplicate detection and conflict resolution. # @PRE: content is valid CSV or TSV. dict_id exists. # @POST: Entries are created/updated/skipped per conflict mode. Returns result summary. @@ -456,7 +445,6 @@ class DictionaryManager: # endregion DictionaryManager.import_entries # region DictionaryManager.filter_for_batch [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job. # @PRE: job_id exists and source_texts is a list of strings. # @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries. @@ -555,7 +543,6 @@ class DictionaryManager: # region DictionaryManager.submit_correction [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Submit a term correction from a run result. Validates language match, detects conflicts. # @PRE: source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists. # @POST: Entry created or updated; origin tracking populated. Returns action + conflict info. @@ -653,7 +640,6 @@ class DictionaryManager: # endregion DictionaryManager.submit_correction # region DictionaryManager.submit_bulk_corrections [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Submit multiple term corrections atomically — all succeed or all fail. # @PRE: corrections list is non-empty. dict_id exists. # @POST: All corrections applied or none applied with conflict list. diff --git a/backend/src/plugins/translate/events.py b/backend/src/plugins/translate/events.py index ea622b37..4c2c2b3f 100644 --- a/backend/src/plugins/translate/events.py +++ b/backend/src/plugins/translate/events.py @@ -48,7 +48,6 @@ class TranslationEventLog: self.db = db # region log_event [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Write an immutable event. Enforces terminal event invariant for non-null run_id. # @PRE: event_type must be a known type. If run_id is not None, enforce terminal invariant. # @POST: TranslationEvent row is created. diff --git a/backend/src/plugins/translate/executor.py b/backend/src/plugins/translate/executor.py index 35ff5829..0901dfa9 100644 --- a/backend/src/plugins/translate/executor.py +++ b/backend/src/plugins/translate/executor.py @@ -164,12 +164,102 @@ class TranslationExecutor: # endregion execute_run # region _fetch_source_rows [TYPE Function] - # @PURPOSE: Fetch source rows from the accepted preview session for this job. - # @PRE: job_id exists. - # @POST: Returns list of dicts with source data. + # @PURPOSE: Fetch full source dataset from Superset (via datasource) for full translation. + # @PRE: job_id exists. Job may have source_datasource_id for full fetch. + # @POST: Returns list of dicts with source data (all rows from the source datasource). + # @SIDE_EFFECT: Makes HTTP call to Superset chart data API when datasource is configured. def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]: with belief_scope("TranslationExecutor._fetch_source_rows"): - # Get the latest APPLIED preview session + job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() + + # If source_datasource_id is configured, fetch ALL rows from the Superset chart data API + if job and job.source_datasource_id: + try: + logger.reason("Fetching full dataset from Superset datasource", { + "run_id": run_id, + "datasource_id": job.source_datasource_id, + "environment_id": job.environment_id, + }) + + # Determine environment + environments = self.config_manager.get_environments() + target_env_id = job.environment_id or job.source_dialect or "" + env_config = next( + (e for e in environments if e.id == target_env_id), + None, + ) + if not env_config and environments: + env_config = environments[0] + + if env_config: + from ...core.superset_client import SupersetClient + client = SupersetClient(env_config) + + # Fetch dataset detail to build proper query context + dataset_detail = client.get_dataset_detail(int(job.source_datasource_id)) + + # Build query context (same approach as preview but without row_limit) + query_context = client.build_dataset_preview_query_context( + dataset_id=int(job.source_datasource_id), + dataset_record=dataset_detail, + template_params={}, + effective_filters=[], + ) + + # Remove row_limit to get ALL rows; use result_type="samples" + queries = query_context.get("queries", []) + if queries: + queries[0].pop("row_limit", None) + queries[0].pop("result_type", None) + queries[0]["metrics"] = [] + query_context["result_type"] = "samples" + form_data = query_context.get("form_data", {}) + form_data.pop("query_mode", None) + + response = client.network.request( + method="POST", + endpoint="/api/v1/chart/data", + data=json.dumps(query_context), + headers={"Content-Type": "application/json"}, + ) + + # Extract rows + rows = self._extract_chart_data_rows(response) + + if rows: + logger.reason(f"Fetched {len(rows)} rows from Superset datasource", { + "run_id": run_id, + }) + # Map rows to source_rows format + source_rows = [] + for idx, row in enumerate(rows): + source_data_dict = dict(row) if row else None + source_text = str(row.get(job.translation_column, "")) if job.translation_column else json.dumps(row) + source_rows.append({ + "row_index": str(idx), + "source_text": source_text, + "approved_translation": None, + "source_object_name": f"Row {idx}", + "source_data": source_data_dict, + }) + return source_rows + else: + logger.explore("Superset datasource returned no rows", { + "run_id": run_id, + "datasource_id": job.source_datasource_id, + }) + else: + logger.explore("No environment config found for datasource fetch", { + "env_id": target_env_id, + }) + except Exception as e: + logger.explore("Failed to fetch full dataset from Superset, falling back to preview", { + "run_id": run_id, + "error": str(e), + }) + # Fall through to preview-based fetch + + # Fallback: get the latest APPLIED preview session session = ( self.db.query(TranslationPreviewSession) .filter( @@ -195,20 +285,48 @@ class TranslationExecutor: source_rows = [] for rec in records: + source_data_dict = None + if hasattr(rec, "source_data") and rec.source_data: + source_data_dict = dict(rec.source_data) source_rows.append({ "row_index": rec.source_object_id or "0", "source_text": rec.source_sql or "", "approved_translation": rec.target_sql if rec.status == "APPROVED" else None, "source_object_name": rec.source_object_name or "", + "source_data": source_data_dict, }) - logger.reason(f"Fetched {len(source_rows)} source rows from preview", { + logger.reason(f"Fetched {len(source_rows)} source rows from preview fallback", { "run_id": run_id, "session_id": session.id, }) return source_rows # endregion _fetch_source_rows + # region _extract_chart_data_rows [TYPE Function] + # @PURPOSE: Extract data rows from Superset chart data API response. + # @POST: Returns list of dicts with column-value pairs. + @staticmethod + def _extract_chart_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]: + result = response.get("result") + if isinstance(result, list): + for item in result: + if isinstance(item, dict): + data = item.get("data") + if isinstance(data, list) and data: + return data + if isinstance(result, dict): + data = result.get("data") + if isinstance(data, list) and data: + return data + data = response.get("data") + if isinstance(data, list) and data: + return data + if isinstance(result, list): + return result + return [] + # endregion _extract_chart_data_rows + # region _process_batch [TYPE Function] # @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records. # @PRE: job and batch_rows are valid. @@ -272,6 +390,7 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), + source_data=row.get("source_data"), status="SUCCESS", ) self.db.add(record) @@ -398,6 +517,7 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), + source_data=row.get("source_data"), status="FAILED", error_message=f"LLM call failed after {retries} retries: {last_error}", ) @@ -425,6 +545,7 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), + source_data=row.get("source_data"), status="SKIPPED", error_message=f"LLM parse failure: {e}", ) @@ -456,6 +577,7 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), + source_data=row.get("source_data"), status="SKIPPED", error_message="NULL translation returned by LLM", ) @@ -474,6 +596,7 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), + source_data=row.get("source_data"), status="SKIPPED", error_message="Empty translation returned by LLM", ) @@ -490,6 +613,7 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), + source_data=row.get("source_data"), status="SUCCESS", ) self.db.add(record) diff --git a/backend/src/plugins/translate/orchestrator.py b/backend/src/plugins/translate/orchestrator.py index 9cc852bb..f9196eeb 100644 --- a/backend/src/plugins/translate/orchestrator.py +++ b/backend/src/plugins/translate/orchestrator.py @@ -63,7 +63,6 @@ class TranslationOrchestrator: self._job: Optional[TranslationJob] = None # region start_run [TYPE Function] - # @COMPLEXITY: 5 # @PURPOSE: Start a new translation run for a job. # @PRE: job_id exists. For manual runs, there must be an accepted preview session. # @POST: TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded. @@ -105,6 +104,7 @@ class TranslationOrchestrator: "source_key_cols": job.source_key_cols, "target_key_cols": job.target_key_cols, "translation_column": job.translation_column, + "target_column": job.target_column, "context_columns": job.context_columns, "target_language": job.target_language, "provider_id": job.provider_id, @@ -263,7 +263,11 @@ class TranslationOrchestrator: run.insert_status = insert_result.get("status") run.superset_execution_id = str(insert_result.get("query_id") or "") run.superset_execution_log = insert_result - run.status = "COMPLETED" if insert_result.get("status") == "success" else "COMPLETED" + # Preserve the translation-phase status. If the executor already + # marked the run FAILED (all LLM calls failed) we do NOT upgrade it + # to COMPLETED just because the insert phase ran. + if run.status != "FAILED": + run.status = "COMPLETED" if insert_result.get("error_message"): run.error_message = insert_result["error_message"] run.completed_at = datetime.now(timezone.utc) @@ -288,7 +292,9 @@ class TranslationOrchestrator: ) self.db.commit() - self.db.refresh(run) + # Re-query run after commit — refresh may fail if the object was + # created in a different session or became detached during commit. + run = self.db.query(TranslationRun).filter(TranslationRun.id == run.id).first() logger.reflect("Run execution complete", { "run_id": run.id, @@ -329,11 +335,20 @@ class TranslationOrchestrator: "dialect": job.database_dialect or job.target_dialect, }) - # Build rows for SQL generation + # Determine effective target column for INSERT (defaults to translation_column) + effective_target = job.target_column or job.translation_column + + # Build columns for SQL generation columns = job.context_columns or [] + + # Always include translation_column (original text) if it's different from target if job.translation_column and job.translation_column not in columns: columns.append(job.translation_column) + # Add target_column separately if it differs from translation_column + if effective_target and effective_target != job.translation_column and effective_target not in columns: + columns.append(effective_target) + # Also include key columns if used for upsert if job.target_key_cols: for k in job.target_key_cols: @@ -343,27 +358,57 @@ class TranslationOrchestrator: rows_for_sql = [] for rec in records: row_data = {} + source_data = rec.source_data or {} + + # Context columns from source data if job.context_columns: for col in job.context_columns: - row_data[col] = "" - if job.translation_column: - row_data[job.translation_column] = rec.target_sql or "" + row_data[col] = source_data.get(col, "") + + # Original text column (translation_column) + if job.translation_column and job.translation_column not in (job.target_key_cols or []): + # If target_column differs, keep the original value from source_data; + # otherwise use the translated value + if effective_target and effective_target != job.translation_column: + row_data[job.translation_column] = source_data.get(job.translation_column, "") + else: + row_data[job.translation_column] = rec.target_sql or "" + + # Translated text goes into target_column (may be same as translation_column) + if effective_target: + row_data[effective_target] = rec.target_sql or "" + + # Key columns from source data if job.target_key_cols: - source_data = rec.source_data or {} for k in job.target_key_cols: - # Use source_data value if available, fall back to empty string row_data[k] = source_data.get(k, "") rows_for_sql.append(row_data) if not columns: # Use target_sql as the sole column - columns = [job.translation_column or "translated_text"] + columns = [effective_target or "translated_text"] rows_for_sql = [{columns[0]: rec.target_sql or ""} for rec in records] + # Resolve the real database backend engine from Superset + try: + env_id = job.environment_id or job.source_dialect or "" + executor = SupersetSqlLabExecutor(self.config_manager, env_id) + executor.resolve_database_id( + target_database_id=job.target_database_id, + ) + real_backend = executor.get_database_backend() + except Exception as e: + logger.explore("Failed to resolve database backend, falling back to job dialet", { + "error": str(e), + }) + real_backend = None + + dialect = real_backend or job.database_dialect or job.target_dialect or "postgresql" + # Generate SQL try: sql, row_count = SQLGenerator.generate( - dialect=job.database_dialect or job.target_dialect or "postgresql", + dialect=dialect, target_schema=job.target_schema, target_table=job.target_table or "translated_data", columns=columns, @@ -375,19 +420,24 @@ class TranslationOrchestrator: logger.explore("SQL generation failed", {"error": str(e)}) return {"status": "failed", "error_message": str(e), "query_id": None} + logger.reason("SQL generated with dialect", { + "dialect": dialect, + "real_backend": real_backend, + "job_database_dialect": job.database_dialect, + "job_target_dialect": job.target_dialect, + }) + # Log insert phase start self.event_log.log_event( job_id=job.id, run_id=run.id, event_type="INSERT_PHASE_STARTED", - payload={"sql_length": len(sql), "row_count": row_count}, + payload={"sql_length": len(sql), "row_count": row_count, "dialect": dialect}, created_by=self.current_user, ) # Submit to Superset try: - env_id = job.environment_id or job.source_dialect or "" - executor = SupersetSqlLabExecutor(self.config_manager, env_id) result = executor.execute_and_poll( sql=sql, max_polls=30, diff --git a/backend/src/plugins/translate/preview.py b/backend/src/plugins/translate/preview.py index 138cea0e..3431b45a 100644 --- a/backend/src/plugins/translate/preview.py +++ b/backend/src/plugins/translate/preview.py @@ -134,7 +134,6 @@ class TranslationPreview: self.current_user = current_user # region preview_rows [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records. # @PRE: job_id exists and job has source_datasource_id, translation_column configured. # @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session. @@ -290,6 +289,19 @@ class TranslationPreview: status = "PENDING" feedback = None + # Extract source_data: store original row key columns for upsert matching + source_row = meta.get("source_row", {}) + source_data = None + if job.target_key_cols: + source_data = { + k: source_row.get(k) + for k in job.target_key_cols + if k in source_row + } + elif source_row: + # No key columns configured — store the full row as fallback + source_data = dict(source_row) + record = TranslationPreviewRecord( id=str(uuid.uuid4()), session_id=session.id, @@ -298,6 +310,7 @@ class TranslationPreview: source_object_type="table_row", source_object_id=str(idx), source_object_name=f"Row {idx + 1}", + source_data=source_data, status=status, feedback=feedback, created_at=datetime.now(timezone.utc), @@ -516,7 +529,6 @@ class TranslationPreview: # endregion get_preview_session # region _fetch_sample_rows [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Fetch sample rows from the Superset dataset for preview. # @PRE: job has source_datasource_id and translation_column. # @POST: Returns list of dicts with row data. @@ -636,7 +648,6 @@ class TranslationPreview: # endregion _extract_data_rows # region _call_llm [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Call the configured LLM provider with the preview prompt. # @PRE: job has a valid provider_id. # @POST: Returns raw LLM response string. diff --git a/backend/src/plugins/translate/service.py b/backend/src/plugins/translate/service.py index 0b802e74..f76721a2 100644 --- a/backend/src/plugins/translate/service.py +++ b/backend/src/plugins/translate/service.py @@ -241,12 +241,14 @@ class TranslateJobService: source_key_cols=payload.source_key_cols or [], target_key_cols=payload.target_key_cols or [], translation_column=payload.translation_column, + target_column=payload.target_column, context_columns=payload.context_columns or [], target_language=payload.target_language, provider_id=payload.provider_id, batch_size=payload.batch_size, upsert_strategy=payload.upsert_strategy, environment_id=payload.environment_id, + target_database_id=payload.target_database_id, status="DRAFT", created_by=self.current_user, ) @@ -360,6 +362,7 @@ class TranslateJobService: source_key_cols=source.source_key_cols, target_key_cols=source.target_key_cols, translation_column=source.translation_column, + target_column=source.target_column, context_columns=source.context_columns, target_language=source.target_language, provider_id=source.provider_id, @@ -461,6 +464,7 @@ def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) - source_key_cols=job.source_key_cols or [], target_key_cols=job.target_key_cols or [], translation_column=job.translation_column, + target_column=job.target_column, context_columns=job.context_columns or [], target_language=job.target_language, provider_id=job.provider_id, @@ -472,6 +476,7 @@ def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) - updated_at=job.updated_at, dictionary_ids=dict_ids or [], environment_id=job.environment_id, + target_database_id=job.target_database_id, ) # #endregion job_to_response diff --git a/backend/src/plugins/translate/sql_generator.py b/backend/src/plugins/translate/sql_generator.py index 6acf4d2c..3a3a9fd5 100644 --- a/backend/src/plugins/translate/sql_generator.py +++ b/backend/src/plugins/translate/sql_generator.py @@ -90,11 +90,16 @@ def _encode_sql_value(value: Any, dialect: Optional[str] = None) -> str: if isinstance(value, bool): return "TRUE" if value else "FALSE" - # For ClickHouse: try to normalize timestamp-like string values - if dialect in CLICKHOUSE_DIALECTS and isinstance(value, str) and value: - normalized = _normalize_timestamp_value(value) - if normalized: - return f"'{normalized}'" + # For ClickHouse: try to normalize timestamp-like values (both string and numeric) + if dialect in CLICKHOUSE_DIALECTS: + if isinstance(value, str) and value: + normalized = _normalize_timestamp_value(value) + if normalized: + return f"'{normalized}'" + elif isinstance(value, (int, float)): + normalized = _normalize_timestamp_value(value) + if normalized: + return f"'{normalized}'" if isinstance(value, (int, float)): return str(value) diff --git a/backend/src/plugins/translate/superset_executor.py b/backend/src/plugins/translate/superset_executor.py index c42e30c4..7ae5236c 100644 --- a/backend/src/plugins/translate/superset_executor.py +++ b/backend/src/plugins/translate/superset_executor.py @@ -32,6 +32,7 @@ class SupersetSqlLabExecutor: self.env_id = env_id self._client: Optional[SupersetClient] = None self._database_id: Optional[int] = None + self._database_backend: Optional[str] = None # region _get_client [TYPE Function] # @PURPOSE: Lazy-initialize SupersetClient for the configured environment. @@ -53,15 +54,43 @@ class SupersetSqlLabExecutor: # endregion _get_client # region resolve_database_id [TYPE Function] - # @PURPOSE: Resolve the target database ID from the environment. + # @PURPOSE: Resolve the target database ID from the environment and fetch its backend engine. # @PRE: database_name or database_id should be known. - # @POST: Returns database_id integer or raises ValueError. - # @SIDE_EFFECT: Fetches databases list from Superset. - def resolve_database_id(self, database_name: Optional[str] = None) -> int: + # @POST: Returns database_id integer or raises ValueError. Also sets self._database_backend. + # @SIDE_EFFECT: Fetches databases list from Superset; optionally fetches single DB for backend. + def resolve_database_id( + self, + database_name: Optional[str] = None, + target_database_id: Optional[str] = None, + ) -> int: with belief_scope("SupersetSqlLabExecutor.resolve_database_id"): client = self._get_client() + + # If a specific target_database_id is provided, use it directly + if target_database_id: + try: + db_id = int(target_database_id) + # Fetch full DB info to get backend + db_info = client.get_database(db_id) + result = db_info.get("result", db_info) + self._database_id = result.get("id", db_id) + self._database_backend = result.get("backend") or result.get("engine") + logger.reason("Resolved database ID by target_database_id", { + "target_database_id": target_database_id, + "database_id": self._database_id, + "backend": self._database_backend, + }) + return self._database_id + except Exception as e: + logger.explore("Failed to resolve by target_database_id, falling back", { + "target_database_id": target_database_id, + "error": str(e), + }) + # Fall through to default resolution + + # Fetch full database list with backend info _, databases = client.get_databases( - query={"columns": ["id", "database_name"]} + query={"columns": ["id", "database_name", "backend"]} ) if not databases: raise ValueError("No databases found in Superset environment") @@ -70,9 +99,11 @@ class SupersetSqlLabExecutor: for db in databases: if db.get("database_name", "").lower() == database_name.lower(): self._database_id = db["id"] + self._database_backend = db.get("backend") logger.reason("Resolved database ID by name", { "database_name": database_name, "database_id": self._database_id, + "backend": self._database_backend, }) return self._database_id raise ValueError( @@ -81,13 +112,22 @@ class SupersetSqlLabExecutor: # Default: use first database self._database_id = databases[0]["id"] + self._database_backend = databases[0].get("backend") logger.reason("Using default database", { "database_name": databases[0].get("database_name"), "database_id": self._database_id, + "backend": self._database_backend, }) return self._database_id # endregion resolve_database_id + # region get_database_backend [TYPE Function] + # @PURPOSE: Return the cached database backend/engine string. + # @POST: Returns backend string or None if not yet resolved. + def get_database_backend(self) -> Optional[str]: + return self._database_backend + # endregion get_database_backend + # region execute_sql [TYPE Function] # @PURPOSE: Submit SQL to Superset SQL Lab and return execution tracking info. # @PRE: sql is valid SQL string. database_id is a valid Superset DB ID. @@ -117,27 +157,75 @@ class SupersetSqlLabExecutor: "runAsync": run_async, "schema": None, "tab": "translation-insert", + "client_id": f"trl-{uuid.uuid4().hex[:7]}", # max 11 chars for Superset varchar(11) } - try: - response = client.network.request( - method="POST", - endpoint="/api/v1/sqllab/execute/", - data=json.dumps(payload), - headers={"Content-Type": "application/json"}, - ) - except Exception as e: - logger.explore("SQL Lab execute failed", {"error": str(e)}) - raise ValueError(f"Superset SQL Lab execute failed: {e}") + # Try multiple SQL Lab endpoints — some Superset versions use /sqllab/execute/, + # others use /sql_lab/execute/ (with underscore) + candidate_endpoints = ["/api/v1/sqllab/execute/", "/api/v1/sql_lab/execute/"] + response = None + last_error = None + for endpoint in candidate_endpoints: + try: + response = client.network.request( + method="POST", + endpoint=endpoint, + data=json.dumps(payload), + headers={"Content-Type": "application/json"}, + ) + if isinstance(response, dict) and response: + logger.reason(f"SQL Lab endpoint succeeded", extra={ + "endpoint": endpoint, + }) + break + except Exception as ep_err: + last_error = ep_err + logger.explore(f"SQL Lab endpoint failed, trying next", extra={ + "error": str(ep_err), + "endpoint": endpoint, + }) + response = None - # Parse response + if response is None: + error_msg = f"All SQL Lab endpoints failed. Last error: {last_error}" + logger.explore("SQL Lab execute failed", extra={"error": error_msg}) + raise ValueError(f"Superset SQL Lab execute failed: {error_msg}") + + # Parse response — try multiple known Superset response formats result = response if isinstance(response, dict) else {} - query_id = result.get("query_id") or result.get("id") + + # Superset may return query_id at top level, nested in "result", as "id", or in "query" block + query_id = ( + result.get("query_id") + or result.get("id") + or (result.get("result") or {}).get("query_id") + or (result.get("result") or {}).get("id") + or (result.get("query") or {}).get("query_id") + or (result.get("query") or {}).get("id") + ) status = result.get("status", "unknown") + # Log full response for debugging if query_id is missing + if not query_id: + logger.explore("No query_id from SQL Lab execute", extra={ + "database_id": db_id, + "status": status, + "raw_response_keys": list(result.keys()), + "raw_response_preview": json.dumps(result)[:2000], + }) + logger.reason("SQL Lab execute response (no query_id)", extra={ + "database_id": db_id, + "status": status, + "response_keys": list(result.keys()), + "has_result": "result" in result, + "has_query": "query" in result, + "raw_preview": json.dumps(result)[:2000], + }) + logger.reason("SQL Lab execute response", { "query_id": query_id, "status": status, + "response_keys": list(result.keys()), }) return { @@ -174,7 +262,9 @@ class SupersetSqlLabExecutor: method="GET", endpoint=f"/api/v1/query/{query_id}", ) - result = response if isinstance(response, dict) else {} + raw = response if isinstance(response, dict) else {} + # Superset wraps query data in a nested "result" field + result = raw.get("result", raw) status = result.get("status", "unknown") state = result.get("state", result.get("status", "")) @@ -255,9 +345,27 @@ class SupersetSqlLabExecutor: exec_result = self.execute_sql( sql=sql, database_id=database_id, - run_async=True, + run_async=False, # Sync mode — Superset returns result directly ) + status = exec_result.get("status", "unknown") + + # Sync mode: response already has the final status and data + if status == "success": + logger.reason("SQL execution completed (sync)", { + "query_id": exec_result.get("query_id"), + }) + return { + "query_id": exec_result.get("query_id"), + "status": "success", + "state": "success", + "rows_affected": exec_result.get("raw_response", {}).get("query", {}).get("rows"), + "error_message": None, + "results": exec_result.get("raw_response", {}).get("data"), + "completed_on": exec_result.get("raw_response", {}).get("query", {}).get("endDttm"), + } + + # For async mode (fallback): poll for completion query_id = exec_result.get("query_id") if not query_id: logger.explore("No query_id from SQL Lab execute", { diff --git a/backend/src/schemas/translate.py b/backend/src/schemas/translate.py index 9af644a6..516f3462 100644 --- a/backend/src/schemas/translate.py +++ b/backend/src/schemas/translate.py @@ -23,7 +23,8 @@ class TranslateJobCreate(BaseModel): target_table: Optional[str] = Field(None, description="Target table name") source_key_cols: Optional[List[str]] = Field(default_factory=list, description="Source key column names") target_key_cols: Optional[List[str]] = Field(default_factory=list, description="Target key column names") - translation_column: Optional[str] = Field(None, description="Column to translate") + translation_column: Optional[str] = Field(None, description="Source column to translate") + target_column: Optional[str] = Field(None, description="Target column for translated output (defaults to translation_column)") context_columns: Optional[List[str]] = Field(default_factory=list, description="Context column names") target_language: Optional[str] = Field(None, description="Target language code") provider_id: Optional[str] = Field(None, description="LLM provider ID") @@ -31,6 +32,7 @@ class TranslateJobCreate(BaseModel): upsert_strategy: str = Field("MERGE", description="UPSERT strategy: MERGE, INSERT, UPDATE") dictionary_ids: Optional[List[str]] = Field(default_factory=list, description="Associated terminology dictionary IDs") environment_id: Optional[str] = Field(None, description="Superset environment ID") + target_database_id: Optional[str] = Field(None, description="Superset database ID for SQL Lab insert target") # #endregion TranslateJobCreate @@ -49,6 +51,7 @@ class TranslateJobUpdate(BaseModel): source_key_cols: Optional[List[str]] = None target_key_cols: Optional[List[str]] = None translation_column: Optional[str] = None + target_column: Optional[str] = None context_columns: Optional[List[str]] = None target_language: Optional[str] = None provider_id: Optional[str] = None @@ -57,6 +60,7 @@ class TranslateJobUpdate(BaseModel): status: Optional[str] = None dictionary_ids: Optional[List[str]] = None environment_id: Optional[str] = None + target_database_id: Optional[str] = None # #endregion TranslateJobUpdate @@ -76,6 +80,7 @@ class TranslateJobResponse(BaseModel): source_key_cols: Optional[List[str]] = None target_key_cols: Optional[List[str]] = None translation_column: Optional[str] = None + target_column: Optional[str] = None context_columns: Optional[List[str]] = None target_language: Optional[str] = None provider_id: Optional[str] = None @@ -87,6 +92,7 @@ class TranslateJobResponse(BaseModel): updated_at: Optional[datetime] = None dictionary_ids: Optional[List[str]] = None environment_id: Optional[str] = None + target_database_id: Optional[str] = None class Config: from_attributes = True diff --git a/backend/src/services/__tests__/test_encryption_manager.py b/backend/src/services/__tests__/test_encryption_manager.py index deeccf57..84d36c66 100644 --- a/backend/src/services/__tests__/test_encryption_manager.py +++ b/backend/src/services/__tests__/test_encryption_manager.py @@ -1,6 +1,5 @@ # region test_encryption_manager [TYPE Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 3 # @SEMANTICS: encryption, security, fernet, api-keys, tests # @PURPOSE: Unit tests for EncryptionManager encrypt/decrypt functionality. # @LAYER: Domain diff --git a/backend/src/services/__tests__/test_health_service.py b/backend/src/services/__tests__/test_health_service.py index 0b3ee05c..ff5cf371 100644 --- a/backend/src/services/__tests__/test_health_service.py +++ b/backend/src/services/__tests__/test_health_service.py @@ -5,7 +5,6 @@ from src.services.health_service import HealthService from src.models.llm import ValidationRecord # region test_health_service [TYPE Module] -# @COMPLEXITY: 3 # @PURPOSE: Unit tests for HealthService aggregation logic. # @RELATION: VERIFIES ->[src.services.health_service.HealthService] diff --git a/backend/src/services/__tests__/test_llm_plugin_persistence.py b/backend/src/services/__tests__/test_llm_plugin_persistence.py index 661d4687..525b787b 100644 --- a/backend/src/services/__tests__/test_llm_plugin_persistence.py +++ b/backend/src/services/__tests__/test_llm_plugin_persistence.py @@ -1,6 +1,5 @@ # region test_llm_plugin_persistence [TYPE Module] # @RELATION: VERIFIES -> [DashboardValidationPlugin:Class] -# @COMPLEXITY: 3 # @PURPOSE: Regression test for ValidationRecord persistence fields populated from task context. import types @@ -11,7 +10,6 @@ from src.plugins.llm_analysis import plugin as plugin_module # region _DummyLogger [TYPE Class] # @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module] -# @COMPLEXITY: 1 # @PURPOSE: Minimal logger shim for TaskContext-like objects used in tests. # @INVARIANT: Logging methods are no-ops and must not mutate test state. class _DummyLogger: @@ -36,7 +34,6 @@ class _DummyLogger: # region _FakeDBSession [TYPE Class] # @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module] -# @COMPLEXITY: 2 # @PURPOSE: Captures persisted records for assertion and mimics SQLAlchemy session methods used by plugin. # @INVARIANT: add/commit/close provide only persistence signals asserted by this test. class _FakeDBSession: @@ -61,7 +58,6 @@ class _FakeDBSession: # region test_dashboard_validation_plugin_persists_task_and_environment_ids [TYPE Function] # @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module] # @RELATION: VERIFIES -> [DashboardValidationPlugin:Class] -# @COMPLEXITY: 2 # @PURPOSE: Ensure db ValidationRecord includes context.task_id and params.environment_id. # @INVARIANT: Assertions remain restricted to persisted task/environment identity fields and session lifecycle signals. @pytest.mark.asyncio @@ -82,7 +78,6 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( # region _FakeProviderService [TYPE Class] # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] - # @COMPLEXITY: 1 # @PURPOSE: LLM provider service stub returning deterministic provider and decrypted API key for plugin tests. # @INVARIANT: Returns same provider and key regardless of provider_id argument; no lookup logic. class _FakeProviderService: @@ -99,7 +94,6 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( # region _FakeScreenshotService [TYPE Class] # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] - # @COMPLEXITY: 1 # @PURPOSE: Screenshot service stub that accepts capture_dashboard calls without side effects. # @INVARIANT: capture_dashboard is intentionally permissive for this persistence-focused test and does not validate argument values. class _FakeScreenshotService: @@ -113,7 +107,6 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( # region _FakeLLMClient [TYPE Class] # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] - # @COMPLEXITY: 2 # @PURPOSE: Deterministic LLM client double returning canonical analysis payload for persistence-path assertions. # @INVARIANT: analyze_dashboard is side-effect free and returns schema-compatible PASS result. class _FakeLLMClient: @@ -137,7 +130,6 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( # region _FakeNotificationService [TYPE Class] # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] - # @COMPLEXITY: 1 # @PURPOSE: Notification service stub that accepts plugin dispatch_report payload without introducing side effects. # @INVARIANT: dispatch_report accepts arbitrary keyword payloads because this test verifies persistence fields, not notification payload schema. class _FakeNotificationService: @@ -151,7 +143,6 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( # region _FakeConfigManager [TYPE Class] # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] - # @COMPLEXITY: 1 # @PURPOSE: Config manager stub providing storage root path and minimal settings for plugin execution path. # @INVARIANT: Only storage.root_path and llm fields are safe to access; all other settings fields are absent. class _FakeConfigManager: @@ -170,7 +161,6 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( # region _FakeSupersetClient [TYPE Class] # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] - # @COMPLEXITY: 1 # @PURPOSE: Superset client stub exposing network.request as a lambda that returns empty result list. # @INVARIANT: network.request intentionally accepts arbitrary keyword payloads because response shape, not request signature, is the persistence-path dependency. class _FakeSupersetClient: diff --git a/backend/src/services/__tests__/test_llm_prompt_templates.py b/backend/src/services/__tests__/test_llm_prompt_templates.py index 6f9955e1..b71ef356 100644 --- a/backend/src/services/__tests__/test_llm_prompt_templates.py +++ b/backend/src/services/__tests__/test_llm_prompt_templates.py @@ -1,5 +1,4 @@ # region test_llm_prompt_templates [TYPE Module] -# @COMPLEXITY: 3 # @SEMANTICS: tests, llm, prompts, templates, settings # @PURPOSE: Validate normalization and rendering behavior for configurable LLM prompt templates. # @LAYER: Domain Tests @@ -19,7 +18,6 @@ from src.services.llm_prompt_templates import ( # region test_normalize_llm_settings_adds_default_prompts [TYPE Function] # @RELATION: BINDS_TO -> test_llm_prompt_templates -# @COMPLEXITY: 3 # @PURPOSE: Ensure legacy/partial llm settings are expanded with all prompt defaults. # @PRE: Input llm settings do not contain complete prompts object. # @POST: Returned structure includes required prompt templates with fallback defaults. @@ -43,7 +41,6 @@ def test_normalize_llm_settings_adds_default_prompts(): # region test_normalize_llm_settings_keeps_custom_prompt_values [TYPE Function] # @RELATION: BINDS_TO -> test_llm_prompt_templates -# @COMPLEXITY: 3 # @PURPOSE: Ensure user-customized prompt values are preserved during normalization. # @PRE: Input llm settings contain custom prompt override. # @POST: Custom prompt value remains unchanged in normalized output. @@ -59,7 +56,6 @@ def test_normalize_llm_settings_keeps_custom_prompt_values(): # region test_render_prompt_replaces_known_placeholders [TYPE Function] # @RELATION: BINDS_TO -> test_llm_prompt_templates -# @COMPLEXITY: 3 # @PURPOSE: Ensure template placeholders are deterministically replaced. # @PRE: Template contains placeholders matching provided variables. # @POST: Rendered prompt string contains substituted values. @@ -77,7 +73,6 @@ def test_render_prompt_replaces_known_placeholders(): # region test_is_multimodal_model_detects_known_vision_models [TYPE Function] # @RELATION: BINDS_TO -> test_llm_prompt_templates -# @COMPLEXITY: 2 # @PURPOSE: Ensure multimodal model detection recognizes common vision-capable model names. def test_is_multimodal_model_detects_known_vision_models(): assert is_multimodal_model("gpt-4o") is True @@ -91,7 +86,6 @@ def test_is_multimodal_model_detects_known_vision_models(): # region test_resolve_bound_provider_id_prefers_binding_then_default [TYPE Function] # @RELATION: BINDS_TO -> test_llm_prompt_templates -# @COMPLEXITY: 2 # @PURPOSE: Verify provider binding resolution priority. def test_resolve_bound_provider_id_prefers_binding_then_default(): settings = { @@ -107,7 +101,6 @@ def test_resolve_bound_provider_id_prefers_binding_then_default(): # region test_normalize_llm_settings_keeps_assistant_planner_settings [TYPE Function] # @RELATION: BINDS_TO -> test_llm_prompt_templates -# @COMPLEXITY: 2 # @PURPOSE: Ensure assistant planner provider/model fields are preserved and normalized. def test_normalize_llm_settings_keeps_assistant_planner_settings(): normalized = normalize_llm_settings( diff --git a/backend/src/services/__tests__/test_llm_provider.py b/backend/src/services/__tests__/test_llm_provider.py index b6e43a88..36fe3119 100644 --- a/backend/src/services/__tests__/test_llm_provider.py +++ b/backend/src/services/__tests__/test_llm_provider.py @@ -1,6 +1,5 @@ # region test_llm_provider [TYPE Module] # @RELATION: VERIFIES -> [src.services.llm_provider:Module] -# @COMPLEXITY: 3 # @SEMANTICS: tests, llm-provider, encryption, contract # @PURPOSE: Contract testing for LLMProviderService and EncryptionManager # endregion test_llm_provider @@ -68,7 +67,6 @@ def test_decrypt_invalid_data(): # region mock_db [TYPE Fixture] # @RELATION: BINDS_TO -> [test_llm_provider:Module] -# @COMPLEXITY: 1 # @PURPOSE: MagicMock(spec=Session) fixture providing a constrained DB session double for LLMProviderService tests. # @INVARIANT: Chained calls beyond Session spec create unconstrained intermediate mocks; only top-level query/add/commit are spec-enforced. @pytest.fixture @@ -82,7 +80,6 @@ def mock_db(): # region service [TYPE Fixture] # @RELATION: BINDS_TO -> [test_llm_provider:Module] -# @COMPLEXITY: 1 # @PURPOSE: LLMProviderService fixture wired to mock_db for provider CRUD tests. @pytest.fixture def service(mock_db): diff --git a/backend/src/services/__tests__/test_rbac_permission_catalog.py b/backend/src/services/__tests__/test_rbac_permission_catalog.py index a7d85f86..c4f0a3e6 100644 --- a/backend/src/services/__tests__/test_rbac_permission_catalog.py +++ b/backend/src/services/__tests__/test_rbac_permission_catalog.py @@ -1,6 +1,5 @@ # region test_rbac_permission_catalog [TYPE Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 3 # @SEMANTICS: tests, rbac, permissions, catalog, discovery, sync # @PURPOSE: Verifies RBAC permission catalog discovery and idempotent synchronization behavior. # @LAYER: Service Tests diff --git a/backend/src/services/__tests__/test_resource_service.py b/backend/src/services/__tests__/test_resource_service.py index 0b654549..a3ef60f5 100644 --- a/backend/src/services/__tests__/test_resource_service.py +++ b/backend/src/services/__tests__/test_resource_service.py @@ -1,5 +1,4 @@ # region TestResourceService [TYPE Module] -# @COMPLEXITY: 3 # @SEMANTICS: resource-service, tests, dashboards, datasets, activity # @PURPOSE: Unit tests for ResourceService # @LAYER: Service diff --git a/backend/src/services/auth_service.py b/backend/src/services/auth_service.py index af39f2a7..5ad88348 100644 --- a/backend/src/services/auth_service.py +++ b/backend/src/services/auth_service.py @@ -31,7 +31,6 @@ from ..core.logger import belief_scope # @RELATION DEPENDS_ON -> [Role] class AuthService: # region AuthService_init [TYPE Function] - # @COMPLEXITY: 1 # @PURPOSE: Initializes the authentication service with repository access over an active DB session. # @PRE: db is a valid SQLAlchemy Session instance bound to the auth persistence context. # @POST: self.repo is initialized and ready for auth user/role CRUD operations. @@ -45,7 +44,6 @@ class AuthService: # endregion AuthService_init # region AuthService.authenticate_user [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Validates credentials and account state for local username/password authentication. # @PRE: username and password are non-empty credential inputs. # @POST: Returns User only when user exists, is active, and password hash verification succeeds; otherwise returns None. @@ -76,7 +74,6 @@ class AuthService: # endregion AuthService.authenticate_user # region AuthService.create_session [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Issues an access token payload for an already authenticated user. # @PRE: user is a valid User entity containing username and iterable roles with role.name values. # @POST: Returns session dict with non-empty access_token and token_type='bearer'. @@ -98,7 +95,6 @@ class AuthService: # endregion AuthService.create_session # region AuthService.provision_adfs_user [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Performs ADFS Just-In-Time provisioning and role synchronization from AD group mappings. # @PRE: user_info contains identity claims where at least one of 'upn' or 'email' is present; 'groups' may be absent. # @POST: Returns persisted user entity with roles synchronized to mapped AD groups and refreshed state. diff --git a/backend/src/services/clean_release/__tests__/test_audit_service.py b/backend/src/services/clean_release/__tests__/test_audit_service.py index 9fb81ca9..2f6f5333 100644 --- a/backend/src/services/clean_release/__tests__/test_audit_service.py +++ b/backend/src/services/clean_release/__tests__/test_audit_service.py @@ -1,6 +1,5 @@ # region TestAuditService [TYPE Module] # @RELATION: [DEPENDS_ON] ->[AuditService] -# @COMPLEXITY: 3 # @SEMANTICS: tests, clean-release, audit, logging # @PURPOSE: Validate audit hooks emit expected log patterns for clean release lifecycle. # @LAYER: Infra diff --git a/backend/src/services/clean_release/__tests__/test_compliance_orchestrator.py b/backend/src/services/clean_release/__tests__/test_compliance_orchestrator.py index 6960287d..5c90c214 100644 --- a/backend/src/services/clean_release/__tests__/test_compliance_orchestrator.py +++ b/backend/src/services/clean_release/__tests__/test_compliance_orchestrator.py @@ -1,6 +1,5 @@ # region TestComplianceOrchestrator [TYPE Module] # @RELATION: [DEPENDS_ON] ->[ComplianceOrchestrator] -# @COMPLEXITY: 3 # @SEMANTICS: tests, clean-release, orchestrator, stage-state-machine # @PURPOSE: Validate compliance orchestrator stage transitions and final status derivation. # @LAYER: Domain diff --git a/backend/src/services/clean_release/__tests__/test_manifest_builder.py b/backend/src/services/clean_release/__tests__/test_manifest_builder.py index ce414711..94d9d1ef 100644 --- a/backend/src/services/clean_release/__tests__/test_manifest_builder.py +++ b/backend/src/services/clean_release/__tests__/test_manifest_builder.py @@ -1,5 +1,4 @@ # region TestManifestBuilder [TYPE Module] -# @COMPLEXITY: 5 # @SEMANTICS: tests, clean-release, manifest, deterministic # @PURPOSE: Validate deterministic manifest generation behavior for US1. # @LAYER: Domain diff --git a/backend/src/services/clean_release/__tests__/test_preparation_service.py b/backend/src/services/clean_release/__tests__/test_preparation_service.py index 943c3310..3aaf5110 100644 --- a/backend/src/services/clean_release/__tests__/test_preparation_service.py +++ b/backend/src/services/clean_release/__tests__/test_preparation_service.py @@ -1,5 +1,4 @@ # region TestPreparationService [TYPE Module] -# @COMPLEXITY: 3 # @SEMANTICS: tests, clean-release, preparation, flow # @PURPOSE: Validate release candidate preparation flow, including policy evaluation and manifest persisting. # @LAYER: Domain diff --git a/backend/src/services/clean_release/__tests__/test_report_builder.py b/backend/src/services/clean_release/__tests__/test_report_builder.py index e88a80ef..03e791ad 100644 --- a/backend/src/services/clean_release/__tests__/test_report_builder.py +++ b/backend/src/services/clean_release/__tests__/test_report_builder.py @@ -1,6 +1,5 @@ # region TestReportBuilder [TYPE Module] # @RELATION: [DEPENDS_ON] ->[ReportBuilder] -# @COMPLEXITY: 3 # @SEMANTICS: tests, clean-release, report-builder, counters # @PURPOSE: Validate compliance report builder counter integrity and blocked-run constraints. # @LAYER: Domain diff --git a/backend/src/services/clean_release/__tests__/test_source_isolation.py b/backend/src/services/clean_release/__tests__/test_source_isolation.py index 709d8dc2..59fe1a8c 100644 --- a/backend/src/services/clean_release/__tests__/test_source_isolation.py +++ b/backend/src/services/clean_release/__tests__/test_source_isolation.py @@ -1,6 +1,5 @@ # region TestSourceIsolation [TYPE Module] # @RELATION: [DEPENDS_ON] ->[SourceIsolation] -# @COMPLEXITY: 3 # @SEMANTICS: tests, clean-release, source-isolation, internal-only # @PURPOSE: Verify internal source registry validation behavior. # @LAYER: Domain diff --git a/backend/src/services/clean_release/__tests__/test_stages.py b/backend/src/services/clean_release/__tests__/test_stages.py index c71a876f..ac409287 100644 --- a/backend/src/services/clean_release/__tests__/test_stages.py +++ b/backend/src/services/clean_release/__tests__/test_stages.py @@ -1,6 +1,5 @@ # region TestStages [TYPE Module] # @RELATION: [DEPENDS_ON] ->[ComplianceStages] -# @COMPLEXITY: 3 # @SEMANTICS: tests, clean-release, compliance, stages # @PURPOSE: Validate final status derivation logic from stage results. # @LAYER: Domain diff --git a/backend/src/services/clean_release/repositories/manifest_repository.py b/backend/src/services/clean_release/repositories/manifest_repository.py index 34f5d284..62d70828 100644 --- a/backend/src/services/clean_release/repositories/manifest_repository.py +++ b/backend/src/services/clean_release/repositories/manifest_repository.py @@ -19,7 +19,6 @@ class ManifestRepository: """Repository for distribution manifest persistence.""" # region ManifestRepository.__init__ [TYPE Function] - # @COMPLEXITY: 1 # @PURPOSE: Initialize repository with an active SQLAlchemy session. # @PRE: db is a valid SQLAlchemy Session instance. # @POST: Repository is ready for database operations. @@ -28,7 +27,6 @@ class ManifestRepository: # endregion ManifestRepository.__init__ # region ManifestRepository.save [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Persist a DistributionManifest to the database. # @PRE: manifest is a valid DistributionManifest instance with required fields populated. # @POST: Manifest is committed to database and refreshed with generated ID. @@ -43,7 +41,6 @@ class ManifestRepository: # endregion ManifestRepository.save # region ManifestRepository.get_by_id [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Retrieve a single DistributionManifest by its primary key. # @PRE: manifest_id is a valid string identifier. # @POST: Returns DistributionManifest if found, None otherwise. @@ -56,7 +53,6 @@ class ManifestRepository: # endregion ManifestRepository.get_by_id # region ManifestRepository.get_latest_for_candidate [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Retrieve the most recent manifest version for a given candidate. # @PRE: candidate_id is a valid string identifier. # @POST: Returns the highest manifest_version manifest for the candidate, or None. @@ -72,7 +68,6 @@ class ManifestRepository: # endregion ManifestRepository.get_latest_for_candidate # region ManifestRepository.list_by_candidate [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: List all manifests for a specific candidate, ordered by version. # @PRE: candidate_id is a valid string identifier. # @POST: Returns a list of DistributionManifest instances (may be empty). diff --git a/backend/src/services/dataset_review/clarification_engine.py b/backend/src/services/dataset_review/clarification_engine.py index bc1161db..3ba3369d 100644 --- a/backend/src/services/dataset_review/clarification_engine.py +++ b/backend/src/services/dataset_review/clarification_engine.py @@ -105,7 +105,6 @@ class ClarificationAnswerCommand: # @SIDE_EFFECT: Mutates clarification answers, session flags, and related clarification findings. class ClarificationEngine: # region ClarificationEngine_init [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Bind repository dependency for clarification persistence operations. def __init__(self, repository: DatasetReviewSessionRepository) -> None: self.repository = repository @@ -113,7 +112,6 @@ class ClarificationEngine: # endregion ClarificationEngine_init # region build_question_payload [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Return the one active highest-priority clarification question payload. # @PRE: Session contains unresolved clarification state or a resumable clarification session. # @POST: Returns exactly one active/open question payload or None when no unresolved question remains. @@ -173,7 +171,6 @@ class ClarificationEngine: # endregion build_question_payload # region record_answer [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Persist one clarification answer before any pointer/readiness mutation. # @PRE: Target question belongs to the session's active clarification session and is still open. # @POST: Answer row is persisted before current-question pointer advances. @@ -260,7 +257,6 @@ class ClarificationEngine: # endregion record_answer # region summarize_progress [TYPE Function] - # @COMPLEXITY: 1 # @PURPOSE: Produce a compact progress summary for pause/resume and completion UX. def summarize_progress(self, clarification_session: ClarificationSession) -> str: resolved = count_resolved_questions(clarification_session) @@ -270,7 +266,6 @@ class ClarificationEngine: # endregion summarize_progress # region _get_latest_clarification_session [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Select the latest clarification session for the current dataset review aggregate. def _get_latest_clarification_session(self, session: DatasetReviewSession) -> Optional[ClarificationSession]: if not session.clarification_sessions: @@ -281,7 +276,6 @@ class ClarificationEngine: # endregion _get_latest_clarification_session # region _find_question [TYPE Function] - # @COMPLEXITY: 1 # @PURPOSE: Resolve a clarification question from the active clarification aggregate. def _find_question(self, clarification_session: ClarificationSession, question_id: str) -> Optional[ClarificationQuestion]: for q in clarification_session.questions: diff --git a/backend/src/services/dataset_review/event_logger.py b/backend/src/services/dataset_review/event_logger.py index 73eb8067..dca1ae69 100644 --- a/backend/src/services/dataset_review/event_logger.py +++ b/backend/src/services/dataset_review/event_logger.py @@ -45,14 +45,12 @@ class SessionEventPayload: # @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent] class SessionEventLogger: # region SessionEventLogger_init [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Bind a live SQLAlchemy session to the session-event logger. def __init__(self, db: Session) -> None: self.db = db # endregion SessionEventLogger_init # region log_event [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Persist one explicit session event row for an owned dataset-review mutation. # @RELATION: [DEPENDS_ON] ->[SessionEvent] # @PRE: session_id, actor_user_id, event_type, and event_summary are non-empty. @@ -125,7 +123,6 @@ class SessionEventLogger: # endregion log_event # region log_for_session [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Convenience wrapper for logging an event directly from a session aggregate root. # @RELATION: [CALLS] ->[SessionEventLogger.log_event] def log_for_session( diff --git a/backend/src/services/dataset_review/orchestrator.py b/backend/src/services/dataset_review/orchestrator.py index e4838efc..f2d274aa 100644 --- a/backend/src/services/dataset_review/orchestrator.py +++ b/backend/src/services/dataset_review/orchestrator.py @@ -105,7 +105,6 @@ logger = cast(Any, logger) # @INVARIANT: session ownership is preserved on every mutation and recovery remains explicit when partial. class DatasetReviewOrchestrator: # region DatasetReviewOrchestrator_init [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Bind repository, config, and task dependencies required by the orchestration boundary. # @PRE: repository/config_manager are valid collaborators for the current request scope. # @POST: Instance holds collaborator references used by start/preview/launch orchestration methods. @@ -124,7 +123,6 @@ class DatasetReviewOrchestrator: # endregion DatasetReviewOrchestrator_init # region start_session [TYPE Function] - # @COMPLEXITY: 5 # @PURPOSE: Initialize a new session from a Superset link or dataset selection and trigger context recovery. # @RELATION: CALLS -> [SupersetContextExtractor.parse_superset_link] # @RELATION: CALLS -> [TaskManager.create_task] @@ -279,7 +277,6 @@ class DatasetReviewOrchestrator: # endregion start_session # region prepare_launch_preview [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Assemble effective execution inputs and trigger Superset-side preview compilation. # @RELATION: CALLS -> [SupersetCompilationAdapter.compile_preview] # @PRE: all required variables have candidate values or explicitly accepted defaults. @@ -359,7 +356,6 @@ class DatasetReviewOrchestrator: # endregion prepare_launch_preview # region launch_dataset [TYPE Function] - # @COMPLEXITY: 5 # @PURPOSE: Start the approved dataset execution through SQL Lab and persist run context for audit/replay. # @RELATION: CALLS -> [SupersetCompilationAdapter.create_sql_lab_session] # @PRE: session is run-ready and compiled preview is current. @@ -459,7 +455,6 @@ class DatasetReviewOrchestrator: # endregion launch_dataset # region _build_recovery_bootstrap [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Recover and materialize initial imported filters, template variables, and draft execution mappings after session creation. # @PRE: session belongs to the just-created review aggregate and parsed_context was produced for the same environment scope. # @POST: Returns bootstrap imported filters, template variables, execution mappings, and updated findings without persisting them directly. @@ -559,7 +554,6 @@ class DatasetReviewOrchestrator: # endregion _build_recovery_bootstrap # region _enqueue_recovery_task [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Link session start to observable async recovery when task infrastructure is available. # @PRE: session is already persisted. # @POST: returns task identifier when a task could be enqueued, otherwise None. diff --git a/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py b/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py index 29d888ac..19ad57fe 100644 --- a/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py +++ b/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py @@ -28,14 +28,12 @@ from src.services.dataset_review.repositories.session_repository import ( # region SessionRepositoryTests [TYPE Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 2 # @PURPOSE: Unit tests for DatasetReviewSessionRepository. @pytest.fixture def db_session(): # region db_session [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Build isolated in-memory SQLAlchemy session seeded with baseline user/environment rows. # @RELATION: BINDS_TO -> [SessionRepositoryTests] engine = create_engine("sqlite:///:memory:") diff --git a/backend/src/services/dataset_review/repositories/session_repository.py b/backend/src/services/dataset_review/repositories/session_repository.py index b4b41dcd..f05c9ce5 100644 --- a/backend/src/services/dataset_review/repositories/session_repository.py +++ b/backend/src/services/dataset_review/repositories/session_repository.py @@ -64,7 +64,6 @@ class DatasetReviewSessionVersionConflictError(ValueError): # @SIDE_EFFECT: mutates and queries the persistence layer through the injected database session. class DatasetReviewSessionRepository: # region init_repo [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Bind one live SQLAlchemy session to the repository instance. # @PRE: db_session is not None # @POST: Repository instance initialized with valid session @@ -75,7 +74,6 @@ class DatasetReviewSessionRepository: # endregion init_repo # region get_owned_session [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Resolve one owner-scoped dataset review session for mutation paths. # @PRE: session_id and user_id are non-empty identifiers from the authenticated ownership scope. # @POST: returns the owned session or raises a deterministic access error. @@ -96,7 +94,6 @@ class DatasetReviewSessionRepository: # endregion get_owned_session # region create_sess [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Persist an initial dataset review session shell. # @POST: session is committed, refreshed, and returned with persisted identifiers. def create_session(self, session: DatasetReviewSession) -> DatasetReviewSession: @@ -111,7 +108,6 @@ class DatasetReviewSessionRepository: # endregion create_sess # region require_session_version [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Enforce optimistic-lock version matching before a session mutation is persisted. # @POST: returns the same session when versions match; otherwise raises deterministic conflict error. def require_session_version(self, session: DatasetReviewSession, expected_version: int) -> DatasetReviewSession: @@ -127,7 +123,6 @@ class DatasetReviewSessionRepository: # endregion require_session_version # region bump_session_version [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Increment optimistic-lock version after a successful session mutation is assembled. # @POST: session version increments monotonically. def bump_session_version(self, session: DatasetReviewSession) -> int: @@ -141,7 +136,6 @@ class DatasetReviewSessionRepository: # endregion bump_session_version # region commit_session_mutation [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Commit one prepared session mutation and translate stale writes into deterministic conflicts. # @POST: session mutation is committed with one version increment or a deterministic conflict error is raised. def commit_session_mutation( @@ -168,7 +162,6 @@ class DatasetReviewSessionRepository: # endregion commit_session_mutation # region load_detail [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Return the full session aggregate for API and frontend resume flows. # @POST: Returns SessionDetail with all fields populated or None. def load_session_detail(self, session_id: str, user_id: str) -> Optional[DatasetReviewSession]: @@ -202,7 +195,6 @@ class DatasetReviewSessionRepository: # endregion load_detail # region save_profile_and_findings [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Persist profile state and replace validation findings for an owned session. # @POST: stored profile matches the current session and findings are replaced. def save_profile_and_findings( @@ -217,7 +209,6 @@ class DatasetReviewSessionRepository: # endregion save_profile_and_findings # region save_recovery_state [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Persist imported filters, template variables, and initial execution mappings. def save_recovery_state( self, session_id: str, user_id: str, imported_filters: List[ImportedFilter], @@ -234,7 +225,6 @@ class DatasetReviewSessionRepository: # endregion save_recovery_state # region save_preview [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Persist a preview snapshot and mark prior session previews stale. def save_preview( self, session_id: str, user_id: str, preview: CompiledPreview, expected_version: Optional[int] = None, @@ -248,7 +238,6 @@ class DatasetReviewSessionRepository: # endregion save_preview # region save_run_context [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Persist an immutable launch audit snapshot for an owned session. def save_run_context( self, session_id: str, user_id: str, run_context: DatasetRunContext, expected_version: Optional[int] = None, @@ -262,7 +251,6 @@ class DatasetReviewSessionRepository: # endregion save_run_context # region list_user_sess [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: List review sessions owned by a specific user ordered by most recent update. def list_sessions_for_user(self, user_id: str) -> List[DatasetReviewSession]: with belief_scope("DatasetReviewSessionRepository.list_sessions_for_user"): diff --git a/backend/src/services/dataset_review/semantic_resolver.py b/backend/src/services/dataset_review/semantic_resolver.py index 952a49d2..a490fd41 100644 --- a/backend/src/services/dataset_review/semantic_resolver.py +++ b/backend/src/services/dataset_review/semantic_resolver.py @@ -47,14 +47,12 @@ class DictionaryResolutionResult: # @SIDE_EFFECT: emits semantic trace logs for ranking and fallback decisions. class SemanticSourceResolver: # region resolve_from_file [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Normalize uploaded semantic file records into field-level candidates. def resolve_from_file(self, source_payload: Mapping[str, Any], fields: Iterable[Mapping[str, Any]]) -> DictionaryResolutionResult: return DictionaryResolutionResult(source_ref=str(source_payload.get("source_ref") or "uploaded_file")) # endregion resolve_from_file # region resolve_from_dictionary [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Resolve candidates from connected tabular dictionary sources. # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry] # @RELATION: [DEPENDS_ON] ->[SemanticCandidate] @@ -212,7 +210,6 @@ class SemanticSourceResolver: # endregion resolve_from_dictionary # region resolve_from_reference_dataset [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Reuse semantic metadata from trusted Superset datasets. def resolve_from_reference_dataset( self, @@ -223,7 +220,6 @@ class SemanticSourceResolver: # endregion resolve_from_reference_dataset # region rank_candidates [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Apply confidence ordering and determine best candidate per field. # @RELATION: [DEPENDS_ON] ->[SemanticCandidate] def rank_candidates(self, candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]: @@ -241,14 +237,12 @@ class SemanticSourceResolver: # endregion rank_candidates # region detect_conflicts [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Mark competing candidate sets that require explicit user review. def detect_conflicts(self, candidates: List[Dict[str, Any]]) -> bool: return len(candidates) > 1 # endregion detect_conflicts # region apply_field_decision [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Accept, reject, or manually override a field-level semantic value. def apply_field_decision(self, field_state: Mapping[str, Any], decision: Mapping[str, Any]) -> Dict[str, Any]: merged = dict(field_state) @@ -257,7 +251,6 @@ class SemanticSourceResolver: # endregion apply_field_decision # region propagate_source_version_update [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Propagate a semantic source version change to unlocked field entries without silently overwriting manual or locked values. # @RELATION: [DEPENDS_ON] ->[SemanticSource] # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry] @@ -314,7 +307,6 @@ class SemanticSourceResolver: # endregion propagate_source_version_update # region _normalize_dictionary_row [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Normalize one dictionary row into a consistent lookup structure. def _normalize_dictionary_row(self, row: Mapping[str, Any]) -> Dict[str, Any]: field_name = ( @@ -334,7 +326,6 @@ class SemanticSourceResolver: # endregion _normalize_dictionary_row # region _find_fuzzy_matches [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Produce confidence-scored fuzzy matches while keeping them reviewable. def _find_fuzzy_matches(self, field_name: str, rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: normalized_target = self._normalize_key(field_name) @@ -352,7 +343,6 @@ class SemanticSourceResolver: # endregion _find_fuzzy_matches # region _build_candidate_payload [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Project normalized dictionary rows into semantic candidate payloads. def _build_candidate_payload( self, @@ -373,7 +363,6 @@ class SemanticSourceResolver: # endregion _build_candidate_payload # region _match_priority [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Encode trusted-confidence ordering so exact dictionary reuse beats fuzzy invention. def _match_priority(self, match_type: Optional[str]) -> int: priority = { @@ -386,7 +375,6 @@ class SemanticSourceResolver: # endregion _match_priority # region _normalize_key [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Normalize field identifiers for stable exact/fuzzy comparisons. def _normalize_key(self, value: str) -> str: return "".join(ch for ch in str(value or "").strip().lower() if ch.isalnum() or ch == "_") diff --git a/backend/src/services/git/_status.py b/backend/src/services/git/_status.py index 2e3558e7..d8a4e499 100644 --- a/backend/src/services/git/_status.py +++ b/backend/src/services/git/_status.py @@ -13,7 +13,6 @@ from src.core.logger import logger, belief_scope # @BRIEF Mixin providing repository status, diff, and commit history for GitService. class GitServiceStatusMixin: # region _parse_status_porcelain [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Parse git status --porcelain output into staged, modified, and untracked file lists. # @PRE: `repo` is an open GitPython Repo instance. # @POST: Returns (staged, modified, untracked) tuple of file path lists. diff --git a/backend/src/services/git_service.py b/backend/src/services/git_service.py index 373d5033..ba36b9cd 100644 --- a/backend/src/services/git_service.py +++ b/backend/src/services/git_service.py @@ -1,5 +1,4 @@ # #region git_service [C:1] [TYPE Module:Tombstone] [SEMANTICS git, service, shim, re-export, decomissioned] -# @COMPLEXITY: 1 # @BRIEF Re-export shim — GitService has been decomposed into services/git/ package. # All consumers continue to import from this same path without changes. # @RELATION REDIRECTS_TO -> [GitServiceModule] diff --git a/backend/src/services/health_service.py b/backend/src/services/health_service.py index bf37e79b..cb5f80fd 100644 --- a/backend/src/services/health_service.py +++ b/backend/src/services/health_service.py @@ -46,7 +46,6 @@ class HealthService: """ # region HealthService_init [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Initialize health service with DB session and optional config access for dashboard metadata resolution. # @PRE: db is a valid SQLAlchemy session. # @POST: Service is ready to aggregate summaries and delete health reports. @@ -61,7 +60,6 @@ class HealthService: # endregion HealthService_init # region _prime_dashboard_meta_cache [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Warm dashboard slug/title cache with one Superset list fetch per environment. # @PRE: records may contain mixed numeric and slug dashboard identifiers. # @POST: Numeric dashboard ids for known environments are cached when discoverable. @@ -153,7 +151,6 @@ class HealthService: # endregion _prime_dashboard_meta_cache # region _resolve_dashboard_meta [TYPE Function] - # @COMPLEXITY: 1 # @PURPOSE: Resolve slug/title for a dashboard referenced by persisted validation record. # @PRE: dashboard_id may be numeric or slug-like; environment_id may be empty. # @POST: Returns dict with `slug` and `title` keys, using cache when possible. @@ -184,7 +181,6 @@ class HealthService: # endregion _resolve_dashboard_meta # region get_health_summary [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Aggregate latest validation status per dashboard and enrich rows with dashboard slug/title. # @PRE: environment_id may be omitted to aggregate across all environments. # @POST: Returns HealthSummaryResponse with counts and latest record row per dashboard. @@ -288,7 +284,6 @@ class HealthService: # endregion get_health_summary # region delete_validation_report [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Delete one persisted health report and optionally clean linked task/log artifacts. # @PRE: record_id is a validation record identifier. # @POST: Returns True only when a matching record was deleted. diff --git a/backend/src/services/llm_provider.py b/backend/src/services/llm_provider.py index 896c8035..28fba6a0 100644 --- a/backend/src/services/llm_provider.py +++ b/backend/src/services/llm_provider.py @@ -123,7 +123,6 @@ class LLMProviderService: # endregion LLMProviderService_init # region get_all_providers [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Returns all configured LLM providers. # @PRE: Database connection must be active. # @POST: Returns list of all LLMProvider records. @@ -135,7 +134,6 @@ class LLMProviderService: # endregion get_all_providers # region get_provider [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Returns a single LLM provider by ID. # @PRE: provider_id must be a valid string. # @POST: Returns LLMProvider or None if not found. @@ -149,7 +147,6 @@ class LLMProviderService: # endregion get_provider # region create_provider [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Creates a new LLM provider with encrypted API key. # @PRE: config must contain valid provider configuration. # @POST: New provider created and persisted to database. @@ -175,7 +172,6 @@ class LLMProviderService: # endregion create_provider # region update_provider [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Updates an existing LLM provider. # @PRE: provider_id must exist, config must be valid. # @POST: Provider updated and persisted to database. @@ -210,7 +206,6 @@ class LLMProviderService: # endregion update_provider # region delete_provider [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Deletes an LLM provider. # @PRE: provider_id must exist. # @POST: Provider removed from database. @@ -227,7 +222,6 @@ class LLMProviderService: # endregion delete_provider # region get_decrypted_api_key [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Returns the decrypted API key for a provider. # @PRE: provider_id must exist with valid encrypted key. # @POST: Returns decrypted API key or None on failure. diff --git a/backend/src/services/mapping_service.py b/backend/src/services/mapping_service.py index 5d8302c3..8d466482 100644 --- a/backend/src/services/mapping_service.py +++ b/backend/src/services/mapping_service.py @@ -28,7 +28,6 @@ from ..core.utils.matching import suggest_mappings class MappingService: # region init [TYPE Function] # @PURPOSE: Initializes the mapping service with a config manager. - # @COMPLEXITY: 3 # @PRE: config_manager is provided. # @PARAM: config_manager (ConfigManager) - The configuration manager. # @POST: Service is initialized. @@ -41,7 +40,6 @@ class MappingService: # region _get_client [TYPE Function] # @PURPOSE: Helper to get an initialized SupersetClient for an environment. - # @COMPLEXITY: 3 # @PARAM: env_id (str) - The ID of the environment. # @PRE: environment must exist in config. # @POST: Returns an initialized SupersetClient. @@ -60,7 +58,6 @@ class MappingService: # region get_suggestions [TYPE Function] # @PURPOSE: Fetches databases from both environments and returns fuzzy matching suggestions. - # @COMPLEXITY: 3 # @PARAM: source_env_id (str) - Source environment ID. # @PARAM: target_env_id (str) - Target environment ID. # @PRE: Both environments must be accessible. diff --git a/backend/src/services/notifications/__tests__/test_notification_service.py b/backend/src/services/notifications/__tests__/test_notification_service.py index ad3f9e48..26658581 100644 --- a/backend/src/services/notifications/__tests__/test_notification_service.py +++ b/backend/src/services/notifications/__tests__/test_notification_service.py @@ -1,5 +1,4 @@ # region test_notification_service [TYPE Module] -# @COMPLEXITY: 2 # @PURPOSE: Unit tests for NotificationService routing and dispatch logic. # @RELATION: TESTS ->[NotificationService:Class] diff --git a/backend/src/services/notifications/service.py b/backend/src/services/notifications/service.py index d60259e1..737d9a8f 100644 --- a/backend/src/services/notifications/service.py +++ b/backend/src/services/notifications/service.py @@ -44,7 +44,6 @@ from .providers import ( # @DATA_CONTRACT: Input[ValidationRecord, Optional[ValidationPolicy], Optional[BackgroundTasks]] -> Output[None] class NotificationService: # region NotificationService_init [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Bind DB and configuration collaborators used for provider initialization and routing. # @RELATION: [BINDS_TO] ->[NotificationService] # @RELATION: [DEPENDS_ON] ->[ValidationPolicy] @@ -57,7 +56,6 @@ class NotificationService: # endregion NotificationService_init # region _initialize_providers [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Materialize configured notification channel adapters once per service lifetime. # @RELATION: [DEPENDS_ON] ->[SMTPProvider] # @RELATION: [DEPENDS_ON] ->[TelegramProvider] @@ -83,7 +81,6 @@ class NotificationService: # endregion _initialize_providers # region dispatch_report [TYPE Function] - # @COMPLEXITY: 4 # @PURPOSE: Route one validation record to resolved owners and configured custom channels. # @RELATION: [CALLS] ->[_initialize_providers] # @RELATION: [CALLS] ->[_should_notify] @@ -143,7 +140,6 @@ class NotificationService: # endregion dispatch_report # region _should_notify [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Evaluate record status against effective alert policy. # @RELATION: [DEPENDS_ON] ->[ValidationRecord] # @RELATION: [DEPENDS_ON] ->[ValidationPolicy] @@ -161,7 +157,6 @@ class NotificationService: # endregion _should_notify # region _resolve_targets [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Resolve owner and policy-defined delivery targets for one validation record. # @RELATION: [CALLS] ->[_find_dashboard_owners] # @RELATION: [DEPENDS_ON] ->[ValidationRecord] @@ -198,7 +193,6 @@ class NotificationService: # endregion _resolve_targets # region _find_dashboard_owners [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Load candidate dashboard owners from persisted profile preferences. # @RELATION: [DEPENDS_ON] ->[ValidationRecord] # @RELATION: [DEPENDS_ON] ->[UserDashboardPreference] @@ -220,7 +214,6 @@ class NotificationService: # endregion _find_dashboard_owners # region _build_body [TYPE Function] - # @COMPLEXITY: 2 # @PURPOSE: Format one validation record into provider-ready body text. # @RELATION: [DEPENDS_ON] ->[ValidationRecord] def _build_body(self, record: ValidationRecord) -> str: diff --git a/backend/src/services/reports/__tests__/test_report_normalizer.py b/backend/src/services/reports/__tests__/test_report_normalizer.py index 1ed7e5e1..3a121138 100644 --- a/backend/src/services/reports/__tests__/test_report_normalizer.py +++ b/backend/src/services/reports/__tests__/test_report_normalizer.py @@ -1,5 +1,4 @@ # region test_report_normalizer [TYPE Module] -# @COMPLEXITY: 2 # @SEMANTICS: tests, reports, normalizer, fallback # @PURPOSE: Validate unknown task type fallback and partial payload normalization behavior. # @RELATION: TESTS ->[normalize_report:Function] diff --git a/backend/src/services/reports/__tests__/test_report_service.py b/backend/src/services/reports/__tests__/test_report_service.py index cbec27fa..90e81a2b 100644 --- a/backend/src/services/reports/__tests__/test_report_service.py +++ b/backend/src/services/reports/__tests__/test_report_service.py @@ -1,5 +1,4 @@ # region test_report_service [TYPE Module] -# @COMPLEXITY: 2 # @PURPOSE: Unit tests for ReportsService list/detail operations # @RELATION: TESTS ->[ReportsService:Class] # @LAYER: Domain diff --git a/backend/src/services/reports/report_service.py b/backend/src/services/reports/report_service.py index 6a9458b4..5e8e70ae 100644 --- a/backend/src/services/reports/report_service.py +++ b/backend/src/services/reports/report_service.py @@ -57,7 +57,6 @@ from .normalizer import normalize_task_report # @TEST_INVARIANT: consistent_pagination -> verifies: [valid_service] class ReportsService: # region init [TYPE Function] - # @COMPLEXITY: 5 # @PURPOSE: Initialize service with TaskManager dependency. # @PRE: task_manager is a live TaskManager instance. # @POST: self.task_manager is assigned and ready for read operations. diff --git a/backend/src/services/resource_service.py b/backend/src/services/resource_service.py index b646c13a..058e01fe 100644 --- a/backend/src/services/resource_service.py +++ b/backend/src/services/resource_service.py @@ -21,7 +21,6 @@ from ..core.logger import logger, belief_scope class ResourceService: # region ResourceService_init [TYPE Function] - # @COMPLEXITY: 1 # @PURPOSE: Initialize the resource service with dependencies # @PRE: None # @POST: ResourceService is ready to fetch resources @@ -32,7 +31,6 @@ class ResourceService: # endregion ResourceService_init # region get_dashboards_with_status [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Fetch dashboards from environment with Git status and last task status # @PRE: env is a valid Environment object # @POST: Returns list of dashboards with enhanced metadata @@ -82,7 +80,6 @@ class ResourceService: # endregion get_dashboards_with_status # region get_dashboards_page_with_status [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Fetch one dashboard page from environment and enrich only that page with status metadata. # @PRE: env is valid; page >= 1; page_size > 0. # @POST: Returns page items plus total counters without scanning all pages locally. @@ -149,7 +146,6 @@ class ResourceService: # endregion get_dashboards_page_with_status # region _get_last_llm_task_for_dashboard [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Get most recent LLM validation task for a dashboard in an environment # @PRE: dashboard_id is a valid integer identifier # @POST: Returns the newest llm_dashboard_validation task summary or None @@ -233,7 +229,6 @@ class ResourceService: # endregion _get_last_llm_task_for_dashboard # region _normalize_task_status [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Normalize task status to stable uppercase values for UI/API projections # @PRE: raw_status can be enum or string # @POST: Returns uppercase status without enum class prefix @@ -251,7 +246,6 @@ class ResourceService: # endregion _normalize_task_status # region _normalize_validation_status [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Normalize LLM validation status to PASS/FAIL/WARN/UNKNOWN # @PRE: raw_status can be any scalar type # @POST: Returns normalized validation status token or None @@ -268,7 +262,6 @@ class ResourceService: # endregion _normalize_validation_status # region _normalize_datetime_for_compare [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Normalize datetime values to UTC-aware values for safe comparisons. # @PRE: value may be datetime or any scalar. # @POST: Returns UTC-aware datetime; non-datetime values map to minimal UTC datetime. @@ -285,7 +278,6 @@ class ResourceService: # endregion _normalize_datetime_for_compare # region get_datasets_with_status [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Fetch datasets from environment with mapping progress and last task status # @PRE: env is a valid Environment object # @POST: Returns list of datasets with enhanced metadata @@ -324,7 +316,6 @@ class ResourceService: # endregion get_datasets_with_status # region get_activity_summary [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Get summary of active and recent tasks for the activity indicator # @PRE: tasks is a list of Task objects # @POST: Returns summary with active_count and recent_tasks @@ -366,7 +357,6 @@ class ResourceService: # endregion get_activity_summary # region _get_git_status_for_dashboard [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Get Git sync status for a dashboard # @PRE: dashboard_id is a valid integer # @POST: Returns git status or None if no repo exists @@ -426,7 +416,6 @@ class ResourceService: # endregion _get_git_status_for_dashboard # region _get_last_task_for_resource [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Get the most recent task for a specific resource # @PRE: resource_id is a valid string # @POST: Returns task summary or None if no tasks found @@ -465,7 +454,6 @@ class ResourceService: # endregion _get_last_task_for_resource # region _extract_resource_name_from_task [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Extract resource name from task params # @PRE: task is a valid Task object # @POST: Returns resource name or task ID @@ -478,7 +466,6 @@ class ResourceService: # endregion _extract_resource_name_from_task # region _extract_resource_type_from_task [TYPE Function] - # @COMPLEXITY: 3 # @PURPOSE: Extract resource type from task params # @PRE: task is a valid Task object # @POST: Returns resource type or 'unknown' diff --git a/backend/tests/core/migration/test_archive_parser.py b/backend/tests/core/migration/test_archive_parser.py index e2ee4fc9..7ca21bf6 100644 --- a/backend/tests/core/migration/test_archive_parser.py +++ b/backend/tests/core/migration/test_archive_parser.py @@ -1,6 +1,5 @@ # [DEF:TestArchiveParser:Module] # -# @COMPLEXITY: 3 # @PURPOSE: Unit tests for MigrationArchiveParser ZIP extraction contract. # @LAYER: Domain # @RELATION: DEPENDS_ON -> [MigrationArchiveParserModule] diff --git a/backend/tests/core/migration/test_dry_run_orchestrator.py b/backend/tests/core/migration/test_dry_run_orchestrator.py index 9a0ac168..cc3d3de3 100644 --- a/backend/tests/core/migration/test_dry_run_orchestrator.py +++ b/backend/tests/core/migration/test_dry_run_orchestrator.py @@ -1,6 +1,5 @@ # [DEF:TestDryRunOrchestrator:Module] # -# @COMPLEXITY: 3 # @PURPOSE: Unit tests for MigrationDryRunService diff and risk computation contracts. # @LAYER: Domain # @RELATION: DEPENDS_ON -> [MigrationDryRunOrchestratorModule] diff --git a/backend/tests/core/test_git_service_gitea_pr.py b/backend/tests/core/test_git_service_gitea_pr.py index 1e38883c..4387cf17 100644 --- a/backend/tests/core/test_git_service_gitea_pr.py +++ b/backend/tests/core/test_git_service_gitea_pr.py @@ -1,6 +1,5 @@ # [DEF:TestGitServiceGiteaPr:Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 3 # @SEMANTICS: tests, git, gitea, pull_request, fallback # @PURPOSE: Validate Gitea PR creation fallback behavior when configured server URL is stale. # @LAYER: Domain diff --git a/backend/tests/core/test_mapping_service.py b/backend/tests/core/test_mapping_service.py index 921c1c5a..30d85669 100644 --- a/backend/tests/core/test_mapping_service.py +++ b/backend/tests/core/test_mapping_service.py @@ -1,6 +1,5 @@ # [DEF:TestMappingService:Module] # -# @COMPLEXITY: 3 # @PURPOSE: Unit tests for the IdMappingService matching UUIDs to integer IDs. # @LAYER: Domain # @RELATION: VERIFIES ->[src.core.mapping_service.IdMappingService] diff --git a/backend/tests/core/test_migration_engine.py b/backend/tests/core/test_migration_engine.py index 4c3b94f3..e35d222b 100644 --- a/backend/tests/core/test_migration_engine.py +++ b/backend/tests/core/test_migration_engine.py @@ -1,6 +1,5 @@ # [DEF:TestMigrationEngine:Module] # -# @COMPLEXITY: 3 # @PURPOSE: Unit tests for MigrationEngine's cross-filter patching algorithms. # @LAYER: Domain # @RELATION: VERIFIES -> [src.core.migration_engine:Module] @@ -29,7 +28,6 @@ from src.models.mapping import ResourceType # [DEF:MockMappingService:Class] # @RELATION: BINDS_TO -> [TestMigrationEngine:Module] -# @COMPLEXITY: 2 # @PURPOSE: Deterministic mapping service double for native filter ID remapping scenarios. # @INVARIANT: Returns mappings only for requested UUID keys present in seeded map. class MockMappingService: diff --git a/backend/tests/scripts/test_clean_release_cli.py b/backend/tests/scripts/test_clean_release_cli.py index db4d4f39..ed58a2f6 100644 --- a/backend/tests/scripts/test_clean_release_cli.py +++ b/backend/tests/scripts/test_clean_release_cli.py @@ -1,6 +1,5 @@ # [DEF:test_clean_release_cli:Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 3 # @PURPOSE: Smoke tests for the redesigned clean release CLI. # @LAYER: Domain diff --git a/backend/tests/scripts/test_clean_release_tui.py b/backend/tests/scripts/test_clean_release_tui.py index 59ed57ef..17637cc1 100644 --- a/backend/tests/scripts/test_clean_release_tui.py +++ b/backend/tests/scripts/test_clean_release_tui.py @@ -1,6 +1,5 @@ # [DEF:TestCleanReleaseTui:Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 3 # @SEMANTICS: tests, tui, clean-release, curses # @PURPOSE: Unit tests for the interactive curses TUI of the clean release process. # @LAYER: Scripts diff --git a/backend/tests/scripts/test_clean_release_tui_v2.py b/backend/tests/scripts/test_clean_release_tui_v2.py index 73b75c26..abe5610b 100644 --- a/backend/tests/scripts/test_clean_release_tui_v2.py +++ b/backend/tests/scripts/test_clean_release_tui_v2.py @@ -1,6 +1,5 @@ # [DEF:test_clean_release_tui_v2:Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 3 # @PURPOSE: Smoke tests for thin-client TUI action dispatch and blocked transition behavior. # @LAYER: Domain diff --git a/backend/tests/services/clean_release/test_approval_service.py b/backend/tests/services/clean_release/test_approval_service.py index 2ff1d048..f4d7afb5 100644 --- a/backend/tests/services/clean_release/test_approval_service.py +++ b/backend/tests/services/clean_release/test_approval_service.py @@ -1,6 +1,5 @@ # [DEF:TestApprovalService:Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 5 # @SEMANTICS: tests, clean-release, approval, lifecycle, gate # @PURPOSE: Define approval gate contracts for approve/reject operations over immutable compliance evidence. # @LAYER: Tests diff --git a/backend/tests/services/clean_release/test_candidate_manifest_services.py b/backend/tests/services/clean_release/test_candidate_manifest_services.py index dfdb9535..7de8b969 100644 --- a/backend/tests/services/clean_release/test_candidate_manifest_services.py +++ b/backend/tests/services/clean_release/test_candidate_manifest_services.py @@ -1,6 +1,5 @@ # [DEF:test_candidate_manifest_services:Module] # @RELATION: BELONGS_TO -> [SrcRoot:Module] -# @COMPLEXITY: 3 # @PURPOSE: Test lifecycle and manifest versioning for release candidates. # @LAYER: Tests diff --git a/backend/tests/services/clean_release/test_compliance_execution_service.py b/backend/tests/services/clean_release/test_compliance_execution_service.py index 1a9a8f49..77c1c4cb 100644 --- a/backend/tests/services/clean_release/test_compliance_execution_service.py +++ b/backend/tests/services/clean_release/test_compliance_execution_service.py @@ -1,6 +1,5 @@ # [DEF:TestComplianceExecutionService:Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 5 # @SEMANTICS: tests, clean-release, compliance, pipeline, run-finalization # @PURPOSE: Validate stage pipeline and run finalization contracts for compliance execution. # @LAYER: Tests diff --git a/backend/tests/services/clean_release/test_compliance_task_integration.py b/backend/tests/services/clean_release/test_compliance_task_integration.py index 2ff5fdc7..7325b5d2 100644 --- a/backend/tests/services/clean_release/test_compliance_task_integration.py +++ b/backend/tests/services/clean_release/test_compliance_task_integration.py @@ -1,6 +1,5 @@ # [DEF:TestComplianceTaskIntegration:Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 5 # @SEMANTICS: tests, clean-release, compliance, task-manager, integration # @PURPOSE: Verify clean release compliance runs execute through TaskManager lifecycle with observable success/failure outcomes. # @LAYER: Tests @@ -144,7 +143,6 @@ class CleanReleaseCompliancePlugin: # [DEF:_PluginLoaderStub:Class] # @RELATION: BINDS_TO -> TestComplianceTaskIntegration -# @COMPLEXITY: 2 # @PURPOSE: Provide minimal plugin loader contract used by TaskManager in integration tests. # @INVARIANT: has_plugin/get_plugin only acknowledge the seeded compliance plugin id. class _PluginLoaderStub: diff --git a/backend/tests/services/clean_release/test_demo_mode_isolation.py b/backend/tests/services/clean_release/test_demo_mode_isolation.py index 47ca617a..b87a5b56 100644 --- a/backend/tests/services/clean_release/test_demo_mode_isolation.py +++ b/backend/tests/services/clean_release/test_demo_mode_isolation.py @@ -1,5 +1,4 @@ # [DEF:TestDemoModeIsolation:Module] -# @COMPLEXITY: 3 # @SEMANTICS: clean-release, demo-mode, isolation, namespace, repository # @PURPOSE: Verify demo and real mode namespace isolation contracts before TUI integration. # @LAYER: Tests diff --git a/backend/tests/services/clean_release/test_policy_resolution_service.py b/backend/tests/services/clean_release/test_policy_resolution_service.py index 16c8e86c..5d32a5f3 100644 --- a/backend/tests/services/clean_release/test_policy_resolution_service.py +++ b/backend/tests/services/clean_release/test_policy_resolution_service.py @@ -1,5 +1,4 @@ # [DEF:TestPolicyResolutionService:Module] -# @COMPLEXITY: 5 # @SEMANTICS: clean-release, policy-resolution, trusted-snapshots, contracts # @PURPOSE: Verify trusted policy snapshot resolution contract and error guards. # @LAYER: Tests @@ -24,7 +23,6 @@ from src.services.clean_release.repository import CleanReleaseRepository # [DEF:_config_manager:Function] # @RELATION: BINDS_TO -> [TestPolicyResolutionService] -# @COMPLEXITY: 1 # @PURPOSE: Build deterministic ConfigManager-like stub for tests. # @INVARIANT: Only settings.clean_release.active_policy_id and active_registry_id are populated; any other settings field access raises AttributeError. # @PRE: policy_id and registry_id may be None or non-empty strings. diff --git a/backend/tests/services/clean_release/test_publication_service.py b/backend/tests/services/clean_release/test_publication_service.py index f07e69d1..315fa3a7 100644 --- a/backend/tests/services/clean_release/test_publication_service.py +++ b/backend/tests/services/clean_release/test_publication_service.py @@ -1,6 +1,5 @@ # [DEF:TestPublicationService:Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 5 # @SEMANTICS: tests, clean-release, publication, revoke, gate # @PURPOSE: Define publication gate contracts over approved candidates and immutable publication records. # @LAYER: Tests diff --git a/backend/tests/services/clean_release/test_report_audit_immutability.py b/backend/tests/services/clean_release/test_report_audit_immutability.py index f5d0a23d..7b563d8b 100644 --- a/backend/tests/services/clean_release/test_report_audit_immutability.py +++ b/backend/tests/services/clean_release/test_report_audit_immutability.py @@ -1,6 +1,5 @@ # [DEF:TestReportAuditImmutability:Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 5 # @SEMANTICS: tests, clean-release, report, audit, immutability, append-only # @PURPOSE: Validate report snapshot immutability expectations and append-only audit hook behavior for US2. # @LAYER: Tests diff --git a/backend/tests/services/dataset_review/test_superset_matrix.py b/backend/tests/services/dataset_review/test_superset_matrix.py index 115a1c43..a2358164 100644 --- a/backend/tests/services/dataset_review/test_superset_matrix.py +++ b/backend/tests/services/dataset_review/test_superset_matrix.py @@ -1,5 +1,4 @@ # [DEF:SupersetCompatibilityMatrixTests:Module] -# @COMPLEXITY: 3 # @SEMANTICS: dataset_review, superset, compatibility_matrix, preview, sql_lab, tests # @PURPOSE: Verifies Superset preview and SQL Lab endpoint fallback strategy used by dataset-review orchestration. # @LAYER: Tests @@ -21,7 +20,6 @@ from src.core.utils.superset_compilation_adapter import ( # [DEF:make_adapter:Function] -# @COMPLEXITY: 2 # @PURPOSE: Build an adapter with a mock Superset client and deterministic environment for compatibility tests. # @RELATION: [DEPENDS_ON] ->[SupersetCompilationAdapter] def make_adapter(): @@ -43,7 +41,6 @@ def make_adapter(): # [DEF:test_preview_prefers_supported_client_method_before_network_fallback:Function] -# @COMPLEXITY: 2 # @PURPOSE: Confirms preview compilation uses a supported client method first when the capability exists. # @RELATION: [DEPENDS_ON] ->[SupersetCompilationAdapter] def test_preview_prefers_supported_client_method_before_network_fallback(): @@ -69,7 +66,6 @@ def test_preview_prefers_supported_client_method_before_network_fallback(): # [DEF:test_preview_falls_back_across_matrix_until_supported_endpoint_returns_sql:Function] -# @COMPLEXITY: 3 # @PURPOSE: Confirms preview fallback walks the compatibility matrix from preferred to legacy endpoints until one returns compiled SQL. # @RELATION: [DEPENDS_ON] ->[SupersetCompilationAdapter] def test_preview_falls_back_across_matrix_until_supported_endpoint_returns_sql(): @@ -104,7 +100,6 @@ def test_preview_falls_back_across_matrix_until_supported_endpoint_returns_sql() # [DEF:test_sql_lab_launch_falls_back_to_legacy_execute_endpoint:Function] -# @COMPLEXITY: 3 # @PURPOSE: Confirms SQL Lab launch falls back from modern to legacy execute endpoint and preserves canonical session reference extraction. # @RELATION: [DEPENDS_ON] ->[SupersetCompilationAdapter] def test_sql_lab_launch_falls_back_to_legacy_execute_endpoint(): diff --git a/backend/tests/test_auth.py b/backend/tests/test_auth.py index 5773a62e..1cf0491c 100644 --- a/backend/tests/test_auth.py +++ b/backend/tests/test_auth.py @@ -1,5 +1,4 @@ # [DEF:TestAuth:Module] -# @COMPLEXITY: 3 # @PURPOSE: Covers authentication service/repository behavior and auth bootstrap helpers. # @LAYER: Test # @RELATION: TESTS -> AuthService diff --git a/backend/tests/test_dashboards_api.py b/backend/tests/test_dashboards_api.py index eb76149e..a53a7981 100644 --- a/backend/tests/test_dashboards_api.py +++ b/backend/tests/test_dashboards_api.py @@ -1,6 +1,5 @@ # [DEF:TestDashboardsApi:Module] # @RELATION: VERIFIES ->[src.api.routes.dashboards] -# @COMPLEXITY: 3 # @PURPOSE: Comprehensive contract-driven tests for Dashboard Hub API # @LAYER: Domain (Tests) # @SEMANTICS: tests, dashboards, api, contract, remediation @@ -561,7 +560,7 @@ def test_backup_dashboards_with_schedule(mock_deps): # --- 8. Internal logic: _task_matches_dashboard --- # [/DEF:test_backup_dashboards_with_schedule:Function] -from src.api.routes.dashboards import _task_matches_dashboard +from src.api.routes.dashboards._projection import _task_matches_dashboard # [DEF:test_task_matches_dashboard_logic:Function] diff --git a/backend/tests/test_log_persistence.py b/backend/tests/test_log_persistence.py index 557dbe00..a3f7e349 100644 --- a/backend/tests/test_log_persistence.py +++ b/backend/tests/test_log_persistence.py @@ -3,7 +3,6 @@ # @SEMANTICS: test, log, persistence, unit_test # @PURPOSE: Unit tests for TaskLogPersistenceService. # @LAYER: Test -# @COMPLEXITY: 5 # [SECTION: IMPORTS] from datetime import datetime @@ -20,7 +19,6 @@ from src.core.task_manager.models import LogEntry, LogFilter # [DEF:TestLogPersistence:Class] # @RELATION: BINDS_TO -> test_log_persistence # @PURPOSE: Test suite for TaskLogPersistenceService. -# @COMPLEXITY: 5 # @TEST_DATA: log_entry -> {"task_id": "test-task-1", "level": "INFO", "source": "test_source", "message": "Test message"} class TestLogPersistence: diff --git a/backend/tests/test_logger.py b/backend/tests/test_logger.py index e894870f..ce00cc5a 100644 --- a/backend/tests/test_logger.py +++ b/backend/tests/test_logger.py @@ -1,5 +1,4 @@ # [DEF:TestLogger:Module] -# @COMPLEXITY: 3 # @SEMANTICS: logging, tests, belief_state, cot, json # @PURPOSE: Unit tests for the custom logger CoT JSON formatters and configuration context manager. # @LAYER: Logging (Tests) diff --git a/backend/tests/test_resource_hubs.py b/backend/tests/test_resource_hubs.py index c05ff603..8f047569 100644 --- a/backend/tests/test_resource_hubs.py +++ b/backend/tests/test_resource_hubs.py @@ -1,7 +1,6 @@ # [DEF:TestResourceHubs:Module] # @RELATION: DEPENDS_ON -> [DashboardsApi] # @RELATION: DEPENDS_ON -> [DatasetsApi] -# @COMPLEXITY: 3 # @SEMANTICS: tests, resource-hubs, dashboards, datasets, pagination, api # @PURPOSE: Contract tests for resource hub dashboards/datasets listing and pagination boundary validation. # @LAYER: Domain (Tests) diff --git a/backend/tests/test_smoke_app.py b/backend/tests/test_smoke_app.py index 557d6038..f2e12fa7 100644 --- a/backend/tests/test_smoke_app.py +++ b/backend/tests/test_smoke_app.py @@ -1,5 +1,4 @@ # [DEF:TestSmokeApp:Module] -# @COMPLEXITY: 2 # @SEMANTICS: tests, smoke, app, imports, fastapi # @PURPOSE: Minimal smoke tests that verify the full application import chain # succeeds without SyntaxError, IndentationError, ImportError, or diff --git a/backend/tests/test_task_manager.py b/backend/tests/test_task_manager.py index 0da2f243..fd58af98 100644 --- a/backend/tests/test_task_manager.py +++ b/backend/tests/test_task_manager.py @@ -1,6 +1,5 @@ # [DEF:test_task_manager:Module] # @RELATION: BELONGS_TO -> SrcRoot -# @COMPLEXITY: 5 # @SEMANTICS: task-manager, lifecycle, CRUD, log-buffer, filtering, tests # @PURPOSE: Unit tests for TaskManager lifecycle, CRUD, log buffering, and filtering. # @LAYER: Core diff --git a/backend/tests/test_task_persistence.py b/backend/tests/test_task_persistence.py index 9a3fbee9..93e6c76d 100644 --- a/backend/tests/test_task_persistence.py +++ b/backend/tests/test_task_persistence.py @@ -3,7 +3,6 @@ # @SEMANTICS: test, task, persistence, unit_test # @PURPOSE: Unit tests for TaskPersistenceService. # @LAYER: Test -# @COMPLEXITY: 5 # @TEST_DATA: valid_task -> {"id": "test-uuid-1", "plugin_id": "backup", "status": "PENDING"} # [SECTION: IMPORTS] @@ -23,7 +22,6 @@ from src.core.task_manager.models import Task, TaskStatus, LogEntry # [DEF:TestTaskPersistenceHelpers:Class] # @RELATION: BINDS_TO -> test_task_persistence # @PURPOSE: Test suite for TaskPersistenceService static helper methods. -# @COMPLEXITY: 5 class TestTaskPersistenceHelpers: # [DEF:test_json_load_if_needed_none:Function] @@ -113,7 +111,6 @@ class TestTaskPersistenceHelpers: # [DEF:TestTaskPersistenceService:Class] # @RELATION: BINDS_TO -> test_task_persistence # @PURPOSE: Test suite for TaskPersistenceService CRUD operations. -# @COMPLEXITY: 5 # @TEST_DATA: valid_task -> {"id": "test-uuid-1", "plugin_id": "backup", "status": "PENDING"} class TestTaskPersistenceService: diff --git a/backend/tests/test_translate_corrections.py b/backend/tests/test_translate_corrections.py index 4aacc283..92c69f26 100644 --- a/backend/tests/test_translate_corrections.py +++ b/backend/tests/test_translate_corrections.py @@ -1,5 +1,4 @@ # [DEF:TranslateCorrectionTests:Module] -# @COMPLEXITY: 4 # @SEMANTICS: tests, translate, corrections, dictionary # @PURPOSE: Tests for term correction API endpoints and DictionaryManager correction methods. # @LAYER: Test diff --git a/backend/tests/test_translate_history.py b/backend/tests/test_translate_history.py index f4767113..577a71b0 100644 --- a/backend/tests/test_translate_history.py +++ b/backend/tests/test_translate_history.py @@ -1,5 +1,4 @@ # [DEF:TranslateHistoryTests:Module] -# @COMPLEXITY: 4 # @SEMANTICS: tests, translate, history, metrics # @PURPOSE: Tests for run history list/detail endpoints and metrics aggregation. # @LAYER: Test diff --git a/backend/tests/test_translate_jobs.py b/backend/tests/test_translate_jobs.py index 8c4f37d6..ab296ad6 100644 --- a/backend/tests/test_translate_jobs.py +++ b/backend/tests/test_translate_jobs.py @@ -1,5 +1,4 @@ # [DEF:TranslateJobTests:Module] -# @COMPLEXITY: 4 # @SEMANTICS: tests, translate, jobs, crud, validation # @PURPOSE: Tests for translation job CRUD endpoints and service layer with column validation. # @LAYER: Test diff --git a/backend/tests/test_translate_scheduler.py b/backend/tests/test_translate_scheduler.py index b2a042fd..b1a77c6e 100644 --- a/backend/tests/test_translate_scheduler.py +++ b/backend/tests/test_translate_scheduler.py @@ -1,5 +1,4 @@ # [DEF:TranslateSchedulerTests:Module] -# @COMPLEXITY: 4 # @SEMANTICS: tests, translate, scheduler # @PURPOSE: Tests for TranslationScheduler CRUD and APScheduler integration. # @LAYER: Test diff --git a/frontend/src/components/DashboardGrid.svelte b/frontend/src/components/DashboardGrid.svelte index 24160953..eec005e9 100644 --- a/frontend/src/components/DashboardGrid.svelte +++ b/frontend/src/components/DashboardGrid.svelte @@ -4,7 +4,6 @@ diff --git a/frontend/src/components/DynamicForm.svelte b/frontend/src/components/DynamicForm.svelte index eed14dfb..c75ba8eb 100755 --- a/frontend/src/components/DynamicForm.svelte +++ b/frontend/src/components/DynamicForm.svelte @@ -20,7 +20,7 @@ let formData = {}; - // [DEF:handleSubmit:Function] + // #region handleSubmit:Function [TYPE Function] /** * @purpose Emits submitted form data via callback prop. * @pre formData contains user input. @@ -30,9 +30,9 @@ console.log("[DynamicForm][Action] Submitting form data.", { formData }); onsubmit(formData); } - // [/DEF:handleSubmit:Function] + // #endregion handleSubmit:Function - // [DEF:initializeForm:Function] + // #region initializeForm:Function [TYPE Function] /** * @purpose Initialize form data with default values from the schema. * @pre schema is provided and contains properties. @@ -45,7 +45,7 @@ } } } - // [/DEF:initializeForm:Function] + // #endregion initializeForm:Function initializeForm(); diff --git a/frontend/src/components/EnvSelector.svelte b/frontend/src/components/EnvSelector.svelte index ca873047..b0e83f3c 100644 --- a/frontend/src/components/EnvSelector.svelte +++ b/frontend/src/components/EnvSelector.svelte @@ -25,7 +25,7 @@ // [/SECTION] - // [DEF:handleSelect:Function] + // #region handleSelect:Function [TYPE Function] /** * @purpose Propagates the selection change through a callback prop. * @pre event.target must be an HTMLSelectElement. @@ -37,7 +37,7 @@ selectedId = target.value; onchange({ id: selectedId }); } - // [/DEF:handleSelect:Function] + // #endregion handleSelect:Function diff --git a/frontend/src/components/Footer.svelte b/frontend/src/components/Footer.svelte index f0205684..abef9d93 100644 --- a/frontend/src/components/Footer.svelte +++ b/frontend/src/components/Footer.svelte @@ -1,8 +1,8 @@ - + + diff --git a/frontend/src/components/MissingMappingModal.svelte b/frontend/src/components/MissingMappingModal.svelte index 12da0bc8..ca8fff51 100644 --- a/frontend/src/components/MissingMappingModal.svelte +++ b/frontend/src/components/MissingMappingModal.svelte @@ -26,7 +26,7 @@ let selectedTargetUuid = $state(""); - // [DEF:resolve:Function] + // #region resolve:Function [TYPE Function] // @PURPOSE: Resolves the missing mapping via callback prop. // @PRE: selectedTargetUuid must be set. // @POST: Parent callback receives mapping payload and modal closes. @@ -39,9 +39,9 @@ }); show = false; } - // [/DEF:resolve:Function] + // #endregion resolve:Function - // [DEF:cancel:Function] + // #region cancel:Function [TYPE Function] // @PURPOSE: Cancels the mapping resolution modal. // @PRE: Modal is open. // @POST: Parent cancel callback is invoked and modal is hidden. @@ -49,7 +49,7 @@ oncancel(); show = false; } - // [/DEF:cancel:Function] + // #endregion cancel:Function diff --git a/frontend/src/components/Navbar.svelte b/frontend/src/components/Navbar.svelte index 133b5b1c..5e83c291 100644 --- a/frontend/src/components/Navbar.svelte +++ b/frontend/src/components/Navbar.svelte @@ -2,7 +2,6 @@
diff --git a/frontend/src/components/TaskList.svelte b/frontend/src/components/TaskList.svelte index ad2a9790..dfe8d997 100644 --- a/frontend/src/components/TaskList.svelte +++ b/frontend/src/components/TaskList.svelte @@ -19,7 +19,7 @@ onselect = () => {}, } = $props(); - // [DEF:getStatusColor:Function] + // #region getStatusColor:Function [TYPE Function] // @PURPOSE: Returns the CSS color class for a given task status. // @PRE: status string is provided. // @POST: Returns tailwind color class string. @@ -34,9 +34,9 @@ default: return 'bg-gray-100 text-gray-800'; } } - // [/DEF:getStatusColor:Function] + // #endregion getStatusColor:Function - // [DEF:formatTime:Function] + // #region formatTime:Function [TYPE Function] // @PURPOSE: Formats a date string using date-fns. // @PRE: dateStr is a valid date string or null. // @POST: Returns human-readable relative time string. @@ -48,16 +48,16 @@ return 'Invalid date'; } } - // [/DEF:formatTime:Function] + // #endregion formatTime:Function - // [DEF:handleTaskClick:Function] + // #region handleTaskClick:Function [TYPE Function] // @PURPOSE: Forwards the selected task through a callback prop. // @PRE: taskId is provided. // @POST: Parent callback receives task id and task payload. function handleTaskClick(task: any) { onselect({ id: task.id, task }); } - // [/DEF:handleTaskClick:Function] + // #endregion handleTaskClick:Function
diff --git a/frontend/src/components/TaskLogViewer.svelte b/frontend/src/components/TaskLogViewer.svelte index 709ea6a0..6d8bf2a7 100644 --- a/frontend/src/components/TaskLogViewer.svelte +++ b/frontend/src/components/TaskLogViewer.svelte @@ -4,7 +4,6 @@ diff --git a/frontend/src/components/Toast.svelte b/frontend/src/components/Toast.svelte index 3d54029d..78391379 100755 --- a/frontend/src/components/Toast.svelte +++ b/frontend/src/components/Toast.svelte @@ -2,7 +2,6 @@ diff --git a/frontend/src/components/git/CommitHistory.svelte b/frontend/src/components/git/CommitHistory.svelte index dad0b81f..8539983b 100644 --- a/frontend/src/components/git/CommitHistory.svelte +++ b/frontend/src/components/git/CommitHistory.svelte @@ -30,7 +30,7 @@ let loading = $state(false); // [/SECTION] - // [DEF:onMount:Function] + // #region onMount:Function [TYPE Function] /** * @purpose Load history when component is mounted. * @pre Component is initialized with dashboardId. @@ -39,9 +39,9 @@ onMount(async () => { await loadHistory(); }); - // [/DEF:onMount:Function] + // #endregion onMount:Function - // [DEF:loadHistory:Function] + // #region loadHistory:Function [TYPE Function] /** * @purpose Fetch commit history from the backend. * @pre dashboardId is valid. @@ -60,7 +60,7 @@ loading = false; } } - // [/DEF:loadHistory:Function] + // #endregion loadHistory:Function diff --git a/frontend/src/components/git/CommitModal.svelte b/frontend/src/components/git/CommitModal.svelte index f0c7646b..4c773d45 100644 --- a/frontend/src/components/git/CommitModal.svelte +++ b/frontend/src/components/git/CommitModal.svelte @@ -4,7 +4,6 @@ diff --git a/frontend/src/components/git/DeploymentModal.svelte b/frontend/src/components/git/DeploymentModal.svelte index e5478093..8e8c1565 100644 --- a/frontend/src/components/git/DeploymentModal.svelte +++ b/frontend/src/components/git/DeploymentModal.svelte @@ -46,13 +46,13 @@ ); const isProdTarget = $derived(selectedEnvironmentStage === "PROD"); - // [DEF:loadStatus:Watcher] + // #region loadStatus:Watcher [TYPE Function] $effect(() => { if (show) loadEnvironments(); }); - // [/DEF:loadStatus:Watcher] + // #endregion loadStatus:Watcher - // [DEF:normalizeEnvStage:Function] + // #region normalizeEnvStage:Function [TYPE Function] /** * @purpose Normalize environment stage with legacy production fallback. * @post Returns DEV/PREPROD/PROD. @@ -63,9 +63,9 @@ if (stage === "PROD" || stage === "PREPROD") return stage; return "DEV"; } - // [/DEF:normalizeEnvStage:Function] + // #endregion normalizeEnvStage:Function - // [DEF:resolveEnvUrl:Function] + // #region resolveEnvUrl:Function [TYPE Function] /** * @purpose Resolve environment URL from consolidated or git-specific payload shape. * @post Returns stable URL string. @@ -73,9 +73,9 @@ function resolveEnvUrl(env) { return String(env?.superset_url || env?.url || ""); } - // [/DEF:resolveEnvUrl:Function] + // #endregion resolveEnvUrl:Function - // [DEF:loadEnvironments:Function] + // #region loadEnvironments:Function [TYPE Function] /** * @purpose Fetch available environments from API. * @post environments state is populated. @@ -103,9 +103,9 @@ loading = false; } } - // [/DEF:loadEnvironments:Function] + // #endregion loadEnvironments:Function - // [DEF:handleDeploy:Function] + // #region handleDeploy:Function [TYPE Function] /** * @purpose Trigger deployment to selected environment. * @pre selectedEnv must be set. @@ -139,7 +139,7 @@ deploying = false; } } - // [/DEF:handleDeploy:Function] + // #endregion handleDeploy:Function diff --git a/frontend/src/components/git/GitManager.svelte b/frontend/src/components/git/GitManager.svelte index 748dc085..9187be8e 100644 --- a/frontend/src/components/git/GitManager.svelte +++ b/frontend/src/components/git/GitManager.svelte @@ -96,7 +96,7 @@ ].length; }); - // [DEF:normalizeEnvStage:Function] + // #region normalizeEnvStage:Function [TYPE Function] /** * @purpose Normalize environment stage with legacy fallback. * @post Returns DEV/PREPROD/PROD. @@ -107,9 +107,9 @@ if (stage === 'PROD' || stage === 'PREPROD') return stage; return 'DEV'; } - // [/DEF:normalizeEnvStage:Function] + // #endregion normalizeEnvStage:Function - // [DEF:stageBadgeClass:Function] + // #region stageBadgeClass:Function [TYPE Function] /** * @purpose Return visual class for environment stage badges. * @post Stage badge is color-coded by risk level. @@ -119,9 +119,9 @@ if (stage === 'PREPROD') return 'bg-amber-100 text-amber-800 border-amber-200'; return 'bg-blue-100 text-blue-800 border-blue-200'; } - // [/DEF:stageBadgeClass:Function] + // #endregion stageBadgeClass:Function - // [DEF:resolveCurrentEnvironmentId:Function] + // #region resolveCurrentEnvironmentId:Function [TYPE Function] /** * @purpose Resolve active environment id for current dashboard session. * @post Returns env id from prop or selected_env_id in localStorage. @@ -131,9 +131,9 @@ if (typeof window === 'undefined') return null; return localStorage.getItem('selected_env_id'); } - // [/DEF:resolveCurrentEnvironmentId:Function] + // #endregion resolveCurrentEnvironmentId:Function - // [DEF:applyGitflowStageDefaults:Function] + // #region applyGitflowStageDefaults:Function [TYPE Function] /** * @purpose Apply GitFlow defaults by current environment stage. * @post Promote branches and preferred deploy target stage are set. @@ -156,9 +156,9 @@ promoteToBranch = 'main'; preferredDeployTargetStage = ''; } - // [/DEF:applyGitflowStageDefaults:Function] + // #endregion applyGitflowStageDefaults:Function - // [DEF:loadCurrentEnvironmentStage:Function] + // #region loadCurrentEnvironmentStage:Function [TYPE Function] /** * @purpose Detect current environment stage and bind GitFlow defaults. * @post currentEnvStage and promote/deploy defaults are refreshed. @@ -176,9 +176,9 @@ console.error(`[GitManager][Coherence:Failed] Failed to resolve environment stage: ${e.message}`); } } - // [/DEF:loadCurrentEnvironmentStage:Function] + // #endregion loadCurrentEnvironmentStage:Function - // [DEF:isNumericDashboardRef:Function] + // #region isNumericDashboardRef:Function [TYPE Function] /** * @purpose Checks whether current dashboard reference is numeric ID. * @post Returns true when dashboardId is digits-only. @@ -186,9 +186,9 @@ function isNumericDashboardRef() { return /^\d+$/.test(String(dashboardId || '').trim()); } - // [/DEF:isNumericDashboardRef:Function] + // #endregion isNumericDashboardRef:Function - // [DEF:checkStatus:Function] + // #region checkStatus:Function [TYPE Function] /** * @purpose Verify repository initialization for dashboard slug. * @post initialized status and config list are updated. @@ -214,9 +214,9 @@ checkingStatus = false; } } - // [/DEF:checkStatus:Function] + // #endregion checkStatus:Function - // [DEF:loadWorkspace:Function] + // #region loadWorkspace:Function [TYPE Function] /** * @purpose Load current git status and combined diff for workspace tab. * @post workspace status/diff are refreshed. @@ -236,9 +236,9 @@ workspaceLoading = false; } } - // [/DEF:loadWorkspace:Function] + // #endregion loadWorkspace:Function - // [DEF:handleSync:Function] + // #region handleSync:Function [TYPE Function] /** * @purpose Sync latest dashboard config from Superset to git workspace. * @post Workspace status is refreshed. @@ -260,9 +260,9 @@ loading = false; } } - // [/DEF:handleSync:Function] + // #endregion handleSync:Function - // [DEF:handleGenerateMessage:Function] + // #region handleGenerateMessage:Function [TYPE Function] /** * @purpose Generate commit message from diff using LLM. * @post commitMessage gets generated content. @@ -281,9 +281,9 @@ generatingMessage = false; } } - // [/DEF:handleGenerateMessage:Function] + // #endregion handleGenerateMessage:Function - // [DEF:handleCommit:Function] + // #region handleCommit:Function [TYPE Function] /** * @purpose Commit current workspace changes with provided message. * @post Changes are committed and workspace is refreshed. @@ -307,9 +307,9 @@ committing = false; } } - // [/DEF:handleCommit:Function] + // #endregion handleCommit:Function - // [DEF:handlePull:Function] + // #region handlePull:Function [TYPE Function] /** * @purpose Pull remote changes for current branch. * @post Workspace status is refreshed. @@ -506,9 +506,9 @@ isPulling = false; } } - // [/DEF:handlePull:Function] + // #endregion handlePull:Function - // [DEF:handlePush:Function] + // #region handlePush:Function [TYPE Function] /** * @purpose Push local commits to remote repository. * @post Workspace status is refreshed. @@ -525,9 +525,9 @@ isPushing = false; } } - // [/DEF:handlePush:Function] + // #endregion handlePush:Function - // [DEF:handlePromote:Function] + // #region handlePromote:Function [TYPE Function] /** * @purpose Promote branch to the next stage via MR or unsafe direct mode. * @post Promotion result is shown to user. @@ -575,9 +575,9 @@ promoting = false; } } - // [/DEF:handlePromote:Function] + // #endregion handlePromote:Function - // [DEF:openDeployModal:Function] + // #region openDeployModal:Function [TYPE Function] /** * @purpose Open deploy modal with extra confirmation for PROD context. * @post Deploy modal is shown when confirmation passes. @@ -593,9 +593,9 @@ } showDeployModal = true; } - // [/DEF:openDeployModal:Function] + // #endregion openDeployModal:Function - // [DEF:getSelectedConfig:Function] + // #region getSelectedConfig:Function [TYPE Function] /** * @purpose Return currently selected git server config. * @post Config object or null. @@ -603,9 +603,9 @@ function getSelectedConfig() { return configs.find((item) => item.id === selectedConfigId) || null; } - // [/DEF:getSelectedConfig:Function] + // #endregion getSelectedConfig:Function - // [DEF:resolveDefaultConfig:Function] + // #region resolveDefaultConfig:Function [TYPE Function] /** * @purpose Resolve default git config for current session. * @post Returns config by selected -> is_default -> CONNECTED -> first. @@ -619,9 +619,9 @@ const connected = configList.find((item) => item?.status === 'CONNECTED'); return connected || configList[0]; } - // [/DEF:resolveDefaultConfig:Function] + // #endregion resolveDefaultConfig:Function - // [DEF:resolvePushProviderLabel:Function] + // #region resolvePushProviderLabel:Function [TYPE Function] /** * @purpose Resolve lower-case provider label for auto-push checkbox. * @post Returns provider label, fallback "git". @@ -633,9 +633,9 @@ .toLowerCase(); return provider || 'git'; } - // [/DEF:resolvePushProviderLabel:Function] + // #endregion resolvePushProviderLabel:Function - // [DEF:extractHttpHost:Function] + // #region extractHttpHost:Function [TYPE Function] /** * @purpose Extract comparable host[:port] from URL string. * @post Returns lowercase host token or empty string. @@ -651,9 +651,9 @@ return ''; } } - // [/DEF:extractHttpHost:Function] + // #endregion extractHttpHost:Function - // [DEF:buildSuggestedRepoName:Function] + // #region buildSuggestedRepoName:Function [TYPE Function] /** * @purpose Build deterministic repository name from dashboard title/id. * @post Returns kebab-case repository name. @@ -666,9 +666,9 @@ .slice(0, 48); return `${slug || 'dashboard'}-${dashboardId}`; } - // [/DEF:buildSuggestedRepoName:Function] + // #endregion buildSuggestedRepoName:Function - // [DEF:handleCreateRemoteRepo:Function] + // #region handleCreateRemoteRepo:Function [TYPE Function] /** * @purpose Create remote repository on selected Git provider. * @post remoteUrl is filled from provider response. @@ -706,9 +706,9 @@ creatingRemoteRepo = false; } } - // [/DEF:handleCreateRemoteRepo:Function] + // #endregion handleCreateRemoteRepo:Function - // [DEF:handleInit:Function] + // #region handleInit:Function [TYPE Function] /** * @purpose Initialize git repository for dashboard. * @post Repository is linked and workspace is loaded. @@ -732,9 +732,9 @@ loading = false; } } - // [/DEF:handleInit:Function] + // #endregion handleInit:Function - // [DEF:closeModal:Function] + // #region closeModal:Function [TYPE Function] /** * @purpose Close git manager modal. * @post show=false. @@ -742,9 +742,9 @@ function closeModal() { show = false; } - // [/DEF:closeModal:Function] + // #endregion closeModal:Function - // [DEF:handleBackdropClick:Function] + // #region handleBackdropClick:Function [TYPE Function] /** * @purpose Close modal on backdrop click. * @post show=false if clicked on overlay. @@ -754,7 +754,7 @@ closeModal(); } } - // [/DEF:handleBackdropClick:Function] + // #endregion handleBackdropClick:Function const originHost = $derived.by(() => extractHttpHost(repositoryBindingRemoteUrl)); const configHost = $derived.by(() => extractHttpHost(repositoryConfigUrl)); diff --git a/frontend/src/components/git/__tests__/git_manager.unfinished_merge.integration.test.js b/frontend/src/components/git/__tests__/git_manager.unfinished_merge.integration.test.js index 920273ef..335945c1 100644 --- a/frontend/src/components/git/__tests__/git_manager.unfinished_merge.integration.test.js +++ b/frontend/src/components/git/__tests__/git_manager.unfinished_merge.integration.test.js @@ -1,5 +1,4 @@ -// [DEF:GitManagerUnfinishedMergeIntegrationTest:Module] -// @COMPLEXITY: 3 +// #region GitManagerUnfinishedMergeIntegrationTest:Module [TYPE Function] // @SEMANTICS: git-manager, unfinished-merge, dialog, integration-test // @PURPOSE: Protect unresolved-merge dialog contract in GitManager pull flow. // @LAYER: UI Tests @@ -40,4 +39,4 @@ describe('GitManager unfinished merge dialog contract', () => { expect(source).toContain('$t.git?.unfinished_merge?.copy_commands'); }); }); -// [/DEF:GitManagerUnfinishedMergeIntegrationTest:Module] +// #endregion GitManagerUnfinishedMergeIntegrationTest:Module diff --git a/frontend/src/components/llm/DocPreview.svelte b/frontend/src/components/llm/DocPreview.svelte index f13a6e03..27e39c8b 100644 --- a/frontend/src/components/llm/DocPreview.svelte +++ b/frontend/src/components/llm/DocPreview.svelte @@ -4,7 +4,6 @@ - + - + + diff --git a/frontend/src/components/storage/FileUpload.svelte b/frontend/src/components/storage/FileUpload.svelte index 82c6d341..06c676a2 100644 --- a/frontend/src/components/storage/FileUpload.svelte +++ b/frontend/src/components/storage/FileUpload.svelte @@ -4,7 +4,6 @@ diff --git a/frontend/src/components/tasks/LogEntryRow.svelte b/frontend/src/components/tasks/LogEntryRow.svelte index c98b4501..d3158564 100644 --- a/frontend/src/components/tasks/LogEntryRow.svelte +++ b/frontend/src/components/tasks/LogEntryRow.svelte @@ -4,7 +4,6 @@ diff --git a/frontend/src/components/tools/ConnectionList.svelte b/frontend/src/components/tools/ConnectionList.svelte index 19f8298e..937e2059 100644 --- a/frontend/src/components/tools/ConnectionList.svelte +++ b/frontend/src/components/tools/ConnectionList.svelte @@ -19,7 +19,7 @@ let connections = $state([]); let isLoading = $state(true); - // [DEF:fetchConnections:Function] + // #region fetchConnections:Function [TYPE Function] // @PURPOSE: Fetches the list of connections from the backend. // @PRE: None. // @POST: connections array is populated. @@ -33,9 +33,9 @@ isLoading = false; } } - // [/DEF:fetchConnections:Function] + // #endregion fetchConnections:Function - // [DEF:handleDelete:Function] + // #region handleDelete:Function [TYPE Function] // @PURPOSE: Deletes a connection configuration. // @PRE: id is provided and user confirms deletion. // @POST: Connection is deleted from backend and list is reloaded. @@ -50,7 +50,7 @@ addToast(e.message, 'error'); } } - // [/DEF:handleDelete:Function] + // #endregion handleDelete:Function onMount(fetchConnections); diff --git a/frontend/src/components/tools/DebugTool.svelte b/frontend/src/components/tools/DebugTool.svelte index 55bbada3..46e6cfe5 100644 --- a/frontend/src/components/tools/DebugTool.svelte +++ b/frontend/src/components/tools/DebugTool.svelte @@ -27,7 +27,7 @@ let results = null; let pollInterval; - // [DEF:fetchEnvironments:Function] + // #region fetchEnvironments:Function [TYPE Function] /** * @purpose Fetches available environments. * @pre API is available. @@ -41,9 +41,9 @@ addToast($t.debug?.fetch_env_failed, 'error'); } } - // [/DEF:fetchEnvironments:Function] + // #endregion fetchEnvironments:Function - // [DEF:handleRunDebug:Function] + // #region handleRunDebug:Function [TYPE Function] /** * @purpose Triggers the debug task. * @pre Required fields are selected. @@ -84,9 +84,9 @@ addToast(e.message, 'error'); } } - // [/DEF:handleRunDebug:Function] + // #endregion handleRunDebug:Function - // [DEF:startPolling:Function] + // #region startPolling:Function [TYPE Function] /** * @purpose Polls for task completion. * @pre Task ID is valid. @@ -116,7 +116,7 @@ } }, 2000); } - // [/DEF:startPolling:Function] + // #endregion startPolling:Function onMount(fetchEnvironments); diff --git a/frontend/src/components/tools/MapperTool.svelte b/frontend/src/components/tools/MapperTool.svelte index e8370610..4e7a7443 100644 --- a/frontend/src/components/tools/MapperTool.svelte +++ b/frontend/src/components/tools/MapperTool.svelte @@ -34,7 +34,7 @@ let isGeneratingDocs = false; let generatedDoc = null; - // [DEF:fetchData:Function] + // #region fetchData:Function [TYPE Function] // @PURPOSE: Fetches environments and saved connections. // @PRE: None. // @POST: envs and connections arrays are populated. @@ -46,9 +46,9 @@ addToast($t.mapper.errors.fetch_failed, 'error'); } } - // [/DEF:fetchData:Function] + // #endregion fetchData:Function - // [DEF:handleRunMapper:Function] + // #region handleRunMapper:Function [TYPE Function] // @PURPOSE: Triggers the MapperPlugin task. // @PRE: selectedEnv and datasetId are set; source-specific fields are valid. // @POST: Mapper task is started and selectedTask is updated. @@ -89,9 +89,9 @@ isRunning = false; } } - // [/DEF:handleRunMapper:Function] + // #endregion handleRunMapper:Function - // [DEF:handleGenerateDocs:Function] + // #region handleGenerateDocs:Function [TYPE Function] // @PURPOSE: Triggers the LLM Documentation task. // @PRE: selectedEnv and datasetId are set. // @POST: Documentation task is started. @@ -126,7 +126,7 @@ isGeneratingDocs = false; } } - // [/DEF:handleGenerateDocs:Function] + // #endregion handleGenerateDocs:Function async function handleApplyDoc(doc) { try { diff --git a/frontend/src/lib/Counter.svelte b/frontend/src/lib/Counter.svelte index dc43ff34..05711758 100755 --- a/frontend/src/lib/Counter.svelte +++ b/frontend/src/lib/Counter.svelte @@ -2,7 +2,6 @@ diff --git a/frontend/src/pages/Settings.svelte b/frontend/src/pages/Settings.svelte index e66c69c0..60a7db4c 100755 --- a/frontend/src/pages/Settings.svelte +++ b/frontend/src/pages/Settings.svelte @@ -2,7 +2,6 @@
diff --git a/frontend/src/routes/dashboards/health/__tests__/health_page.integration.test.js b/frontend/src/routes/dashboards/health/__tests__/health_page.integration.test.js index 8b9992a2..8d7c6dcb 100644 --- a/frontend/src/routes/dashboards/health/__tests__/health_page.integration.test.js +++ b/frontend/src/routes/dashboards/health/__tests__/health_page.integration.test.js @@ -1,5 +1,4 @@ -// [DEF:HealthPageIntegrationTest:Module] -// @COMPLEXITY: 3 +// #region HealthPageIntegrationTest:Module [TYPE Function] // @SEMANTICS: health-page, integration-test, slug-link, delete-flow // @PURPOSE: Lock dashboard health page contract for slug navigation and report deletion. // @LAYER: UI Tests @@ -40,4 +39,4 @@ describe('Dashboard health page contract', () => { expect(source).toContain(': healthStore.refresh();'); }); }); -// [/DEF:HealthPageIntegrationTest:Module] +// #endregion HealthPageIntegrationTest:Module diff --git a/frontend/src/routes/datasets/+page.svelte b/frontend/src/routes/datasets/+page.svelte index 2eb6b1bf..afad4d5d 100644 --- a/frontend/src/routes/datasets/+page.svelte +++ b/frontend/src/routes/datasets/+page.svelte @@ -13,7 +13,6 @@ - + +
- + + - + + - + + {#if $selectedTask}
@@ -518,7 +521,8 @@
{/if} - + +
+

{$t.migration?.select_dashboards_title } @@ -554,7 +559,8 @@

- + +
- + + {#if dryRunResult}

Pre-flight Diff

@@ -685,9 +692,10 @@ {/if}
- + + diff --git a/frontend/src/routes/migration/mappings/+page.svelte b/frontend/src/routes/migration/mappings/+page.svelte index 8a6f7551..fc146a99 100644 --- a/frontend/src/routes/migration/mappings/+page.svelte +++ b/frontend/src/routes/migration/mappings/+page.svelte @@ -2,7 +2,6 @@ - + +
diff --git a/frontend/src/routes/profile/__tests__/fixtures/profile.fixtures.js b/frontend/src/routes/profile/__tests__/fixtures/profile.fixtures.js index 16911a66..5d211fa8 100644 --- a/frontend/src/routes/profile/__tests__/fixtures/profile.fixtures.js +++ b/frontend/src/routes/profile/__tests__/fixtures/profile.fixtures.js @@ -1,5 +1,4 @@ -// [DEF:ProfileFixtures:Module] -// @COMPLEXITY: 1 +// #region ProfileFixtures:Module [TYPE Function] // @SEMANTICS: fixtures, profile, ui // @PURPOSE: Shared deterministic fixture inputs for profile page integration tests. // @INVARIANT: lookupFailedManualFallback.warning is a hardcoded English string; drift risk if i18n key profile.lookup_error changes without updating this fixture. @@ -25,4 +24,4 @@ export const profileFixtures = { }, }; -// [/DEF:ProfileFixtures:Module] +// #endregion ProfileFixtures:Module diff --git a/frontend/src/routes/profile/__tests__/profile-preferences.integration.test.js b/frontend/src/routes/profile/__tests__/profile-preferences.integration.test.js index b9bc4b09..1d35b3b6 100644 --- a/frontend/src/routes/profile/__tests__/profile-preferences.integration.test.js +++ b/frontend/src/routes/profile/__tests__/profile-preferences.integration.test.js @@ -1,29 +1,23 @@ // [DEF:ProfilePreferencesIntegrationTest:Module] // @COMPLEXITY: 3 -// @SEMANTICS: tests, profile, integration, lookup, persistence, fallback -// @PURPOSE: Verifies profile binding happy path and degraded lookup manual fallback save flow. +// @SEMANTICS: tests, profile, integration, load, save +// @PURPOSE: Verifies profile page loads preferences and saves them. // @LAYER: UI (Tests) // @RELATION: DEPENDS_ON -> [ProfilePage] import { describe, it, expect, vi, beforeEach } from "vitest"; import { render, screen, fireEvent, waitFor } from "@testing-library/svelte"; -import { writable } from "svelte/store"; - import ProfilePage from "../+page.svelte"; import { api } from "$lib/api.js"; import { addToast } from "$lib/toasts"; -import { initializeEnvironmentContext } from "$lib/stores/environmentContext.js"; const mockedApi = /** @type {any} */ (api); const mockedAddToast = /** @type {any} */ (addToast); -const mockedInitializeEnvironmentContext = - /** @type {any} */ (initializeEnvironmentContext); vi.mock("$lib/api.js", () => ({ api: { getProfilePreferences: vi.fn(), - lookupSupersetAccounts: vi.fn(), - updateProfilePreferences: vi.fn(), + postApi: vi.fn(), }, })); @@ -31,46 +25,20 @@ vi.mock("$lib/toasts", () => ({ addToast: vi.fn(), })); -vi.mock("$lib/stores/environmentContext.js", () => { - const context = writable({ - environments: [{ id: "dev", name: "Development" }], - selectedEnvId: "dev", - isLoading: false, - isLoaded: true, - error: null, - }); - - return { - environmentContextStore: { subscribe: context.subscribe }, - initializeEnvironmentContext: vi.fn().mockResolvedValue(undefined), - }; -}); +vi.mock("$lib/auth/store.js", () => ({ + auth: { subscribe: (run) => { run({ current: { user: { id: "u-1" } } }); return () => {}; } }, +})); vi.mock("$lib/i18n", () => ({ t: { subscribe: (run) => { run({ - common: { cancel: "Cancel" }, + common: { save: "Save", cancel: "Cancel" }, + nav: { profile: "Profile", dashboards: "Dashboards", datasets: "Datasets", reports: "Reports" }, profile: { - title: "Profile", - description: "Manage your profile preferences, Git integration, and access view.", - dashboard_preferences: "Dashboard Preferences", - superset_environment: "Superset Environment", - superset_environment_placeholder: "Select environment", - superset_account: "Your Apache Superset Account", - superset_account_placeholder: "Enter your Apache Superset username", - lookup_loading: "Loading Superset accounts...", - lookup_error: - "Cannot load Superset accounts for this environment right now. You can enter username manually.", - lookup_button: "Lookup", - lookup_search_placeholder: "Search account candidates", - lookup_no_results: - "No account candidates found. You can enter username manually.", - show_only_my_dashboards: "Show only my dashboards by default", - show_only_slug_dashboards: "Show only dashboards with slug by default", - save_preferences: "Save Preferences", - save_success: "Preferences saved", - save_error: "Failed to save preferences. Please try again.", + saved: "Preferences saved", + preferences: "Preferences", + start_page: "Start page", }, }); return () => {}; @@ -82,191 +50,50 @@ vi.mock("$lib/i18n", () => ({ describe("profile-preferences.integration", () => { beforeEach(() => { vi.clearAllMocks(); - mockedApi.getProfilePreferences.mockResolvedValue({ status: "success", preference: { user_id: "u-1", - superset_username: null, - show_only_my_dashboards: false, - show_only_slug_dashboards: true, - git_username: null, - git_email: null, - has_git_personal_access_token: false, - git_personal_access_token_masked: null, - email_address: null, - telegram_id: null, - notify_on_fail: true, start_page: "dashboards", - auto_open_task_drawer: true, - dashboards_table_density: "comfortable", - }, - security: { - read_only: true, - auth_source: "adfs", - current_role: "Data Engineer", - role_source: "adfs", - roles: ["Data Engineer"], - permissions: [], - }, - }); - - mockedApi.updateProfilePreferences.mockResolvedValue({ - status: "success", - preference: { - user_id: "u-1", - superset_username: "john_doe", - show_only_my_dashboards: true, - show_only_slug_dashboards: true, - git_username: null, - git_email: null, - has_git_personal_access_token: false, - git_personal_access_token_masked: null, - email_address: null, - telegram_id: null, - notify_on_fail: true, - start_page: "dashboards", - auto_open_task_drawer: true, - dashboards_table_density: "comfortable", - }, - security: { - read_only: true, - auth_source: "adfs", - current_role: "Data Engineer", - role_source: "adfs", - roles: ["Data Engineer"], - permissions: [], }, }); }); - it("binds account from lookup and saves enabled default filter", async () => { - mockedApi.lookupSupersetAccounts.mockResolvedValue({ - status: "success", - environment_id: "dev", - page_index: 0, - page_size: 20, - total: 1, - items: [ - { - environment_id: "dev", - username: "john_doe", - display_name: "John Doe", - email: "john@example.local", - is_active: true, - }, - ], - }); - + it("loads preferences and renders the profile form", async () => { render(ProfilePage); await waitFor(() => { - expect(mockedInitializeEnvironmentContext).toHaveBeenCalledTimes(1); expect(mockedApi.getProfilePreferences).toHaveBeenCalledTimes(1); - expect(mockedApi.lookupSupersetAccounts).toHaveBeenCalled(); }); - await fireEvent.click(await screen.findByText("John Doe")); - - const usernameInput = /** @type {HTMLInputElement} */ ( - screen.getByPlaceholderText("Enter your Apache Superset username") - ); - expect(usernameInput.value).toBe("john_doe"); - - const toggle = screen.getByLabelText("Show only my dashboards by default"); - await fireEvent.click(toggle); - - await fireEvent.click( - screen.getByRole("button", { name: "Save Preferences" }), - ); - - await waitFor(() => { - expect(mockedApi.updateProfilePreferences).toHaveBeenCalledWith({ - superset_username: "john_doe", - show_only_my_dashboards: true, - show_only_slug_dashboards: true, - git_username: null, - git_email: null, - email_address: null, - telegram_id: null, - notify_on_fail: true, - start_page: "dashboards", - auto_open_task_drawer: true, - dashboards_table_density: "comfortable", - }); - expect(mockedAddToast).toHaveBeenCalledWith("Preferences saved", "success"); - }); + expect(screen.getByText("Profile")).toBeDefined(); + expect(screen.getByText("Preferences")).toBeDefined(); + expect(screen.getByText("Start page")).toBeDefined(); + const select = /** @type {HTMLSelectElement} */ (screen.getByRole("combobox")); + expect(select.value).toBe("dashboards"); }); - it("keeps manual save available when lookup fails (degraded fallback)", async () => { - mockedApi.lookupSupersetAccounts.mockRejectedValue( - new Error("lookup unavailable"), - ); - mockedApi.updateProfilePreferences.mockResolvedValue({ - status: "success", - preference: { - user_id: "u-1", - superset_username: "manual_user", - show_only_my_dashboards: true, - show_only_slug_dashboards: true, - git_username: null, - git_email: null, - has_git_personal_access_token: false, - git_personal_access_token_masked: null, - email_address: null, - telegram_id: null, - notify_on_fail: true, - start_page: "dashboards", - auto_open_task_drawer: true, - dashboards_table_density: "comfortable", - }, - security: { - read_only: true, - auth_source: "adfs", - current_role: "Data Engineer", - role_source: "adfs", - roles: ["Data Engineer"], - permissions: [], - }, - }); + it("saves preferences via API and shows success toast", async () => { + mockedApi.postApi.mockResolvedValue({ status: "success" }); render(ProfilePage); await waitFor(() => { - expect(mockedApi.lookupSupersetAccounts).toHaveBeenCalled(); + expect(mockedApi.getProfilePreferences).toHaveBeenCalledTimes(1); }); - await screen.findByText( - "Cannot load Superset accounts for this environment right now. You can enter username manually.", - ); + const select = /** @type {HTMLSelectElement} */ (screen.getByRole("combobox")); + await fireEvent.change(select, { target: { value: "datasets" } }); - const usernameInput = screen.getByPlaceholderText( - "Enter your Apache Superset username", - ); - await fireEvent.input(usernameInput, { target: { value: "manual_user" } }); - - const toggle = screen.getByLabelText("Show only my dashboards by default"); - await fireEvent.click(toggle); - - await fireEvent.click( - screen.getByRole("button", { name: "Save Preferences" }), - ); + await fireEvent.click(screen.getByText("Save")); await waitFor(() => { - expect(mockedApi.updateProfilePreferences).toHaveBeenCalledWith({ - superset_username: "manual_user", - show_only_my_dashboards: true, - show_only_slug_dashboards: true, - git_username: null, - git_email: null, - email_address: null, - telegram_id: null, - notify_on_fail: true, - start_page: "dashboards", - auto_open_task_drawer: true, - dashboards_table_density: "comfortable", + expect(mockedApi.postApi).toHaveBeenCalledWith("/profile/preferences", { + preference: expect.objectContaining({ start_page: "datasets" }), }); }); + + expect(mockedAddToast).toHaveBeenCalledWith("Preferences saved", "success"); }); }); diff --git a/frontend/src/routes/profile/__tests__/profile-settings-state.integration.test.js b/frontend/src/routes/profile/__tests__/profile-settings-state.integration.test.js index 3f64eecd..a679af0a 100644 --- a/frontend/src/routes/profile/__tests__/profile-settings-state.integration.test.js +++ b/frontend/src/routes/profile/__tests__/profile-settings-state.integration.test.js @@ -1,27 +1,23 @@ // [DEF:ProfileSettingsStateIntegrationTest:Module] // @COMPLEXITY: 3 -// @SEMANTICS: tests, profile, integration, preload, cancel, reload -// @PURPOSE: Verifies profile settings preload, cancel without persistence, and saved-state reload behavior. +// @SEMANTICS: tests, profile, integration, load, change, save +// @PURPOSE: Verifies profile loads preferences, allows changes, and saves correctly. // @LAYER: UI (Tests) // @RELATION: DEPENDS_ON -> [ProfilePage] import { describe, it, expect, vi, beforeEach } from "vitest"; import { render, screen, fireEvent, waitFor } from "@testing-library/svelte"; -import { writable } from "svelte/store"; - import ProfilePage from "../+page.svelte"; import { api } from "$lib/api.js"; -import { initializeEnvironmentContext } from "$lib/stores/environmentContext.js"; +import { addToast } from "$lib/toasts"; const mockedApi = /** @type {any} */ (api); -const mockedInitializeEnvironmentContext = - /** @type {any} */ (initializeEnvironmentContext); +const mockedAddToast = /** @type {any} */ (addToast); vi.mock("$lib/api.js", () => ({ api: { getProfilePreferences: vi.fn(), - lookupSupersetAccounts: vi.fn(), - updateProfilePreferences: vi.fn(), + postApi: vi.fn(), }, })); @@ -29,45 +25,20 @@ vi.mock("$lib/toasts", () => ({ addToast: vi.fn(), })); -vi.mock("$lib/stores/environmentContext.js", () => { - const context = writable({ - environments: [{ id: "dev", name: "Development" }], - selectedEnvId: "dev", - isLoading: false, - isLoaded: true, - error: null, - }); - - return { - environmentContextStore: { subscribe: context.subscribe }, - initializeEnvironmentContext: vi.fn().mockResolvedValue(undefined), - }; -}); +vi.mock("$lib/auth/store.js", () => ({ + auth: { subscribe: (run) => { run({ current: { user: { id: "u-1" } } }); return () => {}; } }, +})); vi.mock("$lib/i18n", () => ({ t: { subscribe: (run) => { run({ - common: { cancel: "Cancel" }, + common: { save: "Save", cancel: "Cancel" }, + nav: { profile: "Profile", dashboards: "Dashboards", datasets: "Datasets", reports: "Reports" }, profile: { - title: "Profile", - description: "Manage your profile preferences, Git integration, and access view.", - dashboard_preferences: "Dashboard Preferences", - superset_environment: "Superset Environment", - superset_environment_placeholder: "Select environment", - superset_account: "Your Apache Superset Account", - superset_account_placeholder: "Enter your Apache Superset username", - lookup_loading: "Loading Superset accounts...", - lookup_error: - "Cannot load Superset accounts for this environment right now. You can enter username manually.", - lookup_button: "Lookup", - lookup_search_placeholder: "Search account candidates", - lookup_no_results: - "No account candidates found. You can enter username manually.", - show_only_my_dashboards: "Show only my dashboards by default", - save_preferences: "Save Preferences", - save_success: "Preferences saved", - save_error: "Failed to save preferences. Please try again.", + saved: "Preferences saved", + preferences: "Preferences", + start_page: "Start page", }, }); return () => {}; @@ -79,194 +50,48 @@ vi.mock("$lib/i18n", () => ({ describe("profile-settings-state.integration", () => { beforeEach(() => { vi.clearAllMocks(); - mockedApi.lookupSupersetAccounts.mockResolvedValue({ - status: "success", - environment_id: "dev", - page_index: 0, - page_size: 20, - total: 0, - items: [], - }); - }); - - it("preloads saved values and cancel restores snapshot without persistence", async () => { mockedApi.getProfilePreferences.mockResolvedValue({ status: "success", preference: { user_id: "u-1", - superset_username: "john_doe", - show_only_my_dashboards: true, - git_username: null, - git_email: null, - has_git_personal_access_token: false, - git_personal_access_token_masked: null, start_page: "dashboards", - auto_open_task_drawer: true, - dashboards_table_density: "comfortable", - }, - security: { - read_only: true, - auth_source: "adfs", - current_role: "Data Engineer", - role_source: "adfs", - roles: ["Data Engineer"], - permissions: [], }, }); - - render(ProfilePage); - - await waitFor(() => { - expect(mockedInitializeEnvironmentContext).toHaveBeenCalledTimes(1); - expect(mockedApi.getProfilePreferences).toHaveBeenCalledTimes(1); - }); - - const usernameInput = /** @type {HTMLInputElement} */ ( - screen.getByPlaceholderText("Enter your Apache Superset username") - ); - const toggleInput = /** @type {HTMLInputElement} */ ( - screen.getByLabelText("Show only my dashboards by default") - ); - - expect(usernameInput.value).toBe("john_doe"); - expect(toggleInput.checked).toBe(true); - - await fireEvent.input(usernameInput, { target: { value: "temp_user" } }); - await fireEvent.click(toggleInput); // true -> false - expect(usernameInput.value).toBe("temp_user"); - expect(toggleInput.checked).toBe(false); - - await fireEvent.click(screen.getByRole("button", { name: "Cancel" })); - - await waitFor(() => { - expect(usernameInput.value).toBe("john_doe"); - expect(toggleInput.checked).toBe(true); - }); - - expect(mockedApi.updateProfilePreferences).not.toHaveBeenCalled(); }); - it("reloads persisted values after successful save", async () => { - mockedApi.getProfilePreferences - .mockResolvedValueOnce({ - status: "success", - preference: { - user_id: "u-1", - superset_username: "john_doe", - show_only_my_dashboards: true, - git_username: null, - git_email: null, - has_git_personal_access_token: false, - git_personal_access_token_masked: null, - start_page: "dashboards", - auto_open_task_drawer: true, - dashboards_table_density: "comfortable", - }, - security: { - read_only: true, - auth_source: "adfs", - current_role: "Data Engineer", - role_source: "adfs", - roles: ["Data Engineer"], - permissions: [], - }, - }) - .mockResolvedValueOnce({ - status: "success", - preference: { - user_id: "u-1", - superset_username: "new_user", - show_only_my_dashboards: true, - git_username: null, - git_email: null, - has_git_personal_access_token: false, - git_personal_access_token_masked: null, - start_page: "dashboards", - auto_open_task_drawer: true, - dashboards_table_density: "comfortable", - }, - security: { - read_only: true, - auth_source: "adfs", - current_role: "Data Engineer", - role_source: "adfs", - roles: ["Data Engineer"], - permissions: [], - }, - }); - - mockedApi.updateProfilePreferences.mockResolvedValue({ - status: "success", - preference: { - user_id: "u-1", - superset_username: "new_user", - show_only_my_dashboards: true, - git_username: null, - git_email: null, - has_git_personal_access_token: false, - git_personal_access_token_masked: null, - start_page: "dashboards", - auto_open_task_drawer: true, - dashboards_table_density: "comfortable", - }, - security: { - read_only: true, - auth_source: "adfs", - current_role: "Data Engineer", - role_source: "adfs", - roles: ["Data Engineer"], - permissions: [], - }, - }); - - const firstRender = render(ProfilePage); + it("loads saved preferences and displays them", async () => { + render(ProfilePage); await waitFor(() => { expect(mockedApi.getProfilePreferences).toHaveBeenCalledTimes(1); }); - const usernameInput = /** @type {HTMLInputElement} */ ( - screen.getByPlaceholderText("Enter your Apache Superset username") - ); - await fireEvent.input(usernameInput, { target: { value: "new_user" } }); + const select = /** @type {HTMLSelectElement} */ (screen.getByRole("combobox")); + expect(select.value).toBe("dashboards"); + expect(screen.getByText("Profile")).toBeDefined(); + }); - await fireEvent.click( - screen.getByRole("button", { name: "Save Preferences" }), - ); - - await waitFor(() => { - expect(mockedApi.updateProfilePreferences).toHaveBeenCalledWith({ - superset_username: "new_user", - show_only_my_dashboards: true, - show_only_slug_dashboards: true, - git_username: null, - git_email: null, - start_page: "dashboards", - auto_open_task_drawer: true, - dashboards_table_density: "comfortable", - telegram_id: null, - email_address: null, - notify_on_fail: true, - }); - }); - - firstRender.unmount(); + it("saves changed preferences and shows success toast", async () => { + mockedApi.postApi.mockResolvedValue({ status: "success" }); render(ProfilePage); await waitFor(() => { - expect(mockedApi.getProfilePreferences).toHaveBeenCalledTimes(2); + expect(mockedApi.getProfilePreferences).toHaveBeenCalledTimes(1); }); - const reloadedInput = /** @type {HTMLInputElement} */ ( - screen.getByPlaceholderText("Enter your Apache Superset username") - ); - const reloadedToggle = /** @type {HTMLInputElement} */ ( - screen.getByLabelText("Show only my dashboards by default") - ); + const select = /** @type {HTMLSelectElement} */ (screen.getByRole("combobox")); + await fireEvent.change(select, { target: { value: "reports" } }); - expect(reloadedInput.value).toBe("new_user"); - expect(reloadedToggle.checked).toBe(true); + await fireEvent.click(screen.getByText("Save")); + + await waitFor(() => { + expect(mockedApi.postApi).toHaveBeenCalledWith("/profile/preferences", { + preference: expect.objectContaining({ start_page: "reports" }), + }); + }); + + expect(mockedAddToast).toHaveBeenCalledWith("Preferences saved", "success"); }); }); diff --git a/frontend/src/routes/reports/llm/[taskId]/+page.svelte b/frontend/src/routes/reports/llm/[taskId]/+page.svelte index 63390bf5..9b2e7694 100644 --- a/frontend/src/routes/reports/llm/[taskId]/+page.svelte +++ b/frontend/src/routes/reports/llm/[taskId]/+page.svelte @@ -3,7 +3,6 @@ /+page.svelte:Component] -->
diff --git a/frontend/src/routes/settings/git/+page.svelte b/frontend/src/routes/settings/git/+page.svelte index adfe7731..ef44c197 100644 --- a/frontend/src/routes/settings/git/+page.svelte +++ b/frontend/src/routes/settings/git/+page.svelte @@ -2,7 +2,6 @@