From 674a8693622a71bdc7e21c8b3c28ecd987812aa7 Mon Sep 17 00:00:00 2001 From: busya Date: Sun, 15 Mar 2026 20:29:11 +0300 Subject: [PATCH] mcp --- .ai/MODULE_MAP.md | 27 +- .ai/PROJECT_MAP.md | 33 +- .kilocode/mcp.json | 2 +- backend/src/plugins/migration.py | 639 +++++++++++++++---------------- semantics/semantic_map.json | 524 ++++++++++++------------- 5 files changed, 578 insertions(+), 647 deletions(-) diff --git a/.ai/MODULE_MAP.md b/.ai/MODULE_MAP.md index 6108ff5f..bb5ed808 100644 --- a/.ai/MODULE_MAP.md +++ b/.ai/MODULE_MAP.md @@ -2,7 +2,7 @@ > High-level module structure for AI Context. Generated automatically. -**Generated:** 2026-03-15T16:33:41.022837 +**Generated:** 2026-03-15T16:48:04.692685 ## Summary @@ -124,10 +124,10 @@ ### πŸ“ `core/` - - πŸ—οΈ **Layers:** Core, Domain - - πŸ“Š **Tiers:** CRITICAL: 10, STANDARD: 149, TRIVIAL: 9 - - πŸ“„ **Files:** 13 - - πŸ“¦ **Entities:** 168 + - πŸ—οΈ **Layers:** Core, Domain, Infra + - πŸ“Š **Tiers:** CRITICAL: 11, STANDARD: 150, TRIVIAL: 9 + - πŸ“„ **Files:** 14 + - πŸ“¦ **Entities:** 170 **Key Entities:** @@ -411,9 +411,9 @@ ### πŸ“ `plugins/` - πŸ—οΈ **Layers:** App, Plugin, Plugins - - πŸ“Š **Tiers:** CRITICAL: 2, STANDARD: 62 + - πŸ“Š **Tiers:** CRITICAL: 2, STANDARD: 61 - πŸ“„ **Files:** 7 - - πŸ“¦ **Entities:** 64 + - πŸ“¦ **Entities:** 63 **Key Entities:** @@ -426,7 +426,7 @@ - β„‚ **MapperPlugin** (Class) - Plugin for mapping dataset columns verbose names. - β„‚ **MigrationPlugin** (Class) `[CRITICAL]` - - Implementation of the migration plugin logic. + - Implementation of the migration plugin workflow and transfor... - β„‚ **SearchPlugin** (Class) - Plugin for searching text patterns in Superset datasets. - πŸ“¦ **BackupPlugin** (Module) @@ -436,13 +436,15 @@ - πŸ“¦ **MapperPluginModule** (Module) - Implements a plugin for mapping dataset columns using extern... - πŸ“¦ **MigrationPlugin** (Module) `[CRITICAL]` - - A plugin that provides functionality to migrate Superset das... + - Orchestrates export, DB-mapping transformation, and import o... **Dependencies:** + - πŸ”— DEPENDS_ON -> IdMappingService + - πŸ”— DEPENDS_ON -> MigrationEngine + - πŸ”— DEPENDS_ON -> SupersetClient - πŸ”— DEPENDS_ON -> superset_tool.client - πŸ”— DEPENDS_ON -> superset_tool.utils - - πŸ”— IMPLEMENTS -> PluginBase ### πŸ“ `git/` @@ -574,9 +576,9 @@ ### πŸ“ `scripts/` - πŸ—οΈ **Layers:** Scripts, UI, Unknown - - πŸ“Š **Tiers:** CRITICAL: 1, STANDARD: 46, TRIVIAL: 30 + - πŸ“Š **Tiers:** CRITICAL: 1, STANDARD: 45, TRIVIAL: 30 - πŸ“„ **Files:** 9 - - πŸ“¦ **Entities:** 77 + - πŸ“¦ **Entities:** 76 **Key Entities:** @@ -2142,6 +2144,7 @@ graph TD core-->|DEPENDS_ON|backend core-->|DEPENDS_ON|backend core-->|DEPENDS_ON|backend + core-->|DEPENDS_ON|backend __tests__-->|TESTS|backend auth-->|USES|backend auth-->|USES|backend diff --git a/.ai/PROJECT_MAP.md b/.ai/PROJECT_MAP.md index 1c2b68b5..98f7b49b 100644 --- a/.ai/PROJECT_MAP.md +++ b/.ai/PROJECT_MAP.md @@ -2442,8 +2442,6 @@ - πŸ—οΈ Layer: Scripts - πŸ”’ Invariant: Safe to run multiple times (idempotent). - πŸ”— CALLS -> `backend.src.core.database.init_db` - - Ζ’ **ensure_encryption_key** (`Function`) - - πŸ“ Ensure backend runtime has a persistent Fernet encryption key during first-time installation. - Ζ’ **run_init** (`Function`) - πŸ“ Main entry point for the initialization script. - πŸ“¦ **backend.src.scripts.create_admin** (`Module`) @@ -2636,6 +2634,13 @@ - πŸ“ Auto-detected function (orphan) - Ζ’ **as_bool** (`Function`) `[TRIVIAL]` - πŸ“ Auto-detected function (orphan) +- πŸ“¦ **backend.src.core.encryption_key** (`Module`) `[CRITICAL]` + - πŸ“ Resolve and persist the Fernet encryption key required by runtime services. + - πŸ—οΈ Layer: Infra + - πŸ”’ Invariant: Runtime key resolution never falls back to an ephemeral secret. + - πŸ”— DEPENDS_ON -> `backend.src.core.logger` + - Ζ’ **ensure_encryption_key** (`Function`) + - πŸ“ Ensure backend runtime has a persistent valid Fernet key. - πŸ“¦ **ConfigManagerModule** (`Module`) `[CRITICAL]` - πŸ“ Manages application configuration persistence in DB with one-time migration from legacy JSON. - πŸ—οΈ Layer: Domain @@ -6522,29 +6527,29 @@ - Ζ’ **_get_env** (`Function`) - πŸ“ Π’ΡΠΏΠΎΠΌΠΎΠ³Π°Ρ‚Π΅Π»ΡŒΠ½Ρ‹ΠΉ ΠΌΠ΅Ρ‚ΠΎΠ΄ для получСния ΠΊΠΎΠ½Ρ„ΠΈΠ³ΡƒΡ€Π°Ρ†ΠΈΠΈ окруТСния. - πŸ“¦ **MigrationPlugin** (`Module`) `[CRITICAL]` - - πŸ“ A plugin that provides functionality to migrate Superset dashboards between environments. + - πŸ“ Orchestrates export, DB-mapping transformation, and import of Superset dashboards across environments. - πŸ—οΈ Layer: App + - πŸ”’ Invariant: Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution. - πŸ”— IMPLEMENTS -> `PluginBase` - - πŸ”— DEPENDS_ON -> `superset_tool.client` - - πŸ”— DEPENDS_ON -> `superset_tool.utils` + - πŸ”— DEPENDS_ON -> `SupersetClient` + - πŸ”— DEPENDS_ON -> `MigrationEngine` + - πŸ”— DEPENDS_ON -> `IdMappingService` - β„‚ **MigrationPlugin** (`Class`) `[CRITICAL]` - - πŸ“ Implementation of the migration plugin logic. + - πŸ“ Implementation of the migration plugin workflow and transformation orchestration. - Ζ’ **id** (`Function`) - πŸ“ Returns the unique identifier for the migration plugin. - Ζ’ **name** (`Function`) - - πŸ“ Returns the human-readable name of the migration plugin. + - πŸ“ Returns the human-readable name of the plugin. - Ζ’ **description** (`Function`) - - πŸ“ Returns a description of the migration plugin. + - πŸ“ Returns the semantic description of the plugin. - Ζ’ **version** (`Function`) - - πŸ“ Returns the version of the migration plugin. + - πŸ“ Returns the semantic version of the migration plugin. - Ζ’ **ui_route** (`Function`) - - πŸ“ Returns the frontend route for the migration plugin. + - πŸ“ Returns the frontend routing anchor for the plugin. - Ζ’ **get_schema** (`Function`) - - πŸ“ Returns the JSON schema for migration plugin parameters. + - πŸ“ Generates the JSON Schema for the plugin execution form dynamically. - Ζ’ **execute** (`Function`) - - πŸ“ Executes the dashboard migration logic with TaskContext support. - - πŸ“¦ **MigrationPlugin.execute** (`Action`) - - πŸ“ Execute the migration logic with proper task logging. + - πŸ“ Orchestrates the dashboard migration pipeline including extraction, AST mutation, and ingestion. - πŸ“¦ **src.plugins** (`Package`) - πŸ“ Plugin package root for dynamic discovery and runtime imports. - Ζ’ **schedule_dashboard_validation** (`Function`) diff --git a/.kilocode/mcp.json b/.kilocode/mcp.json index 6b0a486d..fe2d5fab 100644 --- a/.kilocode/mcp.json +++ b/.kilocode/mcp.json @@ -1 +1 @@ -{"mcpServers":{}} \ No newline at end of file +{"mcpServers":{"axiom-core":{"command":"/home/busya/dev/ast-mcp-core-server/.venv/bin/python","args":["-c","from src.server import main; main()"],"env":{"PYTHONPATH":"/home/busya/dev/ast-mcp-core-server"},"alwaysAllow":["reindex_workspace_tool","search_contracts_tool","read_grace_outline_tool","ast_search_tool","get_semantic_context_tool","build_task_context_tool","workspace_semantic_health_tool","audit_contracts_tool","diff_contract_semantics_tool","impact_analysis_tool","simulate_patch_tool","patch_contract_tool","guarded_patch_contract_tool","rename_contract_id_tool","move_contract_tool","extract_contract_tool","infer_missing_relations_tool","map_runtime_trace_to_contracts_tool","trace_tests_for_contract_tool","scaffold_contract_tests_tool"]}}} \ No newline at end of file diff --git a/backend/src/plugins/migration.py b/backend/src/plugins/migration.py index aeb87d11..2ffc3c5f 100755 --- a/backend/src/plugins/migration.py +++ b/backend/src/plugins/migration.py @@ -1,11 +1,14 @@ # [DEF:MigrationPlugin:Module] -# @SEMANTICS: migration, superset, automation, dashboard, plugin -# @PURPOSE: A plugin that provides functionality to migrate Superset dashboards between environments. +# @TIER: CRITICAL +# @SEMANTICS: migration, superset, automation, dashboard, plugin, transformation +# @PURPOSE: Orchestrates export, DB-mapping transformation, and import of Superset dashboards across environments. # @LAYER: App # @RELATION: IMPLEMENTS -> PluginBase -# @RELATION: DEPENDS_ON -> superset_tool.client -# @RELATION: DEPENDS_ON -> superset_tool.utils +# @RELATION: DEPENDS_ON -> SupersetClient +# @RELATION: DEPENDS_ON -> MigrationEngine +# @RELATION: DEPENDS_ON -> IdMappingService # @RELATION: USES -> TaskContext +# @INVARIANT: Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution. from typing import Dict, Any, Optional import re @@ -22,7 +25,13 @@ from ..core.mapping_service import IdMappingService from ..core.task_manager.context import TaskContext # [DEF:MigrationPlugin:Class] -# @PURPOSE: Implementation of the migration plugin logic. +# @PURPOSE: Implementation of the migration plugin workflow and transformation orchestration. +# @PRE: Plugin loader must register this instance. +# @POST: Provides migration UI schema and executes atomic dashboard transfers. +# @TEST_FIXTURE: superset_export_zip -> file:backend/tests/fixtures/migration/dashboard_export.zip +# @TEST_FIXTURE: db_mapping_payload -> INLINE_JSON: {"db_mappings": {"source_uuid_1": "target_uuid_2"}} +# @TEST_FIXTURE: password_inject_payload -> INLINE_JSON: {"passwords": {"PostgreSQL": "secret123"}} +# @TEST_INVARIANT: strict_db_isolation -> VERIFIED_BY: [successful_dashboard_transfer, missing_mapping_resolution] class MigrationPlugin(PluginBase): """ A plugin to migrate Superset dashboards between environments. @@ -31,377 +40,345 @@ class MigrationPlugin(PluginBase): @property # [DEF:id:Function] # @PURPOSE: Returns the unique identifier for the migration plugin. - # @PRE: None. - # @POST: Returns "superset-migration". - # @RETURN: str - "superset-migration" + # @PRE: None. + # @POST: Returns stable string "superset-migration". + # @RETURN: str def id(self) -> str: - with belief_scope("id"): + with belief_scope("MigrationPlugin.id"): return "superset-migration" # [/DEF:id:Function] @property # [DEF:name:Function] - # @PURPOSE: Returns the human-readable name of the migration plugin. - # @PRE: None. - # @POST: Returns the plugin name. - # @RETURN: str - Plugin name. + # @PURPOSE: Returns the human-readable name of the plugin. + # @PRE: None. + # @POST: Returns "Superset Dashboard Migration". + # @RETURN: str def name(self) -> str: - with belief_scope("name"): + with belief_scope("MigrationPlugin.name"): return "Superset Dashboard Migration" # [/DEF:name:Function] @property # [DEF:description:Function] - # @PURPOSE: Returns a description of the migration plugin. - # @PRE: None. - # @POST: Returns the plugin description. - # @RETURN: str - Plugin description. + # @PURPOSE: Returns the semantic description of the plugin. + # @PRE: None. + # @POST: Returns description string. + # @RETURN: str def description(self) -> str: - with belief_scope("description"): + with belief_scope("MigrationPlugin.description"): return "Migrates dashboards between Superset environments." # [/DEF:description:Function] @property # [DEF:version:Function] - # @PURPOSE: Returns the version of the migration plugin. - # @PRE: None. - # @POST: Returns "1.0.0". - # @RETURN: str - "1.0.0" + # @PURPOSE: Returns the semantic version of the migration plugin. + # @PRE: None. + # @POST: Returns "1.0.0". + # @RETURN: str def version(self) -> str: - with belief_scope("version"): + with belief_scope("MigrationPlugin.version"): return "1.0.0" # [/DEF:version:Function] @property # [DEF:ui_route:Function] - # @PURPOSE: Returns the frontend route for the migration plugin. - # @RETURN: str - "/migration" + # @PURPOSE: Returns the frontend routing anchor for the plugin. + # @PRE: None. + # @POST: Returns "/migration". + # @RETURN: str def ui_route(self) -> str: - with belief_scope("ui_route"): + with belief_scope("MigrationPlugin.ui_route"): return "/migration" # [/DEF:ui_route:Function] # [DEF:get_schema:Function] - # @PURPOSE: Returns the JSON schema for migration plugin parameters. - # @PRE: Config manager is available. - # @POST: Returns a valid JSON schema dictionary. - # @RETURN: Dict[str, Any] - JSON schema. + # @PURPOSE: Generates the JSON Schema for the plugin execution form dynamically. + # @PRE: ConfigManager is accessible and environments are defined. + # @POST: Returns a JSON Schema dict matching current system environments. + # @RETURN: Dict[str, Any] def get_schema(self) -> Dict[str, Any]: - with belief_scope("get_schema"): + with belief_scope("MigrationPlugin.get_schema"): + app_logger.reason("Generating migration UI schema") config_manager = get_config_manager() - envs = [e.name for e in config_manager.get_environments()] - - return { - "type": "object", - "properties": { - "from_env": { - "type": "string", - "title": "Source Environment", - "description": "The environment to migrate from.", - "enum": envs if envs else ["dev", "prod"], + envs = [e.name for e in config_manager.get_environments()] + + schema = { + "type": "object", + "properties": { + "from_env": { + "type": "string", + "title": "Source Environment", + "description": "The environment to migrate from.", + "enum": envs if envs else ["dev", "prod"], + }, + "to_env": { + "type": "string", + "title": "Target Environment", + "description": "The environment to migrate to.", + "enum": envs if envs else ["dev", "prod"], + }, + "dashboard_regex": { + "type": "string", + "title": "Dashboard Regex", + "description": "A regular expression to filter dashboards to migrate.", + }, + "replace_db_config": { + "type": "boolean", + "title": "Replace DB Config", + "description": "Whether to replace the database configuration.", + "default": False, + }, + "from_db_id": { + "type": "integer", + "title": "Source DB ID", + "description": "The ID of the source database to replace (if replacing).", + }, + "to_db_id": { + "type": "integer", + "title": "Target DB ID", + "description": "The ID of the target database to replace with (if replacing).", + }, }, - "to_env": { - "type": "string", - "title": "Target Environment", - "description": "The environment to migrate to.", - "enum": envs if envs else ["dev", "prod"], - }, - "dashboard_regex": { - "type": "string", - "title": "Dashboard Regex", - "description": "A regular expression to filter dashboards to migrate.", - }, - "replace_db_config": { - "type": "boolean", - "title": "Replace DB Config", - "description": "Whether to replace the database configuration.", - "default": False, - }, - "from_db_id": { - "type": "integer", - "title": "Source DB ID", - "description": "The ID of the source database to replace (if replacing).", - }, - "to_db_id": { - "type": "integer", - "title": "Target DB ID", - "description": "The ID of the target database to replace with (if replacing).", - }, - }, - "required": ["from_env", "to_env", "dashboard_regex"], - } + "required": ["from_env", "to_env", "dashboard_regex"], + } + app_logger.reflect("Schema generated successfully", extra={"environments_count": len(envs)}) + return schema # [/DEF:get_schema:Function] # [DEF:execute:Function] - # @PURPOSE: Executes the dashboard migration logic with TaskContext support. - # @PARAM: params (Dict[str, Any]) - Migration parameters. - # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution. - # @PRE: Source and target environments must be configured. - # @POST: Selected dashboards are migrated. + # @PURPOSE: Orchestrates the dashboard migration pipeline including extraction, AST mutation, and ingestion. + # @PARAM: params (Dict[str, Any]) - Extracted parameters from UI/API execution request. + # @PARAM: context (Optional[TaskContext]) - Dependency injected TaskContext for IO tracing. + # @PRE: Source and target environments must resolve. Matching dashboards must exist. + # @POST: Dashboard ZIP bundles are transformed and imported. ID mappings are synchronized. + # @SIDE_EFFECT: Creates temp files, mutates target Superset state, blocks on user input (passwords/mappings). + # @TEST_CONTRACT: Dict[str, Any] -> Dict[str, Any] + # @TEST_SCENARIO: successful_dashboard_transfer -> ZIP is downloaded, DB mappings applied via AST, target import succeeds. + # @TEST_SCENARIO: missing_password_injection -> Target import fails on auth, TaskManager pauses for user input, retries with password successfully. + # @TEST_SCENARIO: empty_selection -> Returns NO_MATCHES gracefully when regex finds zero dashboards. + # @TEST_EDGE: missing_env_field -> [ValueError: Could not resolve source or target environment] + # @TEST_EDGE: invalid_regex_pattern -> [Regex compilation exception is thrown or caught gracefully] + # @TEST_EDGE: target_api_timeout -> [Dashboard added to failed_dashboards, task concludes with PARTIAL_SUCCESS] async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None): with belief_scope("MigrationPlugin.execute"): + app_logger.reason("Evaluating migration task parameters", extra={"params": params}) + source_env_id = params.get("source_env_id") target_env_id = params.get("target_env_id") selected_ids = params.get("selected_ids") - - # Legacy support or alternative params - from_env_name = params.get("from_env") - to_env_name = params.get("to_env") - dashboard_regex = params.get("dashboard_regex") - - replace_db_config = params.get("replace_db_config", False) - fix_cross_filters = params.get("fix_cross_filters", True) - params.get("from_db_id") - params.get("to_db_id") - - # [DEF:MigrationPlugin.execute:Action] - # @PURPOSE: Execute the migration logic with proper task logging. - task_id = params.get("_task_id") - from ..dependencies import get_task_manager - tm = get_task_manager() - - # Use TaskContext logger if available, otherwise fall back to app_logger - log = context.logger if context else app_logger - - # Create sub-loggers for different components - superset_log = log.with_source("superset_api") if context else log - migration_log = log.with_source("migration") if context else log - - log.info("Starting migration task.") - log.debug(f"Params: {params}") - - try: - with belief_scope("execute"): - config_manager = get_config_manager() - environments = config_manager.get_environments() - # Resolve environments - src_env = None - tgt_env = None + from_env_name = params.get("from_env") + to_env_name = params.get("to_env") + dashboard_regex = params.get("dashboard_regex") + replace_db_config = params.get("replace_db_config", False) + fix_cross_filters = params.get("fix_cross_filters", True) - if source_env_id: - src_env = next((e for e in environments if e.id == source_env_id), None) - elif from_env_name: - src_env = next((e for e in environments if e.name == from_env_name), None) - - if target_env_id: - tgt_env = next((e for e in environments if e.id == target_env_id), None) - elif to_env_name: - tgt_env = next((e for e in environments if e.name == to_env_name), None) + task_id = params.get("_task_id") + from ..dependencies import get_task_manager + tm = get_task_manager() + + log = context.logger if context else app_logger + superset_log = log.with_source("superset_api") if context else log + migration_log = log.with_source("migration") if context else log - if not src_env or not tgt_env: - raise ValueError(f"Could not resolve source or target environment. Source: {source_env_id or from_env_name}, Target: {target_env_id or to_env_name}") + log.info("Starting migration task.") - from_env_name = src_env.name - to_env_name = tgt_env.name - - log.info(f"Resolved environments: {from_env_name} -> {to_env_name}") - migration_result = { - "status": "SUCCESS", - "source_environment": from_env_name, - "target_environment": to_env_name, - "selected_dashboards": 0, - "migrated_dashboards": [], - "failed_dashboards": [], - "mapping_count": 0 - } - - from_c = SupersetClient(src_env) - to_c = SupersetClient(tgt_env) - - if not from_c or not to_c: - raise ValueError(f"Clients not initialized for environments: {from_env_name}, {to_env_name}") - - _, all_dashboards = from_c.get_dashboards() - - dashboards_to_migrate = [] - if selected_ids: - dashboards_to_migrate = [d for d in all_dashboards if d["id"] in selected_ids] - elif dashboard_regex: - regex_pattern = re.compile(str(dashboard_regex), re.IGNORECASE) - dashboards_to_migrate = [ - d for d in all_dashboards if regex_pattern.search(d.get("dashboard_title", "")) - ] - else: - log.warning("No selection criteria provided (selected_ids or dashboard_regex).") - migration_result["status"] = "NO_SELECTION" - return migration_result - - if not dashboards_to_migrate: - log.warning("No dashboards found matching criteria.") - migration_result["status"] = "NO_MATCHES" - return migration_result - - migration_result["selected_dashboards"] = len(dashboards_to_migrate) - - # Get mappings from params - db_mapping = params.get("db_mappings", {}) - if not isinstance(db_mapping, dict): - db_mapping = {} - - # Fetch additional mappings from database if requested - if replace_db_config: - db = SessionLocal() - try: - # Find environment IDs by name - src_env_db = db.query(Environment).filter(Environment.name == from_env_name).first() - tgt_env_db = db.query(Environment).filter(Environment.name == to_env_name).first() - - if src_env_db and tgt_env_db: - stored_mappings = db.query(DatabaseMapping).filter( - DatabaseMapping.source_env_id == src_env_db.id, - DatabaseMapping.target_env_id == tgt_env_db.id - ).all() - # Provided mappings override stored ones - stored_map_dict = {m.source_db_uuid: m.target_db_uuid for m in stored_mappings} - stored_map_dict.update(db_mapping) - db_mapping = stored_map_dict - log.info(f"Loaded {len(stored_mappings)} database mappings from database.") - finally: - db.close() - - migration_result["mapping_count"] = len(db_mapping) - engine = MigrationEngine() - - for dash in dashboards_to_migrate: - dash_id, dash_slug, title = dash["id"], dash.get("slug"), dash["dashboard_title"] - - try: - exported_content, _ = from_c.export_dashboard(dash_id) - with create_temp_file(content=exported_content, dry_run=True, suffix=".zip") as tmp_zip_path: - # Always transform to strip databases to avoid password errors - with create_temp_file(suffix=".zip", dry_run=True) as tmp_new_zip: - success = engine.transform_zip( - str(tmp_zip_path), - str(tmp_new_zip), - db_mapping, - strip_databases=False, - target_env_id=tgt_env.id if tgt_env else None, - fix_cross_filters=fix_cross_filters - ) - - if not success and replace_db_config: - # Signal missing mapping and wait (only if we care about mappings) - if task_id: - log.info(f"Pausing for missing mapping in task {task_id}") - # In a real scenario, we'd pass the missing DB info to the frontend - # For this task, we'll just simulate the wait - await tm.wait_for_resolution(task_id) - # After resolution, retry transformation with updated mappings - # (Mappings would be updated in task.params by resolve_task) - db = SessionLocal() - try: - src_env_rt = db.query(Environment).filter(Environment.name == from_env_name).first() - tgt_env_rt = db.query(Environment).filter(Environment.name == to_env_name).first() - mappings = db.query(DatabaseMapping).filter( - DatabaseMapping.source_env_id == src_env_rt.id, - DatabaseMapping.target_env_id == tgt_env_rt.id - ).all() - db_mapping = {m.source_db_uuid: m.target_db_uuid for m in mappings} - finally: - db.close() - success = engine.transform_zip( - str(tmp_zip_path), - str(tmp_new_zip), - db_mapping, - strip_databases=False, - target_env_id=tgt_env.id if tgt_env else None, - fix_cross_filters=fix_cross_filters - ) - - if success: - to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug) - migration_result["migrated_dashboards"].append({ - "id": dash_id, - "title": title - }) - else: - migration_log.error(f"Failed to transform ZIP for dashboard {title}") - migration_result["failed_dashboards"].append({ - "id": dash_id, - "title": title, - "error": "Failed to transform ZIP" - }) - - superset_log.info(f"Dashboard {title} imported.") - except Exception as exc: - # Check for password error - error_msg = str(exc) - # The error message from Superset is often a JSON string inside a string. - # We need to robustly detect the password requirement. - # Typical error: "Error importing dashboard: databases/PostgreSQL.yaml: {'_schema': ['Must provide a password for the database']}" - - if "Must provide a password for the database" in error_msg: - # Extract database name - # Try to find "databases/DBNAME.yaml" pattern - import re - db_name = "unknown" - match = re.search(r"databases/([^.]+)\.yaml", error_msg) - if match: - db_name = match.group(1) - else: - # Fallback: try to find 'database 'NAME'' pattern - match_alt = re.search(r"database '([^']+)'", error_msg) - if match_alt: - db_name = match_alt.group(1) - - app_logger.warning(f"[MigrationPlugin][Action] Detected missing password for database: {db_name}") - - if task_id: - input_request = { - "type": "database_password", - "databases": [db_name], - "error_message": error_msg - } - tm.await_input(task_id, input_request) - - # Wait for user input - await tm.wait_for_input(task_id) - - # Resume with passwords - task = tm.get_task(task_id) - passwords = task.params.get("passwords", {}) - - # Retry import with password - if passwords: - app_logger.info(f"[MigrationPlugin][Action] Retrying import for {title} with provided passwords.") - to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug, passwords=passwords) - app_logger.info(f"[MigrationPlugin][Success] Dashboard {title} imported after password injection.") - migration_result["migrated_dashboards"].append({ - "id": dash_id, - "title": title - }) - # Clear passwords from params after use for security - if "passwords" in task.params: - del task.params["passwords"] - continue - - app_logger.error(f"[MigrationPlugin][Failure] Failed to migrate dashboard {title}: {exc}", exc_info=True) - migration_result["failed_dashboards"].append({ - "id": dash_id, - "title": title, - "error": str(exc) - }) - - app_logger.info("[MigrationPlugin][Exit] Migration finished.") - if migration_result["failed_dashboards"]: - migration_result["status"] = "PARTIAL_SUCCESS" - - # Perform incremental sync to rapidly update local mappings with new imported resources try: - db_session = SessionLocal() - mapping_service = IdMappingService(db_session) - mapping_service.sync_environment(tgt_env.id, to_c, incremental=True) - db_session.close() - log.info(f"[MigrationPlugin][Action] Completed incremental sync for target environment {to_env_name}") - except Exception as sync_exc: - log.error(f"[MigrationPlugin][Error] Failed incremental sync for {to_env_name}: {sync_exc}") + config_manager = get_config_manager() + environments = config_manager.get_environments() + + # Resolve environments + src_env = next((e for e in environments if e.id == source_env_id), None) if source_env_id else next((e for e in environments if e.name == from_env_name), None) + tgt_env = next((e for e in environments if e.id == target_env_id), None) if target_env_id else next((e for e in environments if e.name == to_env_name), None) + + if not src_env or not tgt_env: + app_logger.explore("Environment resolution failed", extra={"src": source_env_id or from_env_name, "tgt": target_env_id or to_env_name}) + raise ValueError(f"Could not resolve source or target environment. Source: {source_env_id or from_env_name}, Target: {target_env_id or to_env_name}") - return migration_result - except Exception as e: - app_logger.critical(f"[MigrationPlugin][Failure] Fatal error during migration: {e}", exc_info=True) - raise e - # [/DEF:MigrationPlugin.execute:Action] + from_env_name = src_env.name + to_env_name = tgt_env.name + + app_logger.reason("Environments resolved successfully", extra={"from": from_env_name, "to": to_env_name}) + + migration_result = { + "status": "SUCCESS", + "source_environment": from_env_name, + "target_environment": to_env_name, + "selected_dashboards": 0, + "migrated_dashboards": [], + "failed_dashboards": [], + "mapping_count": 0 + } + + from_c = SupersetClient(src_env) + to_c = SupersetClient(tgt_env) + + if not from_c or not to_c: + raise ValueError(f"Clients not initialized for environments: {from_env_name}, {to_env_name}") + + _, all_dashboards = from_c.get_dashboards() + + # Selection Logic + if selected_ids: + dashboards_to_migrate = [d for d in all_dashboards if d["id"] in selected_ids] + elif dashboard_regex: + regex_pattern = re.compile(str(dashboard_regex), re.IGNORECASE) + dashboards_to_migrate = [d for d in all_dashboards if regex_pattern.search(d.get("dashboard_title", ""))] + else: + app_logger.explore("No deterministic selection criteria provided") + migration_result["status"] = "NO_SELECTION" + return migration_result + + if not dashboards_to_migrate: + app_logger.explore("Zero dashboards match selection criteria") + migration_result["status"] = "NO_MATCHES" + return migration_result + + migration_result["selected_dashboards"] = len(dashboards_to_migrate) + + # Database Mapping Resolution + db_mapping = params.get("db_mappings", {}) + if not isinstance(db_mapping, dict): + db_mapping = {} + + if replace_db_config: + app_logger.reason("Fetching environment DB mappings from catalog") + db = SessionLocal() + try: + src_env_db = db.query(Environment).filter(Environment.name == from_env_name).first() + tgt_env_db = db.query(Environment).filter(Environment.name == to_env_name).first() + + if src_env_db and tgt_env_db: + stored_mappings = db.query(DatabaseMapping).filter( + DatabaseMapping.source_env_id == src_env_db.id, + DatabaseMapping.target_env_id == tgt_env_db.id + ).all() + stored_map_dict = {m.source_db_uuid: m.target_db_uuid for m in stored_mappings} + stored_map_dict.update(db_mapping) + db_mapping = stored_map_dict + log.info(f"Loaded {len(stored_mappings)} database mappings from database.") + finally: + db.close() + + migration_result["mapping_count"] = len(db_mapping) + engine = MigrationEngine() + + # Migration Loop + for dash in dashboards_to_migrate: + dash_id, dash_slug, title = dash["id"], dash.get("slug"), dash["dashboard_title"] + app_logger.reason(f"Starting pipeline for dashboard '{title}'", extra={"dash_id": dash_id}) + + try: + exported_content, _ = from_c.export_dashboard(dash_id) + with create_temp_file(content=exported_content, dry_run=True, suffix=".zip") as tmp_zip_path: + with create_temp_file(suffix=".zip", dry_run=True) as tmp_new_zip: + + success = engine.transform_zip( + str(tmp_zip_path), + str(tmp_new_zip), + db_mapping, + strip_databases=False, + target_env_id=tgt_env.id if tgt_env else None, + fix_cross_filters=fix_cross_filters + ) + + if not success and replace_db_config: + if task_id: + app_logger.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", extra={"task_id": task_id}) + await tm.wait_for_resolution(task_id) + + app_logger.reason("Task resumed, re-evaluating mapping states") + db = SessionLocal() + try: + src_env_rt = db.query(Environment).filter(Environment.name == from_env_name).first() + tgt_env_rt = db.query(Environment).filter(Environment.name == to_env_name).first() + mappings = db.query(DatabaseMapping).filter( + DatabaseMapping.source_env_id == src_env_rt.id, + DatabaseMapping.target_env_id == tgt_env_rt.id + ).all() + db_mapping = {m.source_db_uuid: m.target_db_uuid for m in mappings} + finally: + db.close() + + success = engine.transform_zip( + str(tmp_zip_path), + str(tmp_new_zip), + db_mapping, + strip_databases=False, + target_env_id=tgt_env.id if tgt_env else None, + fix_cross_filters=fix_cross_filters + ) + + if success: + app_logger.reason("Pushing transformed ZIP to target Superset") + to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug) + migration_result["migrated_dashboards"].append({"id": dash_id, "title": title}) + app_logger.reflect("Import successful", extra={"title": title}) + else: + app_logger.explore("Transformation strictly failed, bypassing ingestion") + migration_log.error(f"Failed to transform ZIP for dashboard {title}") + migration_result["failed_dashboards"].append({ + "id": dash_id, "title": title, "error": "Failed to transform ZIP" + }) + + except Exception as exc: + error_msg = str(exc) + if "Must provide a password for the database" in error_msg: + db_name = "unknown" + match = re.search(r"databases/([^.]+)\.yaml", error_msg) + if match: + db_name = match.group(1) + else: + match_alt = re.search(r"database '([^']+)'", error_msg) + if match_alt: + db_name = match_alt.group(1) + + app_logger.explore(f"Missing DB password detected during ingestion. Escalating to UI.", extra={"db_name": db_name}) + + if task_id: + tm.await_input(task_id, { + "type": "database_password", + "databases": [db_name], + "error_message": error_msg + }) + + await tm.wait_for_input(task_id) + task = tm.get_task(task_id) + passwords = task.params.get("passwords", {}) + + if passwords: + app_logger.reason(f"Retrying import for {title} with injected credentials") + to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug, passwords=passwords) + migration_result["migrated_dashboards"].append({"id": dash_id, "title": title}) + app_logger.reflect("Password injection unblocked import") + if "passwords" in task.params: + del task.params["passwords"] + continue + + app_logger.explore(f"Catastrophic dashboard ingestion failure: {exc}") + migration_result["failed_dashboards"].append({"id": dash_id, "title": title, "error": str(exc)}) + + if migration_result["failed_dashboards"]: + migration_result["status"] = "PARTIAL_SUCCESS" + + # Post-Migration ID Mapping Synchronization + try: + app_logger.reason("Executing incremental ID catalog sync on target") + db_session = SessionLocal() + mapping_service = IdMappingService(db_session) + mapping_service.sync_environment(tgt_env.id, to_c, incremental=True) + db_session.close() + app_logger.reflect("Incremental catalog sync closed out cleanly") + except Exception as sync_exc: + app_logger.explore(f"ID Mapping sync failed, mapping state might be degraded: {sync_exc}") + + app_logger.reflect("Migration cycle fully resolved", extra={"result": migration_result}) + return migration_result + + except Exception as e: + app_logger.explore(f"Fatal plugin failure: {e}", exc_info=True) + raise e # [/DEF:execute:Function] # [/DEF:MigrationPlugin:Class] -# [/DEF:MigrationPlugin:Module] +# [/DEF:MigrationPlugin:Module] \ No newline at end of file diff --git a/semantics/semantic_map.json b/semantics/semantic_map.json index 65076187..247cab15 100644 --- a/semantics/semantic_map.json +++ b/semantics/semantic_map.json @@ -1,6 +1,6 @@ { "project_root": ".", - "generated_at": "2026-03-15T16:33:40.629628", + "generated_at": "2026-03-15T16:48:04.210646", "modules": [ { "name": "generate_semantic_map", @@ -36517,7 +36517,7 @@ "type": "Module", "tier": "CRITICAL", "start_line": 1, - "end_line": 303, + "end_line": 305, "tags": { "TIER": "CRITICAL", "SEMANTICS": "app, main, entrypoint, fastapi", @@ -36532,8 +36532,8 @@ "name": "App", "type": "Global", "tier": "STANDARD", - "start_line": 27, - "end_line": 35, + "start_line": 28, + "end_line": 36, "tags": { "SEMANTICS": "app, fastapi, instance", "PURPOSE": "The global FastAPI application instance." @@ -36550,8 +36550,8 @@ "name": "startup_event", "type": "Function", "tier": "STANDARD", - "start_line": 37, - "end_line": 47, + "start_line": 38, + "end_line": 49, "tags": { "PURPOSE": "Handles application startup tasks, such as starting the scheduler.", "PRE": "None.", @@ -36565,22 +36565,22 @@ { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 37 + "line_number": 38 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 37 + "line_number": 38 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 37 + "line_number": 38 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 37 + "line_number": 38 } ], "score": 0.28 @@ -36590,8 +36590,8 @@ "name": "shutdown_event", "type": "Function", "tier": "STANDARD", - "start_line": 49, - "end_line": 59, + "start_line": 51, + "end_line": 61, "tags": { "PURPOSE": "Handles application shutdown tasks, such as stopping the scheduler.", "PRE": "None.", @@ -36605,22 +36605,22 @@ { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 49 + "line_number": 51 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 49 + "line_number": 51 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 49 + "line_number": 51 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 49 + "line_number": 51 } ], "score": 0.28 @@ -36630,8 +36630,8 @@ "name": "network_error_handler", "type": "Function", "tier": "STANDARD", - "start_line": 75, - "end_line": 89, + "start_line": 77, + "end_line": 91, "tags": { "PURPOSE": "Global exception handler for NetworkError.", "PRE": "request is a FastAPI Request object.", @@ -36646,22 +36646,22 @@ { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 75 + "line_number": 77 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 75 + "line_number": 77 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 75 + "line_number": 77 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 75 + "line_number": 77 } ], "score": 0.28 @@ -36671,8 +36671,8 @@ "name": "log_requests", "type": "Function", "tier": "STANDARD", - "start_line": 91, - "end_line": 117, + "start_line": 93, + "end_line": 119, "tags": { "PURPOSE": "Middleware to log incoming HTTP requests and their response status.", "PRE": "request is a FastAPI Request object.", @@ -36687,22 +36687,22 @@ { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 91 + "line_number": 93 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 91 + "line_number": 93 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 91 + "line_number": 93 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 91 + "line_number": 93 } ], "score": 0.28 @@ -36712,8 +36712,8 @@ "name": "api.include_routers", "type": "Action", "tier": "STANDARD", - "start_line": 142, - "end_line": 146, + "start_line": 144, + "end_line": 148, "tags": { "PURPOSE": "Registers all API routers with the FastAPI application.", "LAYER": "API", @@ -36731,8 +36731,8 @@ "name": "websocket_endpoint", "type": "Function", "tier": "CRITICAL", - "start_line": 148, - "end_line": 264, + "start_line": 150, + "end_line": 266, "tags": { "PURPOSE": "Provides a WebSocket endpoint for real-time log streaming of a task with server-side filtering.", "PRE": "task_id must be a valid task ID.", @@ -36752,32 +36752,32 @@ { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for CRITICAL tier)", "severity": "ERROR", - "line_number": 148 + "line_number": 150 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for CRITICAL tier)", "severity": "ERROR", - "line_number": 148 + "line_number": 150 }, { "message": "Missing Mandatory Tag: @TEST_SCENARIO (required for CRITICAL tier)", "severity": "ERROR", - "line_number": 148 + "line_number": 150 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for CRITICAL tier)", "severity": "ERROR", - "line_number": 148 + "line_number": 150 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for CRITICAL tier)", "severity": "ERROR", - "line_number": 148 + "line_number": 150 }, { "message": "Missing Mandatory Tag: @TEST_SCENARIO (required for CRITICAL tier)", "severity": "ERROR", - "line_number": 148 + "line_number": 150 } ], "score": 0.0 @@ -36787,8 +36787,8 @@ "name": "StaticFiles", "type": "Mount", "tier": "STANDARD", - "start_line": 266, - "end_line": 302, + "start_line": 268, + "end_line": 304, "tags": { "SEMANTICS": "static, frontend, spa", "PURPOSE": "Mounts the frontend build directory to serve static assets." @@ -36799,8 +36799,8 @@ "name": "serve_spa", "type": "Function", "tier": "STANDARD", - "start_line": 273, - "end_line": 291, + "start_line": 275, + "end_line": 293, "tags": { "PURPOSE": "Serves the SPA frontend for any path not matched by API routes.", "PRE": "frontend_path exists.", @@ -36814,32 +36814,32 @@ { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 273 + "line_number": 275 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 273 + "line_number": 275 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 273 + "line_number": 275 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 273 + "line_number": 275 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 273 + "line_number": 275 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 273 + "line_number": 275 } ], "score": 0.0 @@ -36849,8 +36849,8 @@ "name": "read_root", "type": "Function", "tier": "STANDARD", - "start_line": 293, - "end_line": 301, + "start_line": 295, + "end_line": 303, "tags": { "PURPOSE": "A simple root endpoint to confirm that the API is running when frontend is missing.", "PRE": "None.", @@ -36864,32 +36864,32 @@ { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 293 + "line_number": 295 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 293 + "line_number": 295 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 293 + "line_number": 295 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 293 + "line_number": 295 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 293 + "line_number": 295 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 293 + "line_number": 295 } ], "score": 0.0 @@ -36906,8 +36906,8 @@ "name": "matches_filters", "type": "Function", "tier": "TRIVIAL", - "start_line": 198, - "end_line": 198, + "start_line": 200, + "end_line": 200, "tags": { "PURPOSE": "Auto-detected function (orphan)", "TIER": "TRIVIAL" @@ -36925,7 +36925,7 @@ "valid": true, "issues": [ { - "message": "Fractal limit warning: Module length is 303 lines (must be < 300)", + "message": "Fractal limit warning: Module length is 305 lines (must be < 300)", "severity": "WARNING", "line_number": 1 } @@ -40094,7 +40094,7 @@ "type": "Module", "tier": "CRITICAL", "start_line": 1, - "end_line": 81, + "end_line": 45, "tags": { "SEMANTICS": "setup, database, auth, migration", "PURPOSE": "Initializes the auth database and creates the necessary tables.", @@ -40108,62 +40108,12 @@ } ], "children": [ - { - "name": "ensure_encryption_key", - "type": "Function", - "tier": "STANDARD", - "start_line": 27, - "end_line": 57, - "tags": { - "PURPOSE": "Ensure backend runtime has a persistent Fernet encryption key during first-time installation.", - "PRE": "Backend root is writable or ENCRYPTION_KEY is already provided via environment.", - "POST": "ENCRYPTION_KEY exists in process environment or backend/.env." - }, - "relations": [], - "children": [], - "compliance": { - "valid": true, - "issues": [ - { - "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 27 - }, - { - "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 27 - }, - { - "message": "Missing Belief State Logging: Function should use belief_scope / molecular methods (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 27 - }, - { - "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 27 - }, - { - "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 27 - }, - { - "message": "Missing Belief State Logging: Function should use belief_scope / molecular methods (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 27 - } - ], - "score": 0.0 - } - }, { "name": "run_init", "type": "Function", "tier": "STANDARD", - "start_line": 59, - "end_line": 76, + "start_line": 23, + "end_line": 40, "tags": { "PURPOSE": "Main entry point for the initialization script.", "POST": "auth.db is initialized with the correct schema and seeded permissions." @@ -40176,32 +40126,32 @@ { "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", "severity": "WARNING", - "line_number": 59 + "line_number": 23 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 59 + "line_number": 23 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 59 + "line_number": 23 }, { "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", "severity": "WARNING", - "line_number": 59 + "line_number": 23 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 59 + "line_number": 23 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 59 + "line_number": 23 } ], "score": 0.0 @@ -43816,6 +43766,64 @@ "score": 0.28 } }, + { + "name": "backend.src.core.encryption_key", + "type": "Module", + "tier": "CRITICAL", + "start_line": 1, + "end_line": 56, + "tags": { + "TIER": "CRITICAL", + "SEMANTICS": "encryption, key, bootstrap, environment, startup", + "PURPOSE": "Resolve and persist the Fernet encryption key required by runtime services.", + "LAYER": "Infra", + "INVARIANT": "Runtime key resolution never falls back to an ephemeral secret." + }, + "relations": [ + { + "type": "DEPENDS_ON", + "target": "backend.src.core.logger" + } + ], + "children": [ + { + "name": "ensure_encryption_key", + "type": "Function", + "tier": "STANDARD", + "start_line": 21, + "end_line": 54, + "tags": { + "PURPOSE": "Ensure backend runtime has a persistent valid Fernet key.", + "PRE": "env_file_path points to a writable backend .env file or ENCRYPTION_KEY exists in process environment.", + "POST": "Returns a valid Fernet key and guarantees it is present in process environment.", + "SIDE_EFFECT": "May create or append backend/.env when key is missing." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 21 + }, + { + "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 21 + } + ], + "score": 0.64 + } + } + ], + "compliance": { + "valid": true, + "issues": [], + "score": 0.95 + } + }, { "name": "ConfigManagerModule", "type": "Module", @@ -104221,11 +104229,13 @@ "type": "Module", "tier": "CRITICAL", "start_line": 1, - "end_line": 407, + "end_line": 384, "tags": { - "SEMANTICS": "migration, superset, automation, dashboard, plugin", - "PURPOSE": "A plugin that provides functionality to migrate Superset dashboards between environments.", - "LAYER": "App" + "TIER": "CRITICAL", + "SEMANTICS": "migration, superset, automation, dashboard, plugin, transformation", + "PURPOSE": "Orchestrates export, DB-mapping transformation, and import of Superset dashboards across environments.", + "LAYER": "App", + "INVARIANT": "Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution." }, "relations": [ { @@ -104234,11 +104244,15 @@ }, { "type": "DEPENDS_ON", - "target": "superset_tool.client" + "target": "SupersetClient" }, { "type": "DEPENDS_ON", - "target": "superset_tool.utils" + "target": "MigrationEngine" + }, + { + "type": "DEPENDS_ON", + "target": "IdMappingService" }, { "type": "USES", @@ -104250,10 +104264,14 @@ "name": "MigrationPlugin", "type": "Class", "tier": "CRITICAL", - "start_line": 24, - "end_line": 406, + "start_line": 27, + "end_line": 383, "tags": { - "PURPOSE": "Implementation of the migration plugin logic." + "PURPOSE": "Implementation of the migration plugin workflow and transformation orchestration.", + "PRE": "Plugin loader must register this instance.", + "POST": "Provides migration UI schema and executes atomic dashboard transfers.", + "TEST_FIXTURE": "password_inject_payload -> INLINE_JSON: {\"passwords\": {\"PostgreSQL\": \"secret123\"}}", + "TEST_INVARIANT": "strict_db_isolation -> VERIFIED_BY: [successful_dashboard_transfer, missing_mapping_resolution]" }, "relations": [], "children": [ @@ -104261,13 +104279,13 @@ "name": "id", "type": "Function", "tier": "STANDARD", - "start_line": 32, - "end_line": 40, + "start_line": 41, + "end_line": 49, "tags": { "PURPOSE": "Returns the unique identifier for the migration plugin.", "PRE": "None.", - "POST": "Returns \"superset-migration\".", - "RETURN": "str - \"superset-migration\"" + "POST": "Returns stable string \"superset-migration\".", + "RETURN": "str" }, "relations": [], "children": [], @@ -104277,32 +104295,32 @@ { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 32 + "line_number": 41 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 32 + "line_number": 41 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 32 + "line_number": 41 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 32 + "line_number": 41 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 32 + "line_number": 41 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 32 + "line_number": 41 } ], "score": 0.0 @@ -104312,13 +104330,13 @@ "name": "name", "type": "Function", "tier": "STANDARD", - "start_line": 43, - "end_line": 51, + "start_line": 52, + "end_line": 60, "tags": { - "PURPOSE": "Returns the human-readable name of the migration plugin.", + "PURPOSE": "Returns the human-readable name of the plugin.", "PRE": "None.", - "POST": "Returns the plugin name.", - "RETURN": "str - Plugin name." + "POST": "Returns \"Superset Dashboard Migration\".", + "RETURN": "str" }, "relations": [], "children": [], @@ -104328,32 +104346,32 @@ { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 43 + "line_number": 52 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 43 + "line_number": 52 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 43 + "line_number": 52 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 43 + "line_number": 52 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 43 + "line_number": 52 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 43 + "line_number": 52 } ], "score": 0.0 @@ -104363,13 +104381,13 @@ "name": "description", "type": "Function", "tier": "STANDARD", - "start_line": 54, - "end_line": 62, + "start_line": 63, + "end_line": 71, "tags": { - "PURPOSE": "Returns a description of the migration plugin.", + "PURPOSE": "Returns the semantic description of the plugin.", "PRE": "None.", - "POST": "Returns the plugin description.", - "RETURN": "str - Plugin description." + "POST": "Returns description string.", + "RETURN": "str" }, "relations": [], "children": [], @@ -104379,32 +104397,32 @@ { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 54 + "line_number": 63 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 54 + "line_number": 63 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 54 + "line_number": 63 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 54 + "line_number": 63 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 54 + "line_number": 63 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 54 + "line_number": 63 } ], "score": 0.0 @@ -104414,13 +104432,13 @@ "name": "version", "type": "Function", "tier": "STANDARD", - "start_line": 65, - "end_line": 73, + "start_line": 74, + "end_line": 82, "tags": { - "PURPOSE": "Returns the version of the migration plugin.", + "PURPOSE": "Returns the semantic version of the migration plugin.", "PRE": "None.", "POST": "Returns \"1.0.0\".", - "RETURN": "str - \"1.0.0\"" + "RETURN": "str" }, "relations": [], "children": [], @@ -104430,32 +104448,32 @@ { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 65 + "line_number": 74 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 65 + "line_number": 74 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 65 + "line_number": 74 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 65 + "line_number": 74 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 65 + "line_number": 74 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 65 + "line_number": 74 } ], "score": 0.0 @@ -104465,76 +104483,48 @@ "name": "ui_route", "type": "Function", "tier": "STANDARD", - "start_line": 76, - "end_line": 82, + "start_line": 85, + "end_line": 93, "tags": { - "PURPOSE": "Returns the frontend route for the migration plugin.", - "RETURN": "str - \"/migration\"" + "PURPOSE": "Returns the frontend routing anchor for the plugin.", + "PRE": "None.", + "POST": "Returns \"/migration\".", + "RETURN": "str" }, "relations": [], "children": [], "compliance": { "valid": true, "issues": [ - { - "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 76 - }, - { - "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 76 - }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 76 + "line_number": 85 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 76 - }, - { - "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 76 - }, - { - "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 76 + "line_number": 85 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 76 + "line_number": 85 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 76 - }, - { - "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 76 - }, - { - "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 76 + "line_number": 85 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 76 + "line_number": 85 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 76 + "line_number": 85 } ], "score": 0.0 @@ -104544,13 +104534,13 @@ "name": "get_schema", "type": "Function", "tier": "STANDARD", - "start_line": 84, - "end_line": 133, + "start_line": 95, + "end_line": 147, "tags": { - "PURPOSE": "Returns the JSON schema for migration plugin parameters.", - "PRE": "Config manager is available.", - "POST": "Returns a valid JSON schema dictionary.", - "RETURN": "Dict[str, Any] - JSON schema." + "PURPOSE": "Generates the JSON Schema for the plugin execution form dynamically.", + "PRE": "ConfigManager is accessible and environments are defined.", + "POST": "Returns a JSON Schema dict matching current system environments.", + "RETURN": "Dict[str, Any]" }, "relations": [], "children": [], @@ -104560,32 +104550,32 @@ { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 84 + "line_number": 95 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 84 + "line_number": 95 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 84 + "line_number": 95 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 84 + "line_number": 95 }, { "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 84 + "line_number": 95 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 84 + "line_number": 95 } ], "score": 0.0 @@ -104595,69 +104585,40 @@ "name": "execute", "type": "Function", "tier": "STANDARD", - "start_line": 135, - "end_line": 405, + "start_line": 149, + "end_line": 382, "tags": { - "PURPOSE": "Executes the dashboard migration logic with TaskContext support.", - "PARAM": "context (Optional[TaskContext]) - Task context for logging with source attribution.", - "PRE": "Source and target environments must be configured.", - "POST": "Selected dashboards are migrated." + "PURPOSE": "Orchestrates the dashboard migration pipeline including extraction, AST mutation, and ingestion.", + "PARAM": "context (Optional[TaskContext]) - Dependency injected TaskContext for IO tracing.", + "PRE": "Source and target environments must resolve. Matching dashboards must exist.", + "POST": "Dashboard ZIP bundles are transformed and imported. ID mappings are synchronized.", + "SIDE_EFFECT": "Creates temp files, mutates target Superset state, blocks on user input (passwords/mappings).", + "TEST_CONTRACT": "Dict[str, Any] -> Dict[str, Any]", + "TEST_SCENARIO": "empty_selection -> Returns NO_MATCHES gracefully when regex finds zero dashboards.", + "TEST_EDGE": "target_api_timeout -> [Dashboard added to failed_dashboards, task concludes with PARTIAL_SUCCESS]" }, "relations": [], - "children": [ - { - "name": "MigrationPlugin.execute", - "type": "Action", - "tier": "STANDARD", - "start_line": 157, - "end_line": 404, - "tags": { - "PURPOSE": "Execute the migration logic with proper task logging." - }, - "relations": [], - "children": [], - "compliance": { - "valid": true, - "issues": [], - "score": 1.0 - } - } - ], + "children": [], "compliance": { "valid": true, "issues": [ { - "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", + "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 135 + "line_number": 149 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 135 - }, - { - "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 135 + "line_number": 149 }, { "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", "severity": "WARNING", - "line_number": 135 - }, - { - "message": "Missing Mandatory Tag: @SIDE_EFFECT (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 135 - }, - { - "message": "Missing Mandatory Tag: @DATA_CONTRACT (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 135 + "line_number": 149 } ], - "score": 0.0 + "score": 0.49000000000000005 } } ], @@ -104667,32 +104628,32 @@ { "message": "Missing Mandatory Tag: @TIER (required for CRITICAL tier)", "severity": "ERROR", - "line_number": 24 + "line_number": 27 }, { "message": "Missing Mandatory Tag: @INVARIANT (required for CRITICAL tier)", "severity": "ERROR", - "line_number": 24 + "line_number": 27 }, { "message": "Missing @INVARIANT tag (required for CRITICAL tier)", "severity": "ERROR", - "line_number": 24 + "line_number": 27 }, { "message": "Missing Mandatory Tag: @TIER (required for CRITICAL tier)", "severity": "ERROR", - "line_number": 24 + "line_number": 27 }, { "message": "Missing Mandatory Tag: @INVARIANT (required for CRITICAL tier)", "severity": "ERROR", - "line_number": 24 + "line_number": 27 }, { "message": "Missing @INVARIANT tag (required for CRITICAL tier)", "severity": "ERROR", - "line_number": 24 + "line_number": 27 } ], "score": 0.0 @@ -104702,33 +104663,18 @@ "compliance": { "valid": false, "issues": [ - { - "message": "Missing Mandatory Tag: @TIER (required for CRITICAL tier)", - "severity": "ERROR", - "line_number": 1 - }, - { - "message": "Missing Mandatory Tag: @INVARIANT (required for CRITICAL tier)", - "severity": "ERROR", - "line_number": 1 - }, { "message": "Invalid @RELATION predicate: USES. Allowed: BINDS_TO, CALLS, DEPENDS_ON, DISPATCHES, IMPLEMENTS, INHERITS", "severity": "ERROR", "line_number": 1 }, { - "message": "Missing @INVARIANT tag (required for CRITICAL tier)", - "severity": "ERROR", - "line_number": 1 - }, - { - "message": "Fractal limit warning: Module length is 407 lines (must be < 300)", + "message": "Fractal limit warning: Module length is 384 lines (must be < 300)", "severity": "WARNING", "line_number": 1 } ], - "score": 0.0 + "score": 0.3 } }, {