busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

semantics: complete DEF-to-region migration, fix regressions

Browse Source 
- Convert legacy [DEF🆔Type] anchors to #region/#endregion across 329 files
- Reinstate _normalize_timestamp_value in sql_generator.py
- Fix MarkerLogger→logger migration in events.py (molecular CoT markers)
- Fix dataset_review orchestrator dependencies (_build_execution_snapshot)
- Fix config_manager stale-record deletion (moved to save path only)
- Add 77 missing [/DEF:] closers in 5 unbalanced test files
- Update assistant_chat.integration.test.js for #region format
- Apply molecular-cot-logging markers (REASON/REFLECT/EXPLORE) via logger.* methods
This commit is contained in:
busya
2026-05-12 23:54:55 +03:00
parent fe8978f716
commit 306c5ae742
331 changed files with 9630 additions and 10312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
backend/src/plugins/migration.py
View File

@@ -1,26 +1,23 @@
# #region MigrationPlugin [C:5] [TYPE Module] [SEMANTICS migration, superset, automation, dashboard, plugin, transformation]
# @BRIEF Orchestrates export, DB-mapping transformation, and import of Superset dashboards across environments.
# @LAYER App
# @PRE Plugin loader can resolve infrastructure dependencies and execution requests provide validated migration context.
# @POST Plugin metadata remains stable and migration execution preserves mapped-environment import guarantees.
# @SIDE_EFFECT Reads config, opens database sessions, creates temporary artifacts, and triggers Superset export/import workflows.
# @DATA_CONTRACT Input[TaskContext{from_env,to_env,dashboard_regex,replace_db_config,from_db_id,to_db_id,passwords?}] -> Output[MigrationResult|artifact set]
# @INVARIANT Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution.
# @RELATION IMPLEMENTS -> [PluginBase]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [MigrationEngine]
# @RELATION DEPENDS_ON -> [IdMappingService]
# @RELATION USES -> [TaskContext]
# @LAYER: App
# @RELATION IMPLEMENTS -> PluginBase
# @RELATION DEPENDS_ON -> SupersetClient
# @RELATION DEPENDS_ON -> MigrationEngine
# @RELATION DEPENDS_ON -> IdMappingService
# @RELATION USES -> TaskContext
# @PRE: Plugin loader can resolve infrastructure dependencies and execution requests provide validated migration context.
# @POST: Plugin metadata remains stable and migration execution preserves mapped-environment import guarantees.
# @SIDE_EFFECT: Reads config, opens database sessions, creates temporary artifacts, and triggers Superset export/import workflows.
# @DATA_CONTRACT: Input[TaskContext{from_env,to_env,dashboard_regex,replace_db_config,from_db_id,to_db_id,passwords?}] -> Output[MigrationResult|artifact set]
# @INVARIANT: Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution.
from typing import Dict, Any, Optional
import re
from ..core.plugin_base import PluginBase
from ..core.logger import belief_scope
from ..core.cot_logger import MarkerLogger
from ..core.logger import belief_scope, logger as app_logger
from ..core.superset_client import SupersetClient
log = MarkerLogger("MigrationPlugin")
from ..core.utils.fileio import create_temp_file
from ..dependencies import get_config_manager
from ..core.migration_engine import MigrationEngine
@@ -31,8 +28,8 @@ from ..core.task_manager.context import TaskContext
# #region MigrationPlugin [TYPE Class]
# @BRIEF Implementation of the migration plugin workflow and transformation orchestration.
# @PRE SupersetClient authenticated, database session active
# @POST Returns MigrationResult with success/failure status and artifact list
# @PRE: SupersetClient authenticated, database session active
# @POST: Returns MigrationResult with success/failure status and artifact list
# @TEST_FIXTURE: superset_export_zip -> file:backend/tests/fixtures/migration/dashboard_export.zip
# @TEST_FIXTURE: db_mapping_payload -> INLINE_JSON: {"db_mappings": {"source_uuid_1": "target_uuid_2"}}
# @TEST_FIXTURE: password_inject_payload -> INLINE_JSON: {"passwords": {"PostgreSQL": "secret123"}}
@@ -45,68 +42,68 @@ class MigrationPlugin(PluginBase):
"""
@property
# #region id [TYPE Function]
# @BRIEF Returns the unique identifier for the migration plugin.
# @PRE None.
# @POST Returns stable string "superset-migration".
# @RETURN str
# [DEF:id:Function]
# @PURPOSE: Returns the unique identifier for the migration plugin.
# @PRE: None.
# @POST: Returns stable string "superset-migration".
# @RETURN: str
def id(self) -> str:
with belief_scope("MigrationPlugin.id"):
return "superset-migration"
# #endregion id
# [/DEF:id:Function]
@property
# #region name [TYPE Function]
# @BRIEF Returns the human-readable name of the plugin.
# @PRE None.
# @POST Returns "Superset Dashboard Migration".
# @RETURN str
# [DEF:name:Function]
# @PURPOSE: Returns the human-readable name of the plugin.
# @PRE: None.
# @POST: Returns "Superset Dashboard Migration".
# @RETURN: str
def name(self) -> str:
with belief_scope("MigrationPlugin.name"):
return "Superset Dashboard Migration"
# #endregion name
# [/DEF:name:Function]
@property
# #region description [TYPE Function]
# @BRIEF Returns the semantic description of the plugin.
# @PRE None.
# @POST Returns description string.
# @RETURN str
# [DEF:description:Function]
# @PURPOSE: Returns the semantic description of the plugin.
# @PRE: None.
# @POST: Returns description string.
# @RETURN: str
def description(self) -> str:
with belief_scope("MigrationPlugin.description"):
return "Migrates dashboards between Superset environments."
# #endregion description
# [/DEF:description:Function]
@property
# #region version [TYPE Function]
# @BRIEF Returns the semantic version of the migration plugin.
# @PRE None.
# @POST Returns "1.0.0".
# @RETURN str
# [DEF:version:Function]
# @PURPOSE: Returns the semantic version of the migration plugin.
# @PRE: None.
# @POST: Returns "1.0.0".
# @RETURN: str
def version(self) -> str:
with belief_scope("MigrationPlugin.version"):
return "1.0.0"
# #endregion version
# [/DEF:version:Function]
@property
# #region ui_route [TYPE Function]
# @BRIEF Returns the frontend routing anchor for the plugin.
# @PRE None.
# @POST Returns "/migration".
# @RETURN str
# [DEF:ui_route:Function]
# @PURPOSE: Returns the frontend routing anchor for the plugin.
# @PRE: None.
# @POST: Returns "/migration".
# @RETURN: str
def ui_route(self) -> str:
with belief_scope("MigrationPlugin.ui_route"):
return "/migration"
# #endregion ui_route
# [/DEF:ui_route:Function]
# #region get_schema [TYPE Function]
# @BRIEF Generates the JSON Schema for the plugin execution form dynamically.
# @PRE ConfigManager is accessible and environments are defined.
# @POST Returns a JSON Schema dict matching current system environments.
# @RETURN Dict[str, Any]
# [DEF:get_schema:Function]
# @PURPOSE: Generates the JSON Schema for the plugin execution form dynamically.
# @PRE: ConfigManager is accessible and environments are defined.
# @POST: Returns a JSON Schema dict matching current system environments.
# @RETURN: Dict[str, Any]
def get_schema(self) -> Dict[str, Any]:
with belief_scope("MigrationPlugin.get_schema"):
log.reason("Generating migration UI schema")
app_logger.reason("Generating migration UI schema")
config_manager = get_config_manager()
envs = [e.name for e in config_manager.get_environments()]
@@ -149,12 +146,12 @@ class MigrationPlugin(PluginBase):
},
"required": ["from_env", "to_env", "dashboard_regex"],
}
log.reflect("Schema generated successfully", payload={"environments_count": len(envs)})
app_logger.reflect("Schema generated successfully", extra={"environments_count": len(envs)})
return schema
# #endregion get_schema
# [/DEF:get_schema:Function]
# #region execute [TYPE Function]
# @BRIEF Orchestrates the dashboard migration pipeline including extraction, AST mutation, and ingestion.
# [DEF:execute:Function]
# @PURPOSE: Orchestrates the dashboard migration pipeline including extraction, AST mutation, and ingestion.
# @PARAM: params (Dict[str, Any]) - Extracted parameters from UI/API execution request.
# @PARAM: context (Optional[TaskContext]) - Dependency injected TaskContext for IO tracing.
# @PRE: Source and target environments must resolve. Matching dashboards must exist.
@@ -169,7 +166,7 @@ class MigrationPlugin(PluginBase):
# @TEST_EDGE: target_api_timeout -> [Dashboard added to failed_dashboards, task concludes with PARTIAL_SUCCESS]
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
with belief_scope("MigrationPlugin.execute"):
log.reason("Evaluating migration task parameters", payload={"params": params})
app_logger.reason("Evaluating migration task parameters", extra={"params": params})
source_env_id = params.get("source_env_id")
target_env_id = params.get("target_env_id")
@@ -185,11 +182,11 @@ class MigrationPlugin(PluginBase):
from ..dependencies import get_task_manager
tm = get_task_manager()
exec_log = context.logger if context else log
superset_log = exec_log.with_source("superset_api") if context else exec_log
migration_log = exec_log.with_source("migration") if context else exec_log
log = context.logger if context else app_logger
superset_log = log.with_source("superset_api") if context else log
migration_log = log.with_source("migration") if context else log
exec_log.info("Starting migration task.")
log.info("Starting migration task.")
try:
config_manager = get_config_manager()
@@ -200,13 +197,13 @@ class MigrationPlugin(PluginBase):
tgt_env = next((e for e in environments if e.id == target_env_id), None) if target_env_id else next((e for e in environments if e.name == to_env_name), None)
if not src_env or not tgt_env:
log.explore("Environment resolution failed", error="Could not resolve source or target environment", payload={"src": source_env_id or from_env_name, "tgt": target_env_id or to_env_name})
app_logger.explore("Environment resolution failed", extra={"src": source_env_id or from_env_name, "tgt": target_env_id or to_env_name})
raise ValueError(f"Could not resolve source or target environment. Source: {source_env_id or from_env_name}, Target: {target_env_id or to_env_name}")
from_env_name = src_env.name
to_env_name = tgt_env.name
log.reason("Environments resolved successfully", payload={"from": from_env_name, "to": to_env_name})
app_logger.reason("Environments resolved successfully", extra={"from": from_env_name, "to": to_env_name})
migration_result = {
"status": "SUCCESS",
@@ -233,12 +230,12 @@ class MigrationPlugin(PluginBase):
regex_pattern = re.compile(str(dashboard_regex), re.IGNORECASE)
dashboards_to_migrate = [d for d in all_dashboards if regex_pattern.search(d.get("dashboard_title", ""))]
else:
log.explore("No deterministic selection criteria provided", error="No dashboard selection criteria (selected_ids or dashboard_regex)")
app_logger.explore("No deterministic selection criteria provided")
migration_result["status"] = "NO_SELECTION"
return migration_result
if not dashboards_to_migrate:
log.explore("Zero dashboards match selection criteria", error="No dashboards matched the given selection criteria")
app_logger.explore("Zero dashboards match selection criteria")
migration_result["status"] = "NO_MATCHES"
return migration_result
@@ -250,7 +247,7 @@ class MigrationPlugin(PluginBase):
db_mapping = {}
if replace_db_config:
log.reason("Fetching environment DB mappings from catalog")
app_logger.reason("Fetching environment DB mappings from catalog")
db = SessionLocal()
try:
src_env_db = db.query(Environment).filter(Environment.name == from_env_name).first()
@@ -264,7 +261,7 @@ class MigrationPlugin(PluginBase):
stored_map_dict = {m.source_db_uuid: m.target_db_uuid for m in stored_mappings}
stored_map_dict.update(db_mapping)
db_mapping = stored_map_dict
exec_log.info(f"Loaded {len(stored_mappings)} database mappings from database.")
log.info(f"Loaded {len(stored_mappings)} database mappings from database.")
finally:
db.close()
@@ -274,7 +271,7 @@ class MigrationPlugin(PluginBase):
# Migration Loop
for dash in dashboards_to_migrate:
dash_id, dash_slug, title = dash["id"], dash.get("slug"), dash["dashboard_title"]
log.reason(f"Starting pipeline for dashboard '{title}'", payload={"dash_id": dash_id})
app_logger.reason(f"Starting pipeline for dashboard '{title}'", extra={"dash_id": dash_id})
try:
exported_content, _ = from_c.export_dashboard(dash_id)
@@ -292,10 +289,10 @@ class MigrationPlugin(PluginBase):
if not success and replace_db_config:
if task_id:
log.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", error="AST transform blocked by missing mappings", payload={"task_id": task_id})
app_logger.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", extra={"task_id": task_id})
await tm.wait_for_resolution(task_id)
log.reason("Task resumed, re-evaluating mapping states")
app_logger.reason("Task resumed, re-evaluating mapping states")
db = SessionLocal()
try:
src_env_rt = db.query(Environment).filter(Environment.name == from_env_name).first()
@@ -318,12 +315,12 @@ class MigrationPlugin(PluginBase):
)
if success:
log.reason("Pushing transformed ZIP to target Superset")
app_logger.reason("Pushing transformed ZIP to target Superset")
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug)
migration_result["migrated_dashboards"].append({"id": dash_id, "title": title})
log.reflect("Import successful", payload={"title": title})
app_logger.reflect("Import successful", extra={"title": title})
else:
log.explore("Transformation strictly failed, bypassing ingestion", error="Dashboard ZIP transformation failed")
app_logger.explore("Transformation strictly failed, bypassing ingestion")
migration_log.error(f"Failed to transform ZIP for dashboard {title}")
migration_result["failed_dashboards"].append({
"id": dash_id, "title": title, "error": "Failed to transform ZIP"
@@ -341,7 +338,7 @@ class MigrationPlugin(PluginBase):
if match_alt:
db_name = match_alt.group(1)
log.explore("Missing DB password detected during ingestion. Escalating to UI.", error="Missing DB password for import", payload={"db_name": db_name})
app_logger.explore(f"Missing DB password detected during ingestion. Escalating to UI.", extra={"db_name": db_name})
if task_id:
tm.await_input(task_id, {
@@ -355,15 +352,15 @@ class MigrationPlugin(PluginBase):
passwords = task.params.get("passwords", {})
if passwords:
log.reason(f"Retrying import for {title} with injected credentials")
app_logger.reason(f"Retrying import for {title} with injected credentials")
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug, passwords=passwords)
migration_result["migrated_dashboards"].append({"id": dash_id, "title": title})
log.reflect("Password injection unblocked import")
app_logger.reflect("Password injection unblocked import")
if "passwords" in task.params:
del task.params["passwords"]
continue
log.explore("Catastrophic dashboard ingestion failure", error=f"Dashboard ingestion failed: {exc}")
app_logger.explore(f"Catastrophic dashboard ingestion failure: {exc}")
migration_result["failed_dashboards"].append({"id": dash_id, "title": title, "error": str(exc)})
if migration_result["failed_dashboards"]:
@@ -371,21 +368,21 @@ class MigrationPlugin(PluginBase):
# Post-Migration ID Mapping Synchronization
try:
log.reason("Executing incremental ID catalog sync on target")
app_logger.reason("Executing incremental ID catalog sync on target")
db_session = SessionLocal()
mapping_service = IdMappingService(db_session)
mapping_service.sync_environment(tgt_env.id, to_c, incremental=True)
db_session.close()
log.reflect("Incremental catalog sync closed out cleanly")
app_logger.reflect("Incremental catalog sync closed out cleanly")
except Exception as sync_exc:
log.explore("ID Mapping sync failed, mapping state might be degraded", error=f"ID Mapping sync error: {sync_exc}")
app_logger.explore(f"ID Mapping sync failed, mapping state might be degraded: {sync_exc}")
log.reflect("Migration cycle fully resolved", payload={"result": migration_result})
app_logger.reflect("Migration cycle fully resolved", extra={"result": migration_result})
return migration_result
except Exception as e:
log.explore("Fatal plugin failure", error=f"Plugin execution failed: {e}", exc_info=True)
app_logger.explore(f"Fatal plugin failure: {e}", exc_info=True)
raise e
# #endregion execute
# #endregion MigrationPlugin
# [/DEF:execute:Function]
# #endregion MigrationPlugin
# #endregion MigrationPlugin