busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

log refactor

Browse Source
This commit is contained in:
busya
2026-05-12 19:30:15 +03:00
parent 1d59df2233
commit b17b5333c7
84 changed files with 5827 additions and 3908 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
backend/src/plugins/migration.py
View File

@@ -18,8 +18,11 @@ from typing import Dict, Any, Optional
import re
from ..core.plugin_base import PluginBase
from ..core.logger import belief_scope, logger as app_logger
from ..core.logger import belief_scope
from ..core.cot_logger import MarkerLogger
from ..core.superset_client import SupersetClient
log = MarkerLogger("MigrationPlugin")
from ..core.utils.fileio import create_temp_file
from ..dependencies import get_config_manager
from ..core.migration_engine import MigrationEngine
@@ -105,7 +108,7 @@ class MigrationPlugin(PluginBase):
# @RETURN: Dict[str, Any]
def get_schema(self) -> Dict[str, Any]:
with belief_scope("MigrationPlugin.get_schema"):
app_logger.reason("Generating migration UI schema")
log.reason("Generating migration UI schema")
config_manager = get_config_manager()
envs = [e.name for e in config_manager.get_environments()]
@@ -148,7 +151,7 @@ class MigrationPlugin(PluginBase):
},
"required": ["from_env", "to_env", "dashboard_regex"],
}
app_logger.reflect("Schema generated successfully", extra={"environments_count": len(envs)})
log.reflect("Schema generated successfully", payload={"environments_count": len(envs)})
return schema
# [/DEF:get_schema:Function]
@@ -168,7 +171,7 @@ class MigrationPlugin(PluginBase):
# @TEST_EDGE: target_api_timeout -> [Dashboard added to failed_dashboards, task concludes with PARTIAL_SUCCESS]
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
with belief_scope("MigrationPlugin.execute"):
app_logger.reason("Evaluating migration task parameters", extra={"params": params})
log.reason("Evaluating migration task parameters", payload={"params": params})
source_env_id = params.get("source_env_id")
target_env_id = params.get("target_env_id")
@@ -184,11 +187,11 @@ class MigrationPlugin(PluginBase):
from ..dependencies import get_task_manager
tm = get_task_manager()
log = context.logger if context else app_logger
superset_log = log.with_source("superset_api") if context else log
migration_log = log.with_source("migration") if context else log
exec_log = context.logger if context else log
superset_log = exec_log.with_source("superset_api") if context else exec_log
migration_log = exec_log.with_source("migration") if context else exec_log
log.info("Starting migration task.")
exec_log.info("Starting migration task.")
try:
config_manager = get_config_manager()
@@ -199,13 +202,13 @@ class MigrationPlugin(PluginBase):
tgt_env = next((e for e in environments if e.id == target_env_id), None) if target_env_id else next((e for e in environments if e.name == to_env_name), None)
if not src_env or not tgt_env:
app_logger.explore("Environment resolution failed", extra={"src": source_env_id or from_env_name, "tgt": target_env_id or to_env_name})
log.explore("Environment resolution failed", error="Could not resolve source or target environment", payload={"src": source_env_id or from_env_name, "tgt": target_env_id or to_env_name})
raise ValueError(f"Could not resolve source or target environment. Source: {source_env_id or from_env_name}, Target: {target_env_id or to_env_name}")
from_env_name = src_env.name
to_env_name = tgt_env.name
app_logger.reason("Environments resolved successfully", extra={"from": from_env_name, "to": to_env_name})
log.reason("Environments resolved successfully", payload={"from": from_env_name, "to": to_env_name})
migration_result = {
"status": "SUCCESS",
@@ -232,12 +235,12 @@ class MigrationPlugin(PluginBase):
regex_pattern = re.compile(str(dashboard_regex), re.IGNORECASE)
dashboards_to_migrate = [d for d in all_dashboards if regex_pattern.search(d.get("dashboard_title", ""))]
else:
app_logger.explore("No deterministic selection criteria provided")
log.explore("No deterministic selection criteria provided", error="No dashboard selection criteria (selected_ids or dashboard_regex)")
migration_result["status"] = "NO_SELECTION"
return migration_result
if not dashboards_to_migrate:
app_logger.explore("Zero dashboards match selection criteria")
log.explore("Zero dashboards match selection criteria", error="No dashboards matched the given selection criteria")
migration_result["status"] = "NO_MATCHES"
return migration_result
@@ -249,7 +252,7 @@ class MigrationPlugin(PluginBase):
db_mapping = {}
if replace_db_config:
app_logger.reason("Fetching environment DB mappings from catalog")
log.reason("Fetching environment DB mappings from catalog")
db = SessionLocal()
try:
src_env_db = db.query(Environment).filter(Environment.name == from_env_name).first()
@@ -263,7 +266,7 @@ class MigrationPlugin(PluginBase):
stored_map_dict = {m.source_db_uuid: m.target_db_uuid for m in stored_mappings}
stored_map_dict.update(db_mapping)
db_mapping = stored_map_dict
log.info(f"Loaded {len(stored_mappings)} database mappings from database.")
exec_log.info(f"Loaded {len(stored_mappings)} database mappings from database.")
finally:
db.close()
@@ -273,7 +276,7 @@ class MigrationPlugin(PluginBase):
# Migration Loop
for dash in dashboards_to_migrate:
dash_id, dash_slug, title = dash["id"], dash.get("slug"), dash["dashboard_title"]
app_logger.reason(f"Starting pipeline for dashboard '{title}'", extra={"dash_id": dash_id})
log.reason(f"Starting pipeline for dashboard '{title}'", payload={"dash_id": dash_id})
try:
exported_content, _ = from_c.export_dashboard(dash_id)
@@ -291,10 +294,10 @@ class MigrationPlugin(PluginBase):
if not success and replace_db_config:
if task_id:
app_logger.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", extra={"task_id": task_id})
log.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", error="AST transform blocked by missing mappings", payload={"task_id": task_id})
await tm.wait_for_resolution(task_id)
app_logger.reason("Task resumed, re-evaluating mapping states")
log.reason("Task resumed, re-evaluating mapping states")
db = SessionLocal()
try:
src_env_rt = db.query(Environment).filter(Environment.name == from_env_name).first()
@@ -317,12 +320,12 @@ class MigrationPlugin(PluginBase):
)
if success:
app_logger.reason("Pushing transformed ZIP to target Superset")
log.reason("Pushing transformed ZIP to target Superset")
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug)
migration_result["migrated_dashboards"].append({"id": dash_id, "title": title})
app_logger.reflect("Import successful", extra={"title": title})
log.reflect("Import successful", payload={"title": title})
else:
app_logger.explore("Transformation strictly failed, bypassing ingestion")
log.explore("Transformation strictly failed, bypassing ingestion", error="Dashboard ZIP transformation failed")
migration_log.error(f"Failed to transform ZIP for dashboard {title}")
migration_result["failed_dashboards"].append({
"id": dash_id, "title": title, "error": "Failed to transform ZIP"
@@ -340,7 +343,7 @@ class MigrationPlugin(PluginBase):
if match_alt:
db_name = match_alt.group(1)
app_logger.explore(f"Missing DB password detected during ingestion. Escalating to UI.", extra={"db_name": db_name})
log.explore("Missing DB password detected during ingestion. Escalating to UI.", error="Missing DB password for import", payload={"db_name": db_name})
if task_id:
tm.await_input(task_id, {
@@ -354,15 +357,15 @@ class MigrationPlugin(PluginBase):
passwords = task.params.get("passwords", {})
if passwords:
app_logger.reason(f"Retrying import for {title} with injected credentials")
log.reason(f"Retrying import for {title} with injected credentials")
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug, passwords=passwords)
migration_result["migrated_dashboards"].append({"id": dash_id, "title": title})
app_logger.reflect("Password injection unblocked import")
log.reflect("Password injection unblocked import")
if "passwords" in task.params:
del task.params["passwords"]
continue
app_logger.explore(f"Catastrophic dashboard ingestion failure: {exc}")
log.explore("Catastrophic dashboard ingestion failure", error=f"Dashboard ingestion failed: {exc}")
migration_result["failed_dashboards"].append({"id": dash_id, "title": title, "error": str(exc)})
if migration_result["failed_dashboards"]:
@@ -370,20 +373,20 @@ class MigrationPlugin(PluginBase):
# Post-Migration ID Mapping Synchronization
try:
app_logger.reason("Executing incremental ID catalog sync on target")
log.reason("Executing incremental ID catalog sync on target")
db_session = SessionLocal()
mapping_service = IdMappingService(db_session)
mapping_service.sync_environment(tgt_env.id, to_c, incremental=True)
db_session.close()
app_logger.reflect("Incremental catalog sync closed out cleanly")
log.reflect("Incremental catalog sync closed out cleanly")
except Exception as sync_exc:
app_logger.explore(f"ID Mapping sync failed, mapping state might be degraded: {sync_exc}")
log.explore("ID Mapping sync failed, mapping state might be degraded", error=f"ID Mapping sync error: {sync_exc}")
app_logger.reflect("Migration cycle fully resolved", extra={"result": migration_result})
log.reflect("Migration cycle fully resolved", payload={"result": migration_result})
return migration_result
except Exception as e:
app_logger.explore(f"Fatal plugin failure: {e}", exc_info=True)
log.explore("Fatal plugin failure", error=f"Plugin execution failed: {e}", exc_info=True)
raise e
# [/DEF:execute:Function]
# [/DEF:MigrationPlugin:Class]