busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore(lint): apply ruff --fix (4443 auto-fixes)

Browse Source 
Auto-fixed categories:
- F401: unused imports removed
- I001: import blocks sorted
- W293: trailing whitespace stripped
- UP035: deprecated typing imports replaced
- SIM: simplify suggestions applied
- ARG: unused args prefixed with underscore
- T201: print statements removed
- F841: unused variables removed
- RUF059: unpacked variables prefixed

Remaining ~1500 unfixable errors (C901, B904, N806, E402) require manual work.

Backend smoke tests: 13/13 passed.
This commit is contained in:
busya
2026-05-14 11:20:17 +03:00
parent a9a5eff518
commit c6189876b3
337 changed files with 4677 additions and 4515 deletions
Download Patch File Download Diff File Expand all files Collapse all files

153
backend/src/plugins/migration.py
View File

@@ -12,19 +12,21 @@
# @DATA_CONTRACT: Input[TaskContext{from_env,to_env,dashboard_regex,replace_db_config,from_db_id,to_db_id,passwords?}] -> Output[MigrationResult|artifact set]
# @INVARIANT: Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution.
from typing import Dict, Any, Optional
import re
from typing import Any
from ..core.database import SessionLocal
from ..core.logger import belief_scope
from ..core.logger import logger as app_logger
from ..core.mapping_service import IdMappingService
from ..core.migration_engine import MigrationEngine
from ..core.plugin_base import PluginBase
from ..core.logger import belief_scope, logger as app_logger
from ..core.superset_client import SupersetClient
from ..core.task_manager.context import TaskContext
from ..core.utils.fileio import create_temp_file
from ..dependencies import get_config_manager
from ..core.migration_engine import MigrationEngine
from ..core.database import SessionLocal
from ..models.mapping import DatabaseMapping, Environment
from ..core.mapping_service import IdMappingService
from ..core.task_manager.context import TaskContext
# #region MigrationPlugin [TYPE Class]
# @BRIEF Implementation of the migration plugin workflow and transformation orchestration.
@@ -101,12 +103,12 @@ class MigrationPlugin(PluginBase):
# @PRE: ConfigManager is accessible and environments are defined.
# @POST: Returns a JSON Schema dict matching current system environments.
# @RETURN: Dict[str, Any]
def get_schema(self) -> Dict[str, Any]:
def get_schema(self) -> dict[str, Any]:
with belief_scope("MigrationPlugin.get_schema"):
app_logger.reason("Generating migration UI schema")
config_manager = get_config_manager()
envs = [e.name for e in config_manager.get_environments()]
schema = {
"type": "object",
"properties": {
@@ -164,20 +166,20 @@ class MigrationPlugin(PluginBase):
# @TEST_EDGE: missing_env_field -> [ValueError: Could not resolve source or target environment]
# @TEST_EDGE: invalid_regex_pattern -> [Regex compilation exception is thrown or caught gracefully]
# @TEST_EDGE: target_api_timeout -> [Dashboard added to failed_dashboards, task concludes with PARTIAL_SUCCESS]
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
async def execute(self, params: dict[str, Any], context: TaskContext | None = None):
with belief_scope("MigrationPlugin.execute"):
app_logger.reason("Evaluating migration task parameters", extra={"params": params})
source_env_id = params.get("source_env_id")
target_env_id = params.get("target_env_id")
selected_ids = params.get("selected_ids")
from_env_name = params.get("from_env")
to_env_name = params.get("to_env")
dashboard_regex = params.get("dashboard_regex")
replace_db_config = params.get("replace_db_config", False)
fix_cross_filters = params.get("fix_cross_filters", True)
task_id = params.get("_task_id")
from ..dependencies import get_task_manager
tm = get_task_manager()
@@ -185,17 +187,17 @@ class MigrationPlugin(PluginBase):
log = context.logger if context else app_logger
superset_log = log.with_source("superset_api") if context else log
migration_log = log.with_source("migration") if context else log
log.info("Starting migration task.")
try:
config_manager = get_config_manager()
environments = config_manager.get_environments()
# Resolve environments
src_env = next((e for e in environments if e.id == source_env_id), None) if source_env_id else next((e for e in environments if e.name == from_env_name), None)
tgt_env = next((e for e in environments if e.id == target_env_id), None) if target_env_id else next((e for e in environments if e.name == to_env_name), None)
if not src_env or not tgt_env:
app_logger.explore("Environment resolution failed", extra={"src": source_env_id or from_env_name, "tgt": target_env_id or to_env_name})
raise ValueError(f"Could not resolve source or target environment. Source: {source_env_id or from_env_name}, Target: {target_env_id or to_env_name}")
@@ -204,7 +206,7 @@ class MigrationPlugin(PluginBase):
to_env_name = tgt_env.name
app_logger.reason("Environments resolved successfully", extra={"from": from_env_name, "to": to_env_name})
migration_result = {
"status": "SUCCESS",
"source_environment": from_env_name,
@@ -217,12 +219,12 @@ class MigrationPlugin(PluginBase):
from_c = SupersetClient(src_env)
to_c = SupersetClient(tgt_env)
if not from_c or not to_c:
raise ValueError(f"Clients not initialized for environments: {from_env_name}, {to_env_name}")
_, all_dashboards = from_c.get_dashboards()
# Selection Logic
if selected_ids:
dashboards_to_migrate = [d for d in all_dashboards if d["id"] in selected_ids]
@@ -245,14 +247,14 @@ class MigrationPlugin(PluginBase):
db_mapping = params.get("db_mappings", {})
if not isinstance(db_mapping, dict):
db_mapping = {}
if replace_db_config:
app_logger.reason("Fetching environment DB mappings from catalog")
db = SessionLocal()
try:
src_env_db = db.query(Environment).filter(Environment.name == from_env_name).first()
tgt_env_db = db.query(Environment).filter(Environment.name == to_env_name).first()
if src_env_db and tgt_env_db:
stored_mappings = db.query(DatabaseMapping).filter(
DatabaseMapping.source_env_id == src_env_db.id,
@@ -272,60 +274,59 @@ class MigrationPlugin(PluginBase):
for dash in dashboards_to_migrate:
dash_id, dash_slug, title = dash["id"], dash.get("slug"), dash["dashboard_title"]
app_logger.reason(f"Starting pipeline for dashboard '{title}'", extra={"dash_id": dash_id})
try:
exported_content, _ = from_c.export_dashboard(dash_id)
with create_temp_file(content=exported_content, dry_run=True, suffix=".zip") as tmp_zip_path:
with create_temp_file(suffix=".zip", dry_run=True) as tmp_new_zip:
success = engine.transform_zip(
str(tmp_zip_path),
str(tmp_new_zip),
db_mapping,
strip_databases=False,
target_env_id=tgt_env.id if tgt_env else None,
fix_cross_filters=fix_cross_filters
)
if not success and replace_db_config:
if task_id:
app_logger.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", extra={"task_id": task_id})
await tm.wait_for_resolution(task_id)
app_logger.reason("Task resumed, re-evaluating mapping states")
db = SessionLocal()
try:
src_env_rt = db.query(Environment).filter(Environment.name == from_env_name).first()
tgt_env_rt = db.query(Environment).filter(Environment.name == to_env_name).first()
mappings = db.query(DatabaseMapping).filter(
DatabaseMapping.source_env_id == src_env_rt.id,
DatabaseMapping.target_env_id == tgt_env_rt.id
).all()
db_mapping = {m.source_db_uuid: m.target_db_uuid for m in mappings}
finally:
db.close()
success = engine.transform_zip(
str(tmp_zip_path),
str(tmp_new_zip),
db_mapping,
strip_databases=False,
target_env_id=tgt_env.id if tgt_env else None,
fix_cross_filters=fix_cross_filters
)
with create_temp_file(content=exported_content, dry_run=True, suffix=".zip") as tmp_zip_path, create_temp_file(suffix=".zip", dry_run=True) as tmp_new_zip:
success = engine.transform_zip(
str(tmp_zip_path),
str(tmp_new_zip),
db_mapping,
strip_databases=False,
target_env_id=tgt_env.id if tgt_env else None,
fix_cross_filters=fix_cross_filters
)
if not success and replace_db_config:
if task_id:
app_logger.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", extra={"task_id": task_id})
await tm.wait_for_resolution(task_id)
app_logger.reason("Task resumed, re-evaluating mapping states")
db = SessionLocal()
try:
src_env_rt = db.query(Environment).filter(Environment.name == from_env_name).first()
tgt_env_rt = db.query(Environment).filter(Environment.name == to_env_name).first()
mappings = db.query(DatabaseMapping).filter(
DatabaseMapping.source_env_id == src_env_rt.id,
DatabaseMapping.target_env_id == tgt_env_rt.id
).all()
db_mapping = {m.source_db_uuid: m.target_db_uuid for m in mappings}
finally:
db.close()
success = engine.transform_zip(
str(tmp_zip_path),
str(tmp_new_zip),
db_mapping,
strip_databases=False,
target_env_id=tgt_env.id if tgt_env else None,
fix_cross_filters=fix_cross_filters
)
if success:
app_logger.reason("Pushing transformed ZIP to target Superset")
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug)
migration_result["migrated_dashboards"].append({"id": dash_id, "title": title})
app_logger.reflect("Import successful", extra={"title": title})
else:
app_logger.explore("Transformation strictly failed, bypassing ingestion")
migration_log.error(f"Failed to transform ZIP for dashboard {title}")
migration_result["failed_dashboards"].append({
"id": dash_id, "title": title, "error": "Failed to transform ZIP"
})
if success:
app_logger.reason("Pushing transformed ZIP to target Superset")
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug)
migration_result["migrated_dashboards"].append({"id": dash_id, "title": title})
app_logger.reflect("Import successful", extra={"title": title})
else:
app_logger.explore("Transformation strictly failed, bypassing ingestion")
migration_log.error(f"Failed to transform ZIP for dashboard {title}")
migration_result["failed_dashboards"].append({
"id": dash_id, "title": title, "error": "Failed to transform ZIP"
})
except Exception as exc:
error_msg = str(exc)
if "Must provide a password for the database" in error_msg:
@@ -338,19 +339,19 @@ class MigrationPlugin(PluginBase):
if match_alt:
db_name = match_alt.group(1)
app_logger.explore(f"Missing DB password detected during ingestion. Escalating to UI.", extra={"db_name": db_name})
app_logger.explore("Missing DB password detected during ingestion. Escalating to UI.", extra={"db_name": db_name})
if task_id:
tm.await_input(task_id, {
"type": "database_password",
"databases": [db_name],
"error_message": error_msg
})
await tm.wait_for_input(task_id)
task = tm.get_task(task_id)
passwords = task.params.get("passwords", {})
if passwords:
app_logger.reason(f"Retrying import for {title} with injected credentials")
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug, passwords=passwords)
@@ -385,4 +386,4 @@ class MigrationPlugin(PluginBase):
raise e
# endregion execute
# #endregion MigrationPlugin
# #endregion MigrationPlugin
# #endregion MigrationPlugin