busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor complete

Browse Source
This commit is contained in:
busya
2026-01-22 17:37:17 +03:00
parent 203ce446f4
commit d99a13d91f
41 changed files with 1789 additions and 3052 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
backend/src/plugins/backup.py
View File

@@ -12,10 +12,9 @@ from requests.exceptions import RequestException
from ..core.plugin_base import PluginBase
from ..core.logger import belief_scope
from superset_tool.client import SupersetClient
from superset_tool.exceptions import SupersetAPIError
from superset_tool.utils.logger import SupersetLogger
from superset_tool.utils.fileio import (
from ..core.superset_client import SupersetClient
from ..core.utils.network import SupersetAPIError
from ..core.utils.fileio import (
save_and_unpack_dashboard,
archive_exports,
sanitize_filename,
@@ -23,7 +22,6 @@ from superset_tool.utils.fileio import (
remove_empty_directories,
RetentionPolicy
)
from superset_tool.utils.init_clients import setup_clients
from ..dependencies import get_config_manager
# [DEF:BackupPlugin:Class]
@@ -131,25 +129,25 @@ class BackupPlugin(PluginBase):
backup_path_str = params.get("backup_path") or config_manager.get_config().settings.backup_path
backup_path = Path(backup_path_str)
logger = SupersetLogger(log_dir=backup_path / "Logs", console=True)
logger.info(f"[BackupPlugin][Entry] Starting backup for {env}.")
from ..core.logger import logger as app_logger
app_logger.info(f"[BackupPlugin][Entry] Starting backup for {env}.")
try:
config_manager = get_config_manager()
if not config_manager.has_environments():
raise ValueError("No Superset environments configured. Please add an environment in Settings.")
clients = setup_clients(logger, custom_envs=config_manager.get_environments())
client = clients.get(env)
if not client:
env_config = config_manager.get_environment(env)
if not env_config:
raise ValueError(f"Environment '{env}' not found in configuration.")
client = SupersetClient(env_config)
dashboard_count, dashboard_meta = client.get_dashboards()
logger.info(f"[BackupPlugin][Progress] Found {dashboard_count} dashboards to export in {env}.")
app_logger.info(f"[BackupPlugin][Progress] Found {dashboard_count} dashboards to export in {env}.")
if dashboard_count == 0:
logger.info("[BackupPlugin][Exit] No dashboards to back up.")
app_logger.info("[BackupPlugin][Exit] No dashboards to back up.")
return
for db in dashboard_meta:
@@ -169,23 +167,22 @@ class BackupPlugin(PluginBase):
zip_content=zip_content,
original_filename=filename,
output_dir=dashboard_dir,
unpack=False,
logger=logger
unpack=False
)
archive_exports(str(dashboard_dir), policy=RetentionPolicy(), logger=logger)
archive_exports(str(dashboard_dir), policy=RetentionPolicy())
except (SupersetAPIError, RequestException, IOError, OSError) as db_error:
logger.error(f"[BackupPlugin][Failure] Failed to export dashboard {dashboard_title} (ID: {dashboard_id}): {db_error}", exc_info=True)
app_logger.error(f"[BackupPlugin][Failure] Failed to export dashboard {dashboard_title} (ID: {dashboard_id}): {db_error}", exc_info=True)
continue
consolidate_archive_folders(backup_path / env.upper(), logger=logger)
remove_empty_directories(str(backup_path / env.upper()), logger=logger)
consolidate_archive_folders(backup_path / env.upper())
remove_empty_directories(str(backup_path / env.upper()))
logger.info(f"[BackupPlugin][CoherenceCheck:Passed] Backup logic completed for {env}.")
app_logger.info(f"[BackupPlugin][CoherenceCheck:Passed] Backup logic completed for {env}.")
except (RequestException, IOError, KeyError) as e:
logger.critical(f"[BackupPlugin][Failure] Fatal error during backup for {env}: {e}", exc_info=True)
app_logger.critical(f"[BackupPlugin][Failure] Fatal error during backup for {env}: {e}", exc_info=True)
raise e
# [/DEF:execute:Function]
# [/DEF:BackupPlugin:Class]

28
backend/src/plugins/debug.py
View File

@@ -145,19 +145,7 @@ class DebugPlugin(PluginBase):
if not env_config:
raise ValueError(f"Environment '{name}' not found.")
# Map Environment model to SupersetConfig
from superset_tool.models import SupersetConfig
superset_config = SupersetConfig(
env=env_config.name,
base_url=env_config.url,
auth={
"provider": "db", # Defaulting to db provider
"username": env_config.username,
"password": env_config.password,
"refresh": "false"
}
)
client = SupersetClient(superset_config)
client = SupersetClient(env_config)
client.authenticate()
count, dbs = client.get_databases()
results[name] = {
@@ -188,19 +176,7 @@ class DebugPlugin(PluginBase):
if not env_config:
raise ValueError(f"Environment '{env_name}' not found.")
# Map Environment model to SupersetConfig
from superset_tool.models import SupersetConfig
superset_config = SupersetConfig(
env=env_config.name,
base_url=env_config.url,
auth={
"provider": "db", # Defaulting to db provider
"username": env_config.username,
"password": env_config.password,
"refresh": "false"
}
)
client = SupersetClient(superset_config)
client = SupersetClient(env_config)
client.authenticate()
dataset_response = client.get_dataset(dataset_id)

21
backend/src/plugins/mapper.py
View File

@@ -12,8 +12,7 @@ from ..core.superset_client import SupersetClient
from ..core.logger import logger, belief_scope
from ..core.database import SessionLocal
from ..models.connection import ConnectionConfig
from superset_tool.utils.dataset_mapper import DatasetMapper
from superset_tool.utils.logger import SupersetLogger
from ..core.utils.dataset_mapper import DatasetMapper
# [/SECTION]
# [DEF:MapperPlugin:Class]
@@ -137,25 +136,13 @@ class MapperPlugin(PluginBase):
# Get config and initialize client
from ..dependencies import get_config_manager
from superset_tool.models import SupersetConfig
config_manager = get_config_manager()
env_config = config_manager.get_environment(env_name)
if not env_config:
logger.error(f"[MapperPlugin.execute][State] Environment '{env_name}' not found.")
raise ValueError(f"Environment '{env_name}' not found in configuration.")
# Map Environment model to SupersetConfig
superset_config = SupersetConfig(
env=env_config.name,
base_url=env_config.url,
auth={
"provider": "db", # Defaulting to db provider
"username": env_config.username,
"password": env_config.password,
"refresh": "false"
}
)
client = SupersetClient(superset_config)
client = SupersetClient(env_config)
client.authenticate()
postgres_config = None
@@ -185,9 +172,7 @@ class MapperPlugin(PluginBase):
logger.info(f"[MapperPlugin.execute][Action] Starting mapping for dataset {dataset_id} in {env_name}")
# Use internal SupersetLogger for DatasetMapper
s_logger = SupersetLogger(name="dataset_mapper_plugin")
mapper = DatasetMapper(s_logger)
mapper = DatasetMapper()
try:
mapper.run_mapping(

15
backend/src/plugins/migration.py
View File

@@ -13,11 +13,9 @@ import re
from ..core.plugin_base import PluginBase
from ..core.logger import belief_scope
from superset_tool.client import SupersetClient
from superset_tool.utils.init_clients import setup_clients
from superset_tool.utils.fileio import create_temp_file, update_yamls, create_dashboard_export
from ..core.superset_client import SupersetClient
from ..core.utils.fileio import create_temp_file, update_yamls, create_dashboard_export
from ..dependencies import get_config_manager
from superset_tool.utils.logger import SupersetLogger
from ..core.migration_engine import MigrationEngine
from ..core.database import SessionLocal
from ..models.mapping import DatabaseMapping, Environment
@@ -150,7 +148,7 @@ class MigrationPlugin(PluginBase):
from ..dependencies import get_task_manager
tm = get_task_manager()
class TaskLoggerProxy(SupersetLogger):
class TaskLoggerProxy:
# [DEF:__init__:Function]
# @PURPOSE: Initializes the proxy logger.
# @PRE: None.
@@ -158,7 +156,7 @@ class MigrationPlugin(PluginBase):
def __init__(self):
with belief_scope("__init__"):
# Initialize parent with dummy values since we override methods
super().__init__(console=False)
pass
# [/DEF:__init__:Function]
# [DEF:debug:Function]
@@ -246,9 +244,8 @@ class MigrationPlugin(PluginBase):
logger.info(f"[MigrationPlugin][State] Resolved environments: {from_env_name} -> {to_env_name}")
all_clients = setup_clients(logger, custom_envs=environments)
from_c = all_clients.get(from_env_name)
to_c = all_clients.get(to_env_name)
from_c = SupersetClient(src_env)
to_c = SupersetClient(tgt_env)
if not from_c or not to_c:
raise ValueError(f"Clients not initialized for environments: {from_env_name}, {to_env_name}")

14
backend/src/plugins/search.py
View File

@@ -106,25 +106,13 @@ class SearchPlugin(PluginBase):
# Get config and initialize client
from ..dependencies import get_config_manager
from superset_tool.models import SupersetConfig
config_manager = get_config_manager()
env_config = config_manager.get_environment(env_name)
if not env_config:
logger.error(f"[SearchPlugin.execute][State] Environment '{env_name}' not found.")
raise ValueError(f"Environment '{env_name}' not found in configuration.")
# Map Environment model to SupersetConfig
superset_config = SupersetConfig(
env=env_config.name,
base_url=env_config.url,
auth={
"provider": "db", # Defaulting to db provider
"username": env_config.username,
"password": env_config.password,
"refresh": "false"
}
)
client = SupersetClient(superset_config)
client = SupersetClient(env_config)
client.authenticate()
logger.info(f"[SearchPlugin.execute][Action] Searching for pattern: '{search_query}' in environment: {env_name}")