147 lines
5.7 KiB
Python
147 lines
5.7 KiB
Python
# pylint: disable=too-many-arguments,too-many-locals,too-many-statements,too-many-branches,unused-argument,invalid-name,redefined-outer-name
|
|
"""
|
|
[MODULE] Superset Dashboard Backup Script
|
|
@contract: Автоматизирует процесс резервного копирования дашбордов Superset.
|
|
"""
|
|
|
|
# [IMPORTS] Стандартная библиотека
|
|
import logging
|
|
import sys
|
|
from pathlib import Path
|
|
from dataclasses import dataclass
|
|
|
|
# [IMPORTS] Third-party
|
|
from requests.exceptions import RequestException
|
|
|
|
# [IMPORTS] Локальные модули
|
|
from superset_tool.client import SupersetClient
|
|
from superset_tool.exceptions import SupersetAPIError
|
|
from superset_tool.utils.logger import SupersetLogger
|
|
from superset_tool.utils.fileio import (
|
|
save_and_unpack_dashboard,
|
|
archive_exports,
|
|
sanitize_filename,
|
|
consolidate_archive_folders,
|
|
remove_empty_directories
|
|
)
|
|
from superset_tool.utils.init_clients import setup_clients
|
|
|
|
|
|
# [ENTITY: Dataclass('BackupConfig')]
|
|
# CONTRACT:
|
|
# PURPOSE: Хранит конфигурацию для процесса бэкапа.
|
|
@dataclass
|
|
class BackupConfig:
|
|
"""Конфигурация для процесса бэкапа."""
|
|
consolidate: bool = True
|
|
rotate_archive: bool = True
|
|
clean_folders: bool = True
|
|
|
|
# [ENTITY: Function('backup_dashboards')]
|
|
# CONTRACT:
|
|
# PURPOSE: Выполняет бэкап всех доступных дашбордов для заданного клиента и окружения.
|
|
# PRECONDITIONS:
|
|
# - `client` должен быть инициализированным экземпляром `SupersetClient`.
|
|
# - `env_name` должен быть строкой, обозначающей окружение.
|
|
# - `backup_root` должен быть валидным путем к корневой директории бэкапа.
|
|
# POSTCONDITIONS:
|
|
# - Дашборды экспортируются и сохраняются.
|
|
# - Возвращает `True` если все дашборды были экспортированы без критических ошибок, `False` иначе.
|
|
def backup_dashboards(
|
|
client: SupersetClient,
|
|
env_name: str,
|
|
backup_root: Path,
|
|
logger: SupersetLogger,
|
|
config: BackupConfig
|
|
) -> bool:
|
|
logger.info(f"[STATE][backup_dashboards][ENTER] Starting backup for {env_name}.")
|
|
try:
|
|
dashboard_count, dashboard_meta = client.get_dashboards()
|
|
logger.info(f"[STATE][backup_dashboards][PROGRESS] Found {dashboard_count} dashboards to export in {env_name}.")
|
|
if dashboard_count == 0:
|
|
return True
|
|
|
|
success_count = 0
|
|
for db in dashboard_meta:
|
|
dashboard_id = db.get('id')
|
|
dashboard_title = db.get('dashboard_title', 'Unknown Dashboard')
|
|
if not dashboard_id:
|
|
continue
|
|
|
|
try:
|
|
dashboard_base_dir_name = sanitize_filename(f"{dashboard_title}")
|
|
dashboard_dir = backup_root / env_name / dashboard_base_dir_name
|
|
dashboard_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
zip_content, filename = client.export_dashboard(dashboard_id)
|
|
|
|
save_and_unpack_dashboard(
|
|
zip_content=zip_content,
|
|
original_filename=filename,
|
|
output_dir=dashboard_dir,
|
|
unpack=False,
|
|
logger=logger
|
|
)
|
|
|
|
if config.rotate_archive:
|
|
archive_exports(str(dashboard_dir), logger=logger)
|
|
|
|
success_count += 1
|
|
except (SupersetAPIError, RequestException, IOError, OSError) as db_error:
|
|
logger.error(f"[STATE][backup_dashboards][FAILURE] Failed to export dashboard {dashboard_title}: {db_error}", exc_info=True)
|
|
|
|
if config.consolidate:
|
|
consolidate_archive_folders(backup_root / env_name , logger=logger)
|
|
|
|
if config.clean_folders:
|
|
remove_empty_directories(str(backup_root / env_name), logger=logger)
|
|
|
|
return success_count == dashboard_count
|
|
except (RequestException, IOError) as e:
|
|
logger.critical(f"[STATE][backup_dashboards][FAILURE] Fatal error during backup for {env_name}: {e}", exc_info=True)
|
|
return False
|
|
# END_FUNCTION_backup_dashboards
|
|
|
|
# [ENTITY: Function('main')]
|
|
# CONTRACT:
|
|
# PURPOSE: Основная точка входа скрипта.
|
|
# PRECONDITIONS: None
|
|
# POSTCONDITIONS: Возвращает код выхода.
|
|
def main() -> int:
|
|
log_dir = Path("P:\\Superset\\010 Бекапы\\Logs")
|
|
logger = SupersetLogger(log_dir=log_dir, level=logging.INFO, console=True)
|
|
logger.info("[STATE][main][ENTER] Starting Superset backup process.")
|
|
|
|
exit_code = 0
|
|
try:
|
|
clients = setup_clients(logger)
|
|
superset_backup_repo = Path("P:\\Superset\\010 Бекапы")
|
|
superset_backup_repo.mkdir(parents=True, exist_ok=True)
|
|
|
|
results = {}
|
|
environments = ['dev', 'sbx', 'prod', 'preprod']
|
|
backup_config = BackupConfig(rotate_archive=True)
|
|
|
|
for env in environments:
|
|
results[env] = backup_dashboards(
|
|
clients[env],
|
|
env.upper(),
|
|
superset_backup_repo,
|
|
logger=logger,
|
|
config=backup_config
|
|
)
|
|
|
|
if not all(results.values()):
|
|
exit_code = 1
|
|
|
|
except (RequestException, IOError) as e:
|
|
logger.critical(f"[STATE][main][FAILURE] Fatal error in main execution: {e}", exc_info=True)
|
|
exit_code = 1
|
|
|
|
logger.info("[STATE][main][SUCCESS] Superset backup process finished.")
|
|
return exit_code
|
|
# END_FUNCTION_main
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|