156 lines
6.1 KiB
Python
156 lines
6.1 KiB
Python
import logging
|
||
from datetime import datetime
|
||
import shutil
|
||
import keyring
|
||
import os
|
||
from pathlib import Path
|
||
from superset_tool.models import SupersetConfig, DatabaseConfig
|
||
from superset_tool.client import SupersetClient
|
||
from superset_tool.utils.fileio import save_and_unpack_dashboard, archive_exports, sanitize_filename
|
||
|
||
# Настройка логирования
|
||
LOG_DIR = Path("P:\\Superset\\010 Бекапы\\Logs")
|
||
LOG_DIR.mkdir(exist_ok=True, parents=True)
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||
handlers=[
|
||
logging.FileHandler(LOG_DIR / f"superset_backup_{datetime.now().strftime('%Y%m%d')}.log"),
|
||
logging.StreamHandler()
|
||
]
|
||
)
|
||
logger = logging.getLogger(__name__)
|
||
|
||
def setup_clients():
|
||
"""Инициализация клиентов для разных окружений"""
|
||
clients = {}
|
||
try:
|
||
# Конфигурация для Dev
|
||
dev_config = SupersetConfig(
|
||
base_url="https://devta.bi.dwh.rusal.com/api/v1",
|
||
auth={
|
||
"provider": "db",
|
||
"username": "migrate_user",
|
||
"password": keyring.get_password("system", "dev migrate"),
|
||
"refresh": True
|
||
},
|
||
verify_ssl=False
|
||
)
|
||
|
||
# Конфигурация для Prod
|
||
sandbox_config = SupersetConfig(
|
||
base_url="https://prodta.bi.dwh.rusal.com/api/v1",
|
||
auth={
|
||
"provider": "db",
|
||
"username": "migrate_user",
|
||
"password": keyring.get_password("system", "prod migrate"),
|
||
"refresh": True
|
||
},
|
||
verify_ssl=False
|
||
)
|
||
|
||
# Конфигурация для Sandbox
|
||
prod_config = SupersetConfig(
|
||
base_url="https://sandboxta.bi.dwh.rusal.com/api/v1",
|
||
auth={
|
||
"provider": "db",
|
||
"username": "migrate_user",
|
||
"password": keyring.get_password("system", "sandbox migrate"),
|
||
"refresh": True
|
||
},
|
||
verify_ssl=False
|
||
)
|
||
|
||
clients['dev'] = SupersetClient(dev_config)
|
||
clients['sbx'] = SupersetClient(sandbox_config)
|
||
logger.info("Клиенты для окружений успешно инициализированы")
|
||
return clients
|
||
except Exception as e:
|
||
logger.error(f"Ошибка инициализации клиентов: {str(e)}")
|
||
raise
|
||
|
||
def backup_dashboards(client, env_name, backup_root):
|
||
"""Выполнение бэкапа дашбордов для указанного окружения"""
|
||
logger.info(f"Начало бэкапа для окружения {env_name}")
|
||
try:
|
||
dashboard_count, dashboard_meta = client.get_dashboards()
|
||
total = 0
|
||
success = 0
|
||
|
||
for db in dashboard_meta:
|
||
if db['slug']:
|
||
total += 1
|
||
dashboard_title = db['dashboard_title']
|
||
try:
|
||
dashboard_dir = Path(backup_root , env_name , sanitize_filename(dashboard_title))
|
||
dashboard_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
zip_content, filename = client.export_dashboard(db['id'])
|
||
zip_path = save_and_unpack_dashboard(
|
||
zip_content=zip_content,
|
||
original_filename=filename,
|
||
output_dir=dashboard_dir,
|
||
unpack=False
|
||
)
|
||
|
||
logger.info(f"[{env_name}] Дашборд {dashboard_title} сохранен в {zip_path}")
|
||
success += 1
|
||
|
||
#Очистка старых бэкапов
|
||
try:
|
||
archive_exports(dashboard_dir)
|
||
logger.debug(f"[{env_name}] Выполнена очистка для {dashboard_title}")
|
||
except Exception as cleanup_error:
|
||
logger.error(f"[{env_name}] Ошибка очистки {dashboard_title}: {str(cleanup_error)}")
|
||
|
||
except Exception as db_error:
|
||
logger.error(f"[{env_name}] Ошибка обработки дашборда {dashboard_title}: {str(db_error)}",
|
||
exc_info=True)
|
||
|
||
logger.info(f"Бэкап {env_name} завершен. Успешно: {success}/{total}. Всего на сервере - {dashboard_count}")
|
||
return success == total
|
||
except Exception as e:
|
||
logger.error(f"Критическая ошибка при бэкапе {env_name}: {str(e)}", exc_info=True)
|
||
return False
|
||
|
||
def main():
|
||
"""Основная функция выполнения бэкапа"""
|
||
logger.info("="*50)
|
||
logger.info("Запуск процесса бэкапа Superset")
|
||
logger.info("="*50)
|
||
|
||
try:
|
||
clients = setup_clients()
|
||
superset_backup_repo = Path("P:\\Superset\\010 Бекапы")
|
||
|
||
# Бэкап для DEV
|
||
dev_success = backup_dashboards(
|
||
clients['dev'],
|
||
"DEV",
|
||
superset_backup_repo
|
||
)
|
||
|
||
# Бэкап для Sandbox
|
||
sbx_success = backup_dashboards(
|
||
clients['sbx'],
|
||
"SBX",
|
||
superset_backup_repo
|
||
)
|
||
|
||
# Итоговый отчет
|
||
logger.info("="*50)
|
||
logger.info("Итоги выполнения бэкапа:")
|
||
logger.info(f"DEV: {'Успешно' if dev_success else 'С ошибками'}")
|
||
logger.info(f"SBX: {'Успешно' if sbx_success else 'С ошибками'}")
|
||
logger.info(f"Полный лог доступен в: {LOG_DIR}")
|
||
|
||
except Exception as e:
|
||
logger.critical(f"Фатальная ошибка выполнения скрипта: {str(e)}", exc_info=True)
|
||
return 1
|
||
|
||
logger.info("Процесс бэкапа завершен")
|
||
return 0
|
||
|
||
if __name__ == "__main__":
|
||
exit_code = main()
|
||
exit(exit_code) |