# [DEF:backend.src.plugins.git_plugin:Module] # # @SEMANTICS: git, plugin, dashboard, version_control, sync, deploy # @PURPOSE: Предоставляет плагин для версионирования и развертывания дашбордов Superset. # @LAYER: Plugin # @RELATION: INHERITS_FROM -> src.core.plugin_base.PluginBase # @RELATION: USES -> src.services.git_service.GitService # @RELATION: USES -> src.core.superset_client.SupersetClient # @RELATION: USES -> src.core.config_manager.ConfigManager # # @INVARIANT: Все операции с Git должны выполняться через GitService. # @CONSTRAINT: Плагин работает только с распакованными YAML-экспортами Superset. # [SECTION: IMPORTS] import os import io import shutil import zipfile from pathlib import Path from typing import Dict, Any, Optional from src.core.plugin_base import PluginBase from src.services.git_service import GitService from src.core.logger import logger, belief_scope from src.core.config_manager import ConfigManager from src.core.superset_client import SupersetClient # [/SECTION] # [DEF:GitPlugin:Class] # @PURPOSE: Реализация плагина Git Integration для управления версиями дашбордов. class GitPlugin(PluginBase): # [DEF:__init__:Function] # @PURPOSE: Инициализирует плагин и его зависимости. # @POST: Инициализированы git_service и config_manager. def __init__(self): with belief_scope("GitPlugin.__init__"): logger.info("[GitPlugin.__init__][Entry] Initializing GitPlugin.") self.git_service = GitService() # Robust config path resolution: # 1. Try absolute path from src/dependencies.py style if possible # 2. Try relative paths based on common execution patterns if os.path.exists("../config.json"): config_path = "../config.json" elif os.path.exists("config.json"): config_path = "config.json" else: # Fallback to the one initialized in dependencies if we can import it try: from src.dependencies import config_manager self.config_manager = config_manager logger.info("[GitPlugin.__init__][Exit] GitPlugin initialized using shared config_manager.") return except: config_path = "config.json" self.config_manager = ConfigManager(config_path) logger.info(f"[GitPlugin.__init__][Exit] GitPlugin initialized with {config_path}") # [/DEF:__init__:Function] @property def id(self) -> str: return "git-integration" @property def name(self) -> str: return "Git Integration" @property def description(self) -> str: return "Version control for Superset dashboards" @property def version(self) -> str: return "0.1.0" # [DEF:get_schema:Function] # @PURPOSE: Возвращает JSON-схему параметров для выполнения задач плагина. # @RETURN: Dict[str, Any] - Схема параметров. def get_schema(self) -> Dict[str, Any]: with belief_scope("GitPlugin.get_schema"): return { "type": "object", "properties": { "operation": {"type": "string", "enum": ["sync", "deploy", "history"]}, "dashboard_id": {"type": "integer"}, "environment_id": {"type": "string"}, "source_env_id": {"type": "string"} }, "required": ["operation", "dashboard_id"] } # [/DEF:get_schema:Function] # [DEF:initialize:Function] # @PURPOSE: Выполняет начальную настройку плагина. # @POST: Плагин готов к выполнению задач. async def initialize(self): with belief_scope("GitPlugin.initialize"): logger.info("[GitPlugin.initialize][Action] Initializing Git Integration Plugin logic.") # [DEF:execute:Function] # @PURPOSE: Основной метод выполнения задач плагина. # @PRE: task_data содержит 'operation' и 'dashboard_id'. # @POST: Возвращает результат выполнения операции. # @PARAM: task_data (Dict[str, Any]) - Данные задачи. # @RETURN: Dict[str, Any] - Статус и сообщение. # @RELATION: CALLS -> self._handle_sync # @RELATION: CALLS -> self._handle_deploy async def execute(self, task_data: Dict[str, Any]) -> Dict[str, Any]: with belief_scope("GitPlugin.execute"): operation = task_data.get("operation") dashboard_id = task_data.get("dashboard_id") logger.info(f"[GitPlugin.execute][Entry] Executing operation: {operation} for dashboard {dashboard_id}") if operation == "sync": source_env_id = task_data.get("source_env_id") result = await self._handle_sync(dashboard_id, source_env_id) elif operation == "deploy": env_id = task_data.get("environment_id") result = await self._handle_deploy(dashboard_id, env_id) elif operation == "history": result = {"status": "success", "message": "History available via API"} else: logger.error(f"[GitPlugin.execute][Coherence:Failed] Unknown operation: {operation}") raise ValueError(f"Unknown operation: {operation}") logger.info(f"[GitPlugin.execute][Exit] Operation {operation} completed.") return result # [/DEF:execute:Function] # [DEF:_handle_sync:Function] # @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий. # @PRE: Репозиторий для дашборда должен существовать. # @POST: Файлы в репозитории обновлены до текущего состояния в Superset. # @PARAM: dashboard_id (int) - ID дашборда. # @PARAM: source_env_id (Optional[str]) - ID исходного окружения. # @RETURN: Dict[str, str] - Результат синхронизации. # @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория. # @RELATION: CALLS -> src.services.git_service.GitService.get_repo # @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard async def _handle_sync(self, dashboard_id: int, source_env_id: Optional[str] = None) -> Dict[str, str]: with belief_scope("GitPlugin._handle_sync"): try: # 1. Получение репозитория repo = self.git_service.get_repo(dashboard_id) repo_path = Path(repo.working_dir) logger.info(f"[_handle_sync][Action] Target repo path: {repo_path}") # 2. Настройка клиента Superset env = self._get_env(source_env_id) client = SupersetClient(env) client.authenticate() # 3. Экспорт дашборда logger.info(f"[_handle_sync][Action] Exporting dashboard {dashboard_id} from {env.name}") zip_bytes, _ = client.export_dashboard(dashboard_id) # 4. Распаковка с выравниванием структуры (flattening) logger.info(f"[_handle_sync][Action] Unpacking export to {repo_path}") # Список папок/файлов, которые мы ожидаем от Superset managed_dirs = ["dashboards", "charts", "datasets", "databases"] managed_files = ["metadata.yaml"] # Очистка старых данных перед распаковкой, чтобы не оставалось "призраков" for d in managed_dirs: d_path = repo_path / d if d_path.exists() and d_path.is_dir(): shutil.rmtree(d_path) for f in managed_files: f_path = repo_path / f if f_path.exists(): f_path.unlink() with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf: # Superset экспортирует всё в подпапку dashboard_export_timestamp/ # Нам нужно найти это имя папки namelist = zf.namelist() if not namelist: raise ValueError("Export ZIP is empty") root_folder = namelist[0].split('/')[0] logger.info(f"[_handle_sync][Action] Detected root folder in ZIP: {root_folder}") for member in zf.infolist(): if member.filename.startswith(root_folder + "/") and len(member.filename) > len(root_folder) + 1: # Убираем префикс папки relative_path = member.filename[len(root_folder)+1:] target_path = repo_path / relative_path if member.is_dir(): target_path.mkdir(parents=True, exist_ok=True) else: target_path.parent.mkdir(parents=True, exist_ok=True) with zf.open(member) as source, open(target_path, "wb") as target: shutil.copyfileobj(source, target) # 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff) try: repo.git.add(A=True) logger.info(f"[_handle_sync][Action] Changes staged in git") except Exception as ge: logger.warning(f"[_handle_sync][Action] Failed to stage changes: {ge}") logger.info(f"[_handle_sync][Coherence:OK] Dashboard {dashboard_id} synced successfully.") return {"status": "success", "message": "Dashboard synced and flattened in local repository"} except Exception as e: logger.error(f"[_handle_sync][Coherence:Failed] Sync failed: {e}") raise # [/DEF:_handle_sync:Function] # [DEF:_handle_deploy:Function] # @PURPOSE: Упаковывает репозиторий в ZIP и импортирует в целевое окружение Superset. # @PRE: environment_id должен соответствовать настроенному окружению. # @POST: Дашборд импортирован в целевой Superset. # @PARAM: dashboard_id (int) - ID дашборда. # @PARAM: env_id (str) - ID целевого окружения. # @RETURN: Dict[str, Any] - Результат деплоя. # @SIDE_EFFECT: Создает и удаляет временный ZIP-файл. # @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard async def _handle_deploy(self, dashboard_id: int, env_id: str) -> Dict[str, Any]: with belief_scope("GitPlugin._handle_deploy"): try: if not env_id: raise ValueError("Target environment ID required for deployment") # 1. Получение репозитория repo = self.git_service.get_repo(dashboard_id) repo_path = Path(repo.working_dir) # 2. Упаковка в ZIP logger.info(f"[_handle_deploy][Action] Packing repository {repo_path} for deployment.") zip_buffer = io.BytesIO() # Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/) root_dir_name = f"dashboard_export_{dashboard_id}" with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf: for root, dirs, files in os.walk(repo_path): if ".git" in dirs: dirs.remove(".git") for file in files: if file == ".git" or file.endswith(".zip"): continue file_path = Path(root) / file # Prepend the root directory name to the archive path arcname = Path(root_dir_name) / file_path.relative_to(repo_path) zf.write(file_path, arcname) zip_buffer.seek(0) # 3. Настройка клиента Superset env = self.config_manager.get_environment(env_id) if not env: raise ValueError(f"Environment {env_id} not found") client = SupersetClient(env) client.authenticate() # 4. Импорт temp_zip_path = repo_path / f"deploy_{dashboard_id}.zip" logger.info(f"[_handle_deploy][Action] Saving temporary zip to {temp_zip_path}") with open(temp_zip_path, "wb") as f: f.write(zip_buffer.getvalue()) try: logger.info(f"[_handle_deploy][Action] Importing dashboard to {env.name}") result = client.import_dashboard(temp_zip_path) logger.info(f"[_handle_deploy][Coherence:OK] Deployment successful for dashboard {dashboard_id}.") return {"status": "success", "message": f"Dashboard deployed to {env.name}", "details": result} finally: if temp_zip_path.exists(): os.remove(temp_zip_path) except Exception as e: logger.error(f"[_handle_deploy][Coherence:Failed] Deployment failed: {e}") raise # [/DEF:_handle_deploy:Function] # [DEF:_get_env:Function] # @PURPOSE: Вспомогательный метод для получения конфигурации окружения. # @PARAM: env_id (Optional[str]) - ID окружения. # @RETURN: Environment - Объект конфигурации окружения. def _get_env(self, env_id: Optional[str] = None): with belief_scope("GitPlugin._get_env"): logger.info(f"[_get_env][Entry] Fetching environment for ID: {env_id}") # Priority 1: ConfigManager (config.json) if env_id: env = self.config_manager.get_environment(env_id) if env: logger.info(f"[_get_env][Exit] Found environment by ID in ConfigManager: {env.name}") return env # Priority 2: Database (DeploymentEnvironment) from src.core.database import SessionLocal from src.models.git import DeploymentEnvironment db = SessionLocal() try: if env_id: db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first() else: # If no ID, try to find active or any environment in DB db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active == True).first() if not db_env: db_env = db.query(DeploymentEnvironment).first() if db_env: logger.info(f"[_get_env][Exit] Found environment in DB: {db_env.name}") from src.core.config_models import Environment # Use token as password for SupersetClient return Environment( id=db_env.id, name=db_env.name, url=db_env.superset_url, username="admin", password=db_env.superset_token, verify_ssl=True ) finally: db.close() # Priority 3: ConfigManager Default (if no env_id provided) envs = self.config_manager.get_environments() if envs: if env_id: # If env_id was provided but not found in DB or specifically by ID in config, # but we have other envs, maybe it's one of them? env = next((e for e in envs if e.id == env_id), None) if env: logger.info(f"[_get_env][Exit] Found environment {env_id} in ConfigManager list") return env if not env_id: logger.info(f"[_get_env][Exit] Using first environment from ConfigManager: {envs[0].name}") return envs[0] logger.error(f"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}") raise ValueError("No environments configured. Please add a Superset Environment in Settings.") # [/DEF:_get_env:Function] # [/DEF:GitPlugin:Class] # [/DEF:backend.src.plugins.git_plugin:Module]