Работает создание коммитов и перенос в новый enviroment
This commit is contained in:
345
backend/src/plugins/git_plugin.py
Normal file
345
backend/src/plugins/git_plugin.py
Normal file
@@ -0,0 +1,345 @@
|
||||
# [DEF:backend.src.plugins.git_plugin:Module]
|
||||
#
|
||||
# @SEMANTICS: git, plugin, dashboard, version_control, sync, deploy
|
||||
# @PURPOSE: Предоставляет плагин для версионирования и развертывания дашбордов Superset.
|
||||
# @LAYER: Plugin
|
||||
# @RELATION: INHERITS_FROM -> src.core.plugin_base.PluginBase
|
||||
# @RELATION: USES -> src.services.git_service.GitService
|
||||
# @RELATION: USES -> src.core.superset_client.SupersetClient
|
||||
# @RELATION: USES -> src.core.config_manager.ConfigManager
|
||||
#
|
||||
# @INVARIANT: Все операции с Git должны выполняться через GitService.
|
||||
# @CONSTRAINT: Плагин работает только с распакованными YAML-экспортами Superset.
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
import os
|
||||
import io
|
||||
import shutil
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional
|
||||
from src.core.plugin_base import PluginBase
|
||||
from src.services.git_service import GitService
|
||||
from src.core.logger import logger, belief_scope
|
||||
from src.core.config_manager import ConfigManager
|
||||
from src.core.superset_client import SupersetClient
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:GitPlugin:Class]
|
||||
# @PURPOSE: Реализация плагина Git Integration для управления версиями дашбордов.
|
||||
class GitPlugin(PluginBase):
|
||||
|
||||
# [DEF:__init__:Function]
|
||||
# @PURPOSE: Инициализирует плагин и его зависимости.
|
||||
# @POST: Инициализированы git_service и config_manager.
|
||||
def __init__(self):
|
||||
with belief_scope("GitPlugin.__init__"):
|
||||
logger.info("[GitPlugin.__init__][Entry] Initializing GitPlugin.")
|
||||
self.git_service = GitService()
|
||||
|
||||
# Robust config path resolution:
|
||||
# 1. Try absolute path from src/dependencies.py style if possible
|
||||
# 2. Try relative paths based on common execution patterns
|
||||
if os.path.exists("../config.json"):
|
||||
config_path = "../config.json"
|
||||
elif os.path.exists("config.json"):
|
||||
config_path = "config.json"
|
||||
else:
|
||||
# Fallback to the one initialized in dependencies if we can import it
|
||||
try:
|
||||
from src.dependencies import config_manager
|
||||
self.config_manager = config_manager
|
||||
logger.info("[GitPlugin.__init__][Exit] GitPlugin initialized using shared config_manager.")
|
||||
return
|
||||
except:
|
||||
config_path = "config.json"
|
||||
|
||||
self.config_manager = ConfigManager(config_path)
|
||||
logger.info(f"[GitPlugin.__init__][Exit] GitPlugin initialized with {config_path}")
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
return "git-integration"
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "Git Integration"
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return "Version control for Superset dashboards"
|
||||
|
||||
@property
|
||||
def version(self) -> str:
|
||||
return "0.1.0"
|
||||
|
||||
# [DEF:get_schema:Function]
|
||||
# @PURPOSE: Возвращает JSON-схему параметров для выполнения задач плагина.
|
||||
# @RETURN: Dict[str, Any] - Схема параметров.
|
||||
def get_schema(self) -> Dict[str, Any]:
|
||||
with belief_scope("GitPlugin.get_schema"):
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"operation": {"type": "string", "enum": ["sync", "deploy", "history"]},
|
||||
"dashboard_id": {"type": "integer"},
|
||||
"environment_id": {"type": "string"},
|
||||
"source_env_id": {"type": "string"}
|
||||
},
|
||||
"required": ["operation", "dashboard_id"]
|
||||
}
|
||||
# [/DEF:get_schema:Function]
|
||||
|
||||
# [DEF:initialize:Function]
|
||||
# @PURPOSE: Выполняет начальную настройку плагина.
|
||||
# @POST: Плагин готов к выполнению задач.
|
||||
async def initialize(self):
|
||||
with belief_scope("GitPlugin.initialize"):
|
||||
logger.info("[GitPlugin.initialize][Action] Initializing Git Integration Plugin logic.")
|
||||
|
||||
# [DEF:execute:Function]
|
||||
# @PURPOSE: Основной метод выполнения задач плагина.
|
||||
# @PRE: task_data содержит 'operation' и 'dashboard_id'.
|
||||
# @POST: Возвращает результат выполнения операции.
|
||||
# @PARAM: task_data (Dict[str, Any]) - Данные задачи.
|
||||
# @RETURN: Dict[str, Any] - Статус и сообщение.
|
||||
# @RELATION: CALLS -> self._handle_sync
|
||||
# @RELATION: CALLS -> self._handle_deploy
|
||||
async def execute(self, task_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
with belief_scope("GitPlugin.execute"):
|
||||
operation = task_data.get("operation")
|
||||
dashboard_id = task_data.get("dashboard_id")
|
||||
|
||||
logger.info(f"[GitPlugin.execute][Entry] Executing operation: {operation} for dashboard {dashboard_id}")
|
||||
|
||||
if operation == "sync":
|
||||
source_env_id = task_data.get("source_env_id")
|
||||
result = await self._handle_sync(dashboard_id, source_env_id)
|
||||
elif operation == "deploy":
|
||||
env_id = task_data.get("environment_id")
|
||||
result = await self._handle_deploy(dashboard_id, env_id)
|
||||
elif operation == "history":
|
||||
result = {"status": "success", "message": "History available via API"}
|
||||
else:
|
||||
logger.error(f"[GitPlugin.execute][Coherence:Failed] Unknown operation: {operation}")
|
||||
raise ValueError(f"Unknown operation: {operation}")
|
||||
|
||||
logger.info(f"[GitPlugin.execute][Exit] Operation {operation} completed.")
|
||||
return result
|
||||
# [/DEF:execute:Function]
|
||||
|
||||
# [DEF:_handle_sync:Function]
|
||||
# @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий.
|
||||
# @PRE: Репозиторий для дашборда должен существовать.
|
||||
# @POST: Файлы в репозитории обновлены до текущего состояния в Superset.
|
||||
# @PARAM: dashboard_id (int) - ID дашборда.
|
||||
# @PARAM: source_env_id (Optional[str]) - ID исходного окружения.
|
||||
# @RETURN: Dict[str, str] - Результат синхронизации.
|
||||
# @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория.
|
||||
# @RELATION: CALLS -> src.services.git_service.GitService.get_repo
|
||||
# @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard
|
||||
async def _handle_sync(self, dashboard_id: int, source_env_id: Optional[str] = None) -> Dict[str, str]:
|
||||
with belief_scope("GitPlugin._handle_sync"):
|
||||
try:
|
||||
# 1. Получение репозитория
|
||||
repo = self.git_service.get_repo(dashboard_id)
|
||||
repo_path = Path(repo.working_dir)
|
||||
logger.info(f"[_handle_sync][Action] Target repo path: {repo_path}")
|
||||
|
||||
# 2. Настройка клиента Superset
|
||||
env = self._get_env(source_env_id)
|
||||
client = SupersetClient(env)
|
||||
client.authenticate()
|
||||
|
||||
# 3. Экспорт дашборда
|
||||
logger.info(f"[_handle_sync][Action] Exporting dashboard {dashboard_id} from {env.name}")
|
||||
zip_bytes, _ = client.export_dashboard(dashboard_id)
|
||||
|
||||
# 4. Распаковка с выравниванием структуры (flattening)
|
||||
logger.info(f"[_handle_sync][Action] Unpacking export to {repo_path}")
|
||||
|
||||
# Список папок/файлов, которые мы ожидаем от Superset
|
||||
managed_dirs = ["dashboards", "charts", "datasets", "databases"]
|
||||
managed_files = ["metadata.yaml"]
|
||||
|
||||
# Очистка старых данных перед распаковкой, чтобы не оставалось "призраков"
|
||||
for d in managed_dirs:
|
||||
d_path = repo_path / d
|
||||
if d_path.exists() and d_path.is_dir():
|
||||
shutil.rmtree(d_path)
|
||||
for f in managed_files:
|
||||
f_path = repo_path / f
|
||||
if f_path.exists():
|
||||
f_path.unlink()
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
|
||||
# Superset экспортирует всё в подпапку dashboard_export_timestamp/
|
||||
# Нам нужно найти это имя папки
|
||||
namelist = zf.namelist()
|
||||
if not namelist:
|
||||
raise ValueError("Export ZIP is empty")
|
||||
|
||||
root_folder = namelist[0].split('/')[0]
|
||||
logger.info(f"[_handle_sync][Action] Detected root folder in ZIP: {root_folder}")
|
||||
|
||||
for member in zf.infolist():
|
||||
if member.filename.startswith(root_folder + "/") and len(member.filename) > len(root_folder) + 1:
|
||||
# Убираем префикс папки
|
||||
relative_path = member.filename[len(root_folder)+1:]
|
||||
target_path = repo_path / relative_path
|
||||
|
||||
if member.is_dir():
|
||||
target_path.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
target_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(member) as source, open(target_path, "wb") as target:
|
||||
shutil.copyfileobj(source, target)
|
||||
|
||||
# 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff)
|
||||
try:
|
||||
repo.git.add(A=True)
|
||||
logger.info(f"[_handle_sync][Action] Changes staged in git")
|
||||
except Exception as ge:
|
||||
logger.warning(f"[_handle_sync][Action] Failed to stage changes: {ge}")
|
||||
|
||||
logger.info(f"[_handle_sync][Coherence:OK] Dashboard {dashboard_id} synced successfully.")
|
||||
return {"status": "success", "message": "Dashboard synced and flattened in local repository"}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[_handle_sync][Coherence:Failed] Sync failed: {e}")
|
||||
raise
|
||||
# [/DEF:_handle_sync:Function]
|
||||
|
||||
# [DEF:_handle_deploy:Function]
|
||||
# @PURPOSE: Упаковывает репозиторий в ZIP и импортирует в целевое окружение Superset.
|
||||
# @PRE: environment_id должен соответствовать настроенному окружению.
|
||||
# @POST: Дашборд импортирован в целевой Superset.
|
||||
# @PARAM: dashboard_id (int) - ID дашборда.
|
||||
# @PARAM: env_id (str) - ID целевого окружения.
|
||||
# @RETURN: Dict[str, Any] - Результат деплоя.
|
||||
# @SIDE_EFFECT: Создает и удаляет временный ZIP-файл.
|
||||
# @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard
|
||||
async def _handle_deploy(self, dashboard_id: int, env_id: str) -> Dict[str, Any]:
|
||||
with belief_scope("GitPlugin._handle_deploy"):
|
||||
try:
|
||||
if not env_id:
|
||||
raise ValueError("Target environment ID required for deployment")
|
||||
|
||||
# 1. Получение репозитория
|
||||
repo = self.git_service.get_repo(dashboard_id)
|
||||
repo_path = Path(repo.working_dir)
|
||||
|
||||
# 2. Упаковка в ZIP
|
||||
logger.info(f"[_handle_deploy][Action] Packing repository {repo_path} for deployment.")
|
||||
zip_buffer = io.BytesIO()
|
||||
|
||||
# Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/)
|
||||
root_dir_name = f"dashboard_export_{dashboard_id}"
|
||||
|
||||
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
for root, dirs, files in os.walk(repo_path):
|
||||
if ".git" in dirs:
|
||||
dirs.remove(".git")
|
||||
for file in files:
|
||||
if file == ".git" or file.endswith(".zip"): continue
|
||||
file_path = Path(root) / file
|
||||
# Prepend the root directory name to the archive path
|
||||
arcname = Path(root_dir_name) / file_path.relative_to(repo_path)
|
||||
zf.write(file_path, arcname)
|
||||
|
||||
zip_buffer.seek(0)
|
||||
|
||||
# 3. Настройка клиента Superset
|
||||
env = self.config_manager.get_environment(env_id)
|
||||
if not env:
|
||||
raise ValueError(f"Environment {env_id} not found")
|
||||
|
||||
client = SupersetClient(env)
|
||||
client.authenticate()
|
||||
|
||||
# 4. Импорт
|
||||
temp_zip_path = repo_path / f"deploy_{dashboard_id}.zip"
|
||||
logger.info(f"[_handle_deploy][Action] Saving temporary zip to {temp_zip_path}")
|
||||
with open(temp_zip_path, "wb") as f:
|
||||
f.write(zip_buffer.getvalue())
|
||||
|
||||
try:
|
||||
logger.info(f"[_handle_deploy][Action] Importing dashboard to {env.name}")
|
||||
result = client.import_dashboard(temp_zip_path)
|
||||
logger.info(f"[_handle_deploy][Coherence:OK] Deployment successful for dashboard {dashboard_id}.")
|
||||
return {"status": "success", "message": f"Dashboard deployed to {env.name}", "details": result}
|
||||
finally:
|
||||
if temp_zip_path.exists():
|
||||
os.remove(temp_zip_path)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[_handle_deploy][Coherence:Failed] Deployment failed: {e}")
|
||||
raise
|
||||
# [/DEF:_handle_deploy:Function]
|
||||
|
||||
# [DEF:_get_env:Function]
|
||||
# @PURPOSE: Вспомогательный метод для получения конфигурации окружения.
|
||||
# @PARAM: env_id (Optional[str]) - ID окружения.
|
||||
# @RETURN: Environment - Объект конфигурации окружения.
|
||||
def _get_env(self, env_id: Optional[str] = None):
|
||||
with belief_scope("GitPlugin._get_env"):
|
||||
logger.info(f"[_get_env][Entry] Fetching environment for ID: {env_id}")
|
||||
|
||||
# Priority 1: ConfigManager (config.json)
|
||||
if env_id:
|
||||
env = self.config_manager.get_environment(env_id)
|
||||
if env:
|
||||
logger.info(f"[_get_env][Exit] Found environment by ID in ConfigManager: {env.name}")
|
||||
return env
|
||||
|
||||
# Priority 2: Database (DeploymentEnvironment)
|
||||
from src.core.database import SessionLocal
|
||||
from src.models.git import DeploymentEnvironment
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
if env_id:
|
||||
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()
|
||||
else:
|
||||
# If no ID, try to find active or any environment in DB
|
||||
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active == True).first()
|
||||
if not db_env:
|
||||
db_env = db.query(DeploymentEnvironment).first()
|
||||
|
||||
if db_env:
|
||||
logger.info(f"[_get_env][Exit] Found environment in DB: {db_env.name}")
|
||||
from src.core.config_models import Environment
|
||||
# Use token as password for SupersetClient
|
||||
return Environment(
|
||||
id=db_env.id,
|
||||
name=db_env.name,
|
||||
url=db_env.superset_url,
|
||||
username="admin",
|
||||
password=db_env.superset_token,
|
||||
verify_ssl=True
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
# Priority 3: ConfigManager Default (if no env_id provided)
|
||||
envs = self.config_manager.get_environments()
|
||||
if envs:
|
||||
if env_id:
|
||||
# If env_id was provided but not found in DB or specifically by ID in config,
|
||||
# but we have other envs, maybe it's one of them?
|
||||
env = next((e for e in envs if e.id == env_id), None)
|
||||
if env:
|
||||
logger.info(f"[_get_env][Exit] Found environment {env_id} in ConfigManager list")
|
||||
return env
|
||||
|
||||
if not env_id:
|
||||
logger.info(f"[_get_env][Exit] Using first environment from ConfigManager: {envs[0].name}")
|
||||
return envs[0]
|
||||
|
||||
logger.error(f"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}")
|
||||
raise ValueError("No environments configured. Please add a Superset Environment in Settings.")
|
||||
# [/DEF:_get_env:Function]
|
||||
|
||||
# [/DEF:GitPlugin:Class]
|
||||
# [/DEF:backend.src.plugins.git_plugin:Module]
|
||||
Reference in New Issue
Block a user