feat: implement plugin architecture and application settings with Svelte UI
- Added plugin base and loader for backend extensibility - Implemented application settings management with config persistence - Created Svelte-based frontend with Dashboard and Settings pages - Added API routes for plugins, tasks, and settings - Updated documentation and specifications - Improved project structure and developer tools
This commit is contained in:
20
backend/requirements.txt
Normal file → Executable file
20
backend/requirements.txt
Normal file → Executable file
@@ -1,9 +1,11 @@
|
||||
fastapi
|
||||
uvicorn
|
||||
pydantic
|
||||
authlib
|
||||
python-multipart
|
||||
starlette
|
||||
jsonschema
|
||||
requests
|
||||
keyring
|
||||
fastapi
|
||||
uvicorn
|
||||
pydantic
|
||||
authlib
|
||||
python-multipart
|
||||
starlette
|
||||
jsonschema
|
||||
requests
|
||||
keyring
|
||||
httpx
|
||||
PyYAML
|
||||
102
backend/src/api/auth.py
Normal file → Executable file
102
backend/src/api/auth.py
Normal file → Executable file
@@ -1,52 +1,52 @@
|
||||
# [DEF:AuthModule:Module]
|
||||
# @SEMANTICS: auth, authentication, adfs, oauth, middleware
|
||||
# @PURPOSE: Implements ADFS authentication using Authlib for FastAPI. It provides a dependency to protect endpoints.
|
||||
# @LAYER: UI (API)
|
||||
# @RELATION: Used by API routers to protect endpoints that require authentication.
|
||||
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from fastapi.security import OAuth2AuthorizationCodeBearer
|
||||
from authlib.integrations.starlette_client import OAuth
|
||||
from starlette.config import Config
|
||||
|
||||
# Placeholder for ADFS configuration. In a real app, this would come from a secure source.
|
||||
# Create an in-memory .env file
|
||||
from io import StringIO
|
||||
config_data = StringIO("""
|
||||
ADFS_CLIENT_ID=your-client-id
|
||||
ADFS_CLIENT_SECRET=your-client-secret
|
||||
ADFS_SERVER_METADATA_URL=https://your-adfs-server/.well-known/openid-configuration
|
||||
""")
|
||||
config = Config(config_data)
|
||||
oauth = OAuth(config)
|
||||
|
||||
oauth.register(
|
||||
name='adfs',
|
||||
server_metadata_url=config('ADFS_SERVER_METADATA_URL'),
|
||||
client_kwargs={'scope': 'openid profile email'}
|
||||
)
|
||||
|
||||
oauth2_scheme = OAuth2AuthorizationCodeBearer(
|
||||
authorizationUrl="https://your-adfs-server/adfs/oauth2/authorize",
|
||||
tokenUrl="https://your-adfs-server/adfs/oauth2/token",
|
||||
)
|
||||
|
||||
async def get_current_user(token: str = Depends(oauth2_scheme)):
|
||||
"""
|
||||
Dependency to get the current user from the ADFS token.
|
||||
This is a placeholder and needs to be fully implemented.
|
||||
"""
|
||||
# In a real implementation, you would:
|
||||
# 1. Validate the token with ADFS.
|
||||
# 2. Fetch user information.
|
||||
# 3. Create a user object.
|
||||
# For now, we'll just check if a token exists.
|
||||
if not token:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Not authenticated",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
# A real implementation would return a user object.
|
||||
return {"placeholder_user": "user@example.com"}
|
||||
# [DEF:AuthModule:Module]
|
||||
# @SEMANTICS: auth, authentication, adfs, oauth, middleware
|
||||
# @PURPOSE: Implements ADFS authentication using Authlib for FastAPI. It provides a dependency to protect endpoints.
|
||||
# @LAYER: UI (API)
|
||||
# @RELATION: Used by API routers to protect endpoints that require authentication.
|
||||
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from fastapi.security import OAuth2AuthorizationCodeBearer
|
||||
from authlib.integrations.starlette_client import OAuth
|
||||
from starlette.config import Config
|
||||
|
||||
# Placeholder for ADFS configuration. In a real app, this would come from a secure source.
|
||||
# Create an in-memory .env file
|
||||
from io import StringIO
|
||||
config_data = StringIO("""
|
||||
ADFS_CLIENT_ID=your-client-id
|
||||
ADFS_CLIENT_SECRET=your-client-secret
|
||||
ADFS_SERVER_METADATA_URL=https://your-adfs-server/.well-known/openid-configuration
|
||||
""")
|
||||
config = Config(config_data)
|
||||
oauth = OAuth(config)
|
||||
|
||||
oauth.register(
|
||||
name='adfs',
|
||||
server_metadata_url=config('ADFS_SERVER_METADATA_URL'),
|
||||
client_kwargs={'scope': 'openid profile email'}
|
||||
)
|
||||
|
||||
oauth2_scheme = OAuth2AuthorizationCodeBearer(
|
||||
authorizationUrl="https://your-adfs-server/adfs/oauth2/authorize",
|
||||
tokenUrl="https://your-adfs-server/adfs/oauth2/token",
|
||||
)
|
||||
|
||||
async def get_current_user(token: str = Depends(oauth2_scheme)):
|
||||
"""
|
||||
Dependency to get the current user from the ADFS token.
|
||||
This is a placeholder and needs to be fully implemented.
|
||||
"""
|
||||
# In a real implementation, you would:
|
||||
# 1. Validate the token with ADFS.
|
||||
# 2. Fetch user information.
|
||||
# 3. Create a user object.
|
||||
# For now, we'll just check if a token exists.
|
||||
if not token:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Not authenticated",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
# A real implementation would return a user object.
|
||||
return {"placeholder_user": "user@example.com"}
|
||||
# [/DEF]
|
||||
1
backend/src/api/routes/__init__.py
Executable file
1
backend/src/api/routes/__init__.py
Executable file
@@ -0,0 +1 @@
|
||||
from . import plugins, tasks, settings
|
||||
42
backend/src/api/routes/plugins.py
Normal file → Executable file
42
backend/src/api/routes/plugins.py
Normal file → Executable file
@@ -1,22 +1,22 @@
|
||||
# [DEF:PluginsRouter:Module]
|
||||
# @SEMANTICS: api, router, plugins, list
|
||||
# @PURPOSE: Defines the FastAPI router for plugin-related endpoints, allowing clients to list available plugins.
|
||||
# @LAYER: UI (API)
|
||||
# @RELATION: Depends on the PluginLoader and PluginConfig. It is included by the main app.
|
||||
from typing import List
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from ...core.plugin_base import PluginConfig
|
||||
from ...dependencies import get_plugin_loader
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@router.get("/", response_model=List[PluginConfig])
|
||||
async def list_plugins(
|
||||
plugin_loader = Depends(get_plugin_loader)
|
||||
):
|
||||
"""
|
||||
Retrieve a list of all available plugins.
|
||||
"""
|
||||
return plugin_loader.get_all_plugin_configs()
|
||||
# [DEF:PluginsRouter:Module]
|
||||
# @SEMANTICS: api, router, plugins, list
|
||||
# @PURPOSE: Defines the FastAPI router for plugin-related endpoints, allowing clients to list available plugins.
|
||||
# @LAYER: UI (API)
|
||||
# @RELATION: Depends on the PluginLoader and PluginConfig. It is included by the main app.
|
||||
from typing import List
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from ...core.plugin_base import PluginConfig
|
||||
from ...dependencies import get_plugin_loader
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@router.get("/", response_model=List[PluginConfig])
|
||||
async def list_plugins(
|
||||
plugin_loader = Depends(get_plugin_loader)
|
||||
):
|
||||
"""
|
||||
Retrieve a list of all available plugins.
|
||||
"""
|
||||
return plugin_loader.get_all_plugin_configs()
|
||||
# [/DEF]
|
||||
185
backend/src/api/routes/settings.py
Executable file
185
backend/src/api/routes/settings.py
Executable file
@@ -0,0 +1,185 @@
|
||||
# [DEF:SettingsRouter:Module]
|
||||
#
|
||||
# @SEMANTICS: settings, api, router, fastapi
|
||||
# @PURPOSE: Provides API endpoints for managing application settings and Superset environments.
|
||||
# @LAYER: UI (API)
|
||||
# @RELATION: DEPENDS_ON -> ConfigManager
|
||||
# @RELATION: DEPENDS_ON -> ConfigModels
|
||||
#
|
||||
# @INVARIANT: All settings changes must be persisted via ConfigManager.
|
||||
# @PUBLIC_API: router
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from typing import List
|
||||
from ...core.config_models import AppConfig, Environment, GlobalSettings
|
||||
from ...dependencies import get_config_manager
|
||||
from ...core.config_manager import ConfigManager
|
||||
from ...core.logger import logger
|
||||
from superset_tool.client import SupersetClient
|
||||
from superset_tool.models import SupersetConfig
|
||||
import os
|
||||
# [/SECTION]
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# [DEF:get_settings:Function]
|
||||
# @PURPOSE: Retrieves all application settings.
|
||||
# @RETURN: AppConfig - The current configuration.
|
||||
@router.get("/", response_model=AppConfig)
|
||||
async def get_settings(config_manager: ConfigManager = Depends(get_config_manager)):
|
||||
logger.info("[get_settings][Entry] Fetching all settings")
|
||||
config = config_manager.get_config().copy(deep=True)
|
||||
# Mask passwords
|
||||
for env in config.environments:
|
||||
if env.password:
|
||||
env.password = "********"
|
||||
return config
|
||||
# [/DEF:get_settings]
|
||||
|
||||
# [DEF:update_global_settings:Function]
|
||||
# @PURPOSE: Updates global application settings.
|
||||
# @PARAM: settings (GlobalSettings) - The new global settings.
|
||||
# @RETURN: GlobalSettings - The updated settings.
|
||||
@router.patch("/global", response_model=GlobalSettings)
|
||||
async def update_global_settings(
|
||||
settings: GlobalSettings,
|
||||
config_manager: ConfigManager = Depends(get_config_manager)
|
||||
):
|
||||
logger.info("[update_global_settings][Entry] Updating global settings")
|
||||
config_manager.update_global_settings(settings)
|
||||
return settings
|
||||
# [/DEF:update_global_settings]
|
||||
|
||||
# [DEF:get_environments:Function]
|
||||
# @PURPOSE: Lists all configured Superset environments.
|
||||
# @RETURN: List[Environment] - List of environments.
|
||||
@router.get("/environments", response_model=List[Environment])
|
||||
async def get_environments(config_manager: ConfigManager = Depends(get_config_manager)):
|
||||
logger.info("[get_environments][Entry] Fetching environments")
|
||||
return config_manager.get_environments()
|
||||
# [/DEF:get_environments]
|
||||
|
||||
# [DEF:add_environment:Function]
|
||||
# @PURPOSE: Adds a new Superset environment.
|
||||
# @PARAM: env (Environment) - The environment to add.
|
||||
# @RETURN: Environment - The added environment.
|
||||
@router.post("/environments", response_model=Environment)
|
||||
async def add_environment(
|
||||
env: Environment,
|
||||
config_manager: ConfigManager = Depends(get_config_manager)
|
||||
):
|
||||
logger.info(f"[add_environment][Entry] Adding environment {env.id}")
|
||||
config_manager.add_environment(env)
|
||||
return env
|
||||
# [/DEF:add_environment]
|
||||
|
||||
# [DEF:update_environment:Function]
|
||||
# @PURPOSE: Updates an existing Superset environment.
|
||||
# @PARAM: id (str) - The ID of the environment to update.
|
||||
# @PARAM: env (Environment) - The updated environment data.
|
||||
# @RETURN: Environment - The updated environment.
|
||||
@router.put("/environments/{id}", response_model=Environment)
|
||||
async def update_environment(
|
||||
id: str,
|
||||
env: Environment,
|
||||
config_manager: ConfigManager = Depends(get_config_manager)
|
||||
):
|
||||
logger.info(f"[update_environment][Entry] Updating environment {id}")
|
||||
if config_manager.update_environment(id, env):
|
||||
return env
|
||||
raise HTTPException(status_code=404, detail=f"Environment {id} not found")
|
||||
# [/DEF:update_environment]
|
||||
|
||||
# [DEF:delete_environment:Function]
|
||||
# @PURPOSE: Deletes a Superset environment.
|
||||
# @PARAM: id (str) - The ID of the environment to delete.
|
||||
@router.delete("/environments/{id}")
|
||||
async def delete_environment(
|
||||
id: str,
|
||||
config_manager: ConfigManager = Depends(get_config_manager)
|
||||
):
|
||||
logger.info(f"[delete_environment][Entry] Deleting environment {id}")
|
||||
config_manager.delete_environment(id)
|
||||
return {"message": f"Environment {id} deleted"}
|
||||
# [/DEF:delete_environment]
|
||||
|
||||
# [DEF:test_environment_connection:Function]
|
||||
# @PURPOSE: Tests the connection to a Superset environment.
|
||||
# @PARAM: id (str) - The ID of the environment to test.
|
||||
# @RETURN: dict - Success message or error.
|
||||
@router.post("/environments/{id}/test")
|
||||
async def test_environment_connection(
|
||||
id: str,
|
||||
config_manager: ConfigManager = Depends(get_config_manager)
|
||||
):
|
||||
logger.info(f"[test_environment_connection][Entry] Testing environment {id}")
|
||||
|
||||
# Find environment
|
||||
env = next((e for e in config_manager.get_environments() if e.id == id), None)
|
||||
if not env:
|
||||
raise HTTPException(status_code=404, detail=f"Environment {id} not found")
|
||||
|
||||
try:
|
||||
# Create SupersetConfig
|
||||
# Note: SupersetConfig expects 'auth' dict with specific keys
|
||||
superset_config = SupersetConfig(
|
||||
env=env.name,
|
||||
base_url=env.url,
|
||||
auth={
|
||||
"provider": "db", # Defaulting to db for now
|
||||
"username": env.username,
|
||||
"password": env.password,
|
||||
"refresh": "true"
|
||||
}
|
||||
)
|
||||
|
||||
# Initialize client (this will trigger authentication)
|
||||
client = SupersetClient(config=superset_config)
|
||||
|
||||
# Try a simple request to verify
|
||||
client.get_dashboards(query={"page_size": 1})
|
||||
|
||||
logger.info(f"[test_environment_connection][Coherence:OK] Connection successful for {id}")
|
||||
return {"status": "success", "message": "Connection successful"}
|
||||
except Exception as e:
|
||||
logger.error(f"[test_environment_connection][Coherence:Failed] Connection failed for {id}: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
# [/DEF:test_environment_connection]
|
||||
|
||||
# [DEF:validate_backup_path:Function]
|
||||
# @PURPOSE: Validates if a backup path exists and is writable.
|
||||
# @PARAM: path (str) - The path to validate.
|
||||
# @RETURN: dict - Validation result.
|
||||
@router.post("/validate-path")
|
||||
async def validate_backup_path(path_data: dict):
|
||||
path = path_data.get("path")
|
||||
if not path:
|
||||
raise HTTPException(status_code=400, detail="Path is required")
|
||||
|
||||
logger.info(f"[validate_backup_path][Entry] Validating path: {path}")
|
||||
|
||||
p = os.path.abspath(path)
|
||||
exists = os.path.exists(p)
|
||||
writable = os.access(p, os.W_OK) if exists else os.access(os.path.dirname(p), os.W_OK)
|
||||
|
||||
if not exists:
|
||||
# Try to create it
|
||||
try:
|
||||
os.makedirs(p, exist_ok=True)
|
||||
exists = True
|
||||
writable = os.access(p, os.W_OK)
|
||||
logger.info(f"[validate_backup_path][Action] Created directory: {p}")
|
||||
except Exception as e:
|
||||
logger.error(f"[validate_backup_path][Coherence:Failed] Failed to create directory: {e}")
|
||||
return {"status": "error", "message": f"Path does not exist and could not be created: {e}"}
|
||||
|
||||
if not writable:
|
||||
logger.warning(f"[validate_backup_path][Coherence:Failed] Path not writable: {p}")
|
||||
return {"status": "error", "message": "Path is not writable"}
|
||||
|
||||
logger.info(f"[validate_backup_path][Coherence:OK] Path valid: {p}")
|
||||
return {"status": "success", "message": "Path is valid and writable"}
|
||||
# [/DEF:validate_backup_path]
|
||||
|
||||
# [/DEF:SettingsRouter]
|
||||
112
backend/src/api/routes/tasks.py
Normal file → Executable file
112
backend/src/api/routes/tasks.py
Normal file → Executable file
@@ -1,57 +1,57 @@
|
||||
# [DEF:TasksRouter:Module]
|
||||
# @SEMANTICS: api, router, tasks, create, list, get
|
||||
# @PURPOSE: Defines the FastAPI router for task-related endpoints, allowing clients to create, list, and get the status of tasks.
|
||||
# @LAYER: UI (API)
|
||||
# @RELATION: Depends on the TaskManager. It is included by the main app.
|
||||
from typing import List, Dict, Any
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ...core.task_manager import TaskManager, Task
|
||||
from ...dependencies import get_task_manager
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
class CreateTaskRequest(BaseModel):
|
||||
plugin_id: str
|
||||
params: Dict[str, Any]
|
||||
|
||||
@router.post("/", response_model=Task, status_code=status.HTTP_201_CREATED)
|
||||
async def create_task(
|
||||
request: CreateTaskRequest,
|
||||
task_manager: TaskManager = Depends(get_task_manager)
|
||||
):
|
||||
"""
|
||||
Create and start a new task for a given plugin.
|
||||
"""
|
||||
try:
|
||||
task = await task_manager.create_task(
|
||||
plugin_id=request.plugin_id,
|
||||
params=request.params
|
||||
)
|
||||
return task
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
|
||||
|
||||
@router.get("/", response_model=List[Task])
|
||||
async def list_tasks(
|
||||
task_manager: TaskManager = Depends(get_task_manager)
|
||||
):
|
||||
"""
|
||||
Retrieve a list of all tasks.
|
||||
"""
|
||||
return task_manager.get_all_tasks()
|
||||
|
||||
@router.get("/{task_id}", response_model=Task)
|
||||
async def get_task(
|
||||
task_id: str,
|
||||
task_manager: TaskManager = Depends(get_task_manager)
|
||||
):
|
||||
"""
|
||||
Retrieve the details of a specific task.
|
||||
"""
|
||||
task = task_manager.get_task(task_id)
|
||||
if not task:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
|
||||
return task
|
||||
# [DEF:TasksRouter:Module]
|
||||
# @SEMANTICS: api, router, tasks, create, list, get
|
||||
# @PURPOSE: Defines the FastAPI router for task-related endpoints, allowing clients to create, list, and get the status of tasks.
|
||||
# @LAYER: UI (API)
|
||||
# @RELATION: Depends on the TaskManager. It is included by the main app.
|
||||
from typing import List, Dict, Any
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ...core.task_manager import TaskManager, Task
|
||||
from ...dependencies import get_task_manager
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
class CreateTaskRequest(BaseModel):
|
||||
plugin_id: str
|
||||
params: Dict[str, Any]
|
||||
|
||||
@router.post("/", response_model=Task, status_code=status.HTTP_201_CREATED)
|
||||
async def create_task(
|
||||
request: CreateTaskRequest,
|
||||
task_manager: TaskManager = Depends(get_task_manager)
|
||||
):
|
||||
"""
|
||||
Create and start a new task for a given plugin.
|
||||
"""
|
||||
try:
|
||||
task = await task_manager.create_task(
|
||||
plugin_id=request.plugin_id,
|
||||
params=request.params
|
||||
)
|
||||
return task
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
|
||||
|
||||
@router.get("/", response_model=List[Task])
|
||||
async def list_tasks(
|
||||
task_manager: TaskManager = Depends(get_task_manager)
|
||||
):
|
||||
"""
|
||||
Retrieve a list of all tasks.
|
||||
"""
|
||||
return task_manager.get_all_tasks()
|
||||
|
||||
@router.get("/{task_id}", response_model=Task)
|
||||
async def get_task(
|
||||
task_id: str,
|
||||
task_manager: TaskManager = Depends(get_task_manager)
|
||||
):
|
||||
"""
|
||||
Retrieve the details of a specific task.
|
||||
"""
|
||||
task = task_manager.get_task(task_id)
|
||||
if not task:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
|
||||
return task
|
||||
# [/DEF]
|
||||
155
backend/src/app.py
Normal file → Executable file
155
backend/src/app.py
Normal file → Executable file
@@ -1,77 +1,78 @@
|
||||
# [DEF:AppModule:Module]
|
||||
# @SEMANTICS: app, main, entrypoint, fastapi
|
||||
# @PURPOSE: The main entry point for the FastAPI application. It initializes the app, configures CORS, sets up dependencies, includes API routers, and defines the WebSocket endpoint for log streaming.
|
||||
# @LAYER: UI (API)
|
||||
# @RELATION: Depends on the dependency module and API route modules.
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to sys.path to allow importing superset_tool
|
||||
# Assuming app.py is in backend/src/
|
||||
project_root = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.append(str(project_root))
|
||||
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Depends
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
import asyncio
|
||||
|
||||
from .dependencies import get_task_manager
|
||||
from .core.logger import logger
|
||||
from .api.routes import plugins, tasks
|
||||
|
||||
# [DEF:App:Global]
|
||||
# @SEMANTICS: app, fastapi, instance
|
||||
# @PURPOSE: The global FastAPI application instance.
|
||||
app = FastAPI(
|
||||
title="Superset Tools API",
|
||||
description="API for managing Superset automation tools and plugins.",
|
||||
version="1.0.0",
|
||||
)
|
||||
|
||||
# Configure CORS
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"], # Adjust this in production
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
# Include API routes
|
||||
app.include_router(plugins.router, prefix="/plugins", tags=["Plugins"])
|
||||
app.include_router(tasks.router, prefix="/tasks", tags=["Tasks"])
|
||||
|
||||
# [DEF:WebSocketEndpoint:Endpoint]
|
||||
# @SEMANTICS: websocket, logs, streaming, real-time
|
||||
# @PURPOSE: Provides a WebSocket endpoint for clients to connect to and receive real-time log entries for a specific task.
|
||||
@app.websocket("/ws/logs/{task_id}")
|
||||
async def websocket_endpoint(websocket: WebSocket, task_id: str, task_manager=Depends(get_task_manager)):
|
||||
await websocket.accept()
|
||||
logger.info(f"WebSocket connection established for task {task_id}")
|
||||
try:
|
||||
# Send initial logs if any
|
||||
initial_logs = task_manager.get_task_logs(task_id)
|
||||
for log_entry in initial_logs:
|
||||
await websocket.send_json(log_entry.dict())
|
||||
|
||||
# Keep connection alive, ideally stream new logs as they come
|
||||
# This part requires a more sophisticated log streaming mechanism (e.g., queues, pub/sub)
|
||||
# For now, it will just keep the connection open and send initial logs.
|
||||
while True:
|
||||
await asyncio.sleep(1) # Keep connection alive, send heartbeat or check for new logs
|
||||
# In a real system, new logs would be pushed here
|
||||
except WebSocketDisconnect:
|
||||
logger.info(f"WebSocket connection disconnected for task {task_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"WebSocket error for task {task_id}: {e}")
|
||||
|
||||
# [/DEF]
|
||||
|
||||
# [DEF:RootEndpoint:Endpoint]
|
||||
# @SEMANTICS: root, healthcheck
|
||||
# @PURPOSE: A simple root endpoint to confirm that the API is running.
|
||||
@app.get("/")
|
||||
async def read_root():
|
||||
return {"message": "Superset Tools API is running"}
|
||||
# [/DEF]
|
||||
# [DEF:AppModule:Module]
|
||||
# @SEMANTICS: app, main, entrypoint, fastapi
|
||||
# @PURPOSE: The main entry point for the FastAPI application. It initializes the app, configures CORS, sets up dependencies, includes API routers, and defines the WebSocket endpoint for log streaming.
|
||||
# @LAYER: UI (API)
|
||||
# @RELATION: Depends on the dependency module and API route modules.
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to sys.path to allow importing superset_tool
|
||||
# Assuming app.py is in backend/src/
|
||||
project_root = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.append(str(project_root))
|
||||
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Depends
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
import asyncio
|
||||
|
||||
from .dependencies import get_task_manager
|
||||
from .core.logger import logger
|
||||
from .api.routes import plugins, tasks, settings
|
||||
|
||||
# [DEF:App:Global]
|
||||
# @SEMANTICS: app, fastapi, instance
|
||||
# @PURPOSE: The global FastAPI application instance.
|
||||
app = FastAPI(
|
||||
title="Superset Tools API",
|
||||
description="API for managing Superset automation tools and plugins.",
|
||||
version="1.0.0",
|
||||
)
|
||||
|
||||
# Configure CORS
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"], # Adjust this in production
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
# Include API routes
|
||||
app.include_router(plugins.router, prefix="/plugins", tags=["Plugins"])
|
||||
app.include_router(tasks.router, prefix="/tasks", tags=["Tasks"])
|
||||
app.include_router(settings.router, prefix="/settings", tags=["Settings"])
|
||||
|
||||
# [DEF:WebSocketEndpoint:Endpoint]
|
||||
# @SEMANTICS: websocket, logs, streaming, real-time
|
||||
# @PURPOSE: Provides a WebSocket endpoint for clients to connect to and receive real-time log entries for a specific task.
|
||||
@app.websocket("/ws/logs/{task_id}")
|
||||
async def websocket_endpoint(websocket: WebSocket, task_id: str, task_manager=Depends(get_task_manager)):
|
||||
await websocket.accept()
|
||||
logger.info(f"WebSocket connection established for task {task_id}")
|
||||
try:
|
||||
# Send initial logs if any
|
||||
initial_logs = task_manager.get_task_logs(task_id)
|
||||
for log_entry in initial_logs:
|
||||
await websocket.send_json(log_entry.dict())
|
||||
|
||||
# Keep connection alive, ideally stream new logs as they come
|
||||
# This part requires a more sophisticated log streaming mechanism (e.g., queues, pub/sub)
|
||||
# For now, it will just keep the connection open and send initial logs.
|
||||
while True:
|
||||
await asyncio.sleep(1) # Keep connection alive, send heartbeat or check for new logs
|
||||
# In a real system, new logs would be pushed here
|
||||
except WebSocketDisconnect:
|
||||
logger.info(f"WebSocket connection disconnected for task {task_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"WebSocket error for task {task_id}: {e}")
|
||||
|
||||
# [/DEF]
|
||||
|
||||
# [DEF:RootEndpoint:Endpoint]
|
||||
# @SEMANTICS: root, healthcheck
|
||||
# @PURPOSE: A simple root endpoint to confirm that the API is running.
|
||||
@app.get("/")
|
||||
async def read_root():
|
||||
return {"message": "Superset Tools API is running"}
|
||||
# [/DEF]
|
||||
|
||||
205
backend/src/core/config_manager.py
Executable file
205
backend/src/core/config_manager.py
Executable file
@@ -0,0 +1,205 @@
|
||||
# [DEF:ConfigManagerModule:Module]
|
||||
#
|
||||
# @SEMANTICS: config, manager, persistence, json
|
||||
# @PURPOSE: Manages application configuration, including loading/saving to JSON and CRUD for environments.
|
||||
# @LAYER: Core
|
||||
# @RELATION: DEPENDS_ON -> ConfigModels
|
||||
# @RELATION: CALLS -> logger
|
||||
# @RELATION: WRITES_TO -> config.json
|
||||
#
|
||||
# @INVARIANT: Configuration must always be valid according to AppConfig model.
|
||||
# @PUBLIC_API: ConfigManager
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
from .config_models import AppConfig, Environment, GlobalSettings
|
||||
from .logger import logger
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:ConfigManager:Class]
|
||||
# @PURPOSE: A class to handle application configuration persistence and management.
|
||||
# @RELATION: WRITES_TO -> config.json
|
||||
class ConfigManager:
|
||||
|
||||
# [DEF:__init__:Function]
|
||||
# @PURPOSE: Initializes the ConfigManager.
|
||||
# @PRE: isinstance(config_path, str) and len(config_path) > 0
|
||||
# @POST: self.config is an instance of AppConfig
|
||||
# @PARAM: config_path (str) - Path to the configuration file.
|
||||
def __init__(self, config_path: str = "config.json"):
|
||||
# 1. Runtime check of @PRE
|
||||
assert isinstance(config_path, str) and config_path, "config_path must be a non-empty string"
|
||||
|
||||
logger.info(f"[ConfigManager][Entry] Initializing with {config_path}")
|
||||
|
||||
# 2. Logic implementation
|
||||
self.config_path = Path(config_path)
|
||||
self.config: AppConfig = self._load_config()
|
||||
|
||||
# 3. Runtime check of @POST
|
||||
assert isinstance(self.config, AppConfig), "self.config must be an instance of AppConfig"
|
||||
|
||||
logger.info(f"[ConfigManager][Exit] Initialized")
|
||||
# [/DEF:__init__]
|
||||
|
||||
# [DEF:_load_config:Function]
|
||||
# @PURPOSE: Loads the configuration from disk or creates a default one.
|
||||
# @POST: isinstance(return, AppConfig)
|
||||
# @RETURN: AppConfig - The loaded or default configuration.
|
||||
def _load_config(self) -> AppConfig:
|
||||
logger.debug(f"[_load_config][Entry] Loading from {self.config_path}")
|
||||
|
||||
if not self.config_path.exists():
|
||||
logger.info(f"[_load_config][Action] Config file not found. Creating default.")
|
||||
default_config = AppConfig(
|
||||
environments=[],
|
||||
settings=GlobalSettings(backup_path="backups")
|
||||
)
|
||||
self._save_config_to_disk(default_config)
|
||||
return default_config
|
||||
|
||||
try:
|
||||
with open(self.config_path, "r") as f:
|
||||
data = json.load(f)
|
||||
config = AppConfig(**data)
|
||||
logger.info(f"[_load_config][Coherence:OK] Configuration loaded")
|
||||
return config
|
||||
except Exception as e:
|
||||
logger.error(f"[_load_config][Coherence:Failed] Error loading config: {e}")
|
||||
return AppConfig(
|
||||
environments=[],
|
||||
settings=GlobalSettings(backup_path="backups")
|
||||
)
|
||||
# [/DEF:_load_config]
|
||||
|
||||
# [DEF:_save_config_to_disk:Function]
|
||||
# @PURPOSE: Saves the provided configuration object to disk.
|
||||
# @PRE: isinstance(config, AppConfig)
|
||||
# @PARAM: config (AppConfig) - The configuration to save.
|
||||
def _save_config_to_disk(self, config: AppConfig):
|
||||
logger.debug(f"[_save_config_to_disk][Entry] Saving to {self.config_path}")
|
||||
|
||||
# 1. Runtime check of @PRE
|
||||
assert isinstance(config, AppConfig), "config must be an instance of AppConfig"
|
||||
|
||||
# 2. Logic implementation
|
||||
try:
|
||||
with open(self.config_path, "w") as f:
|
||||
json.dump(config.dict(), f, indent=4)
|
||||
logger.info(f"[_save_config_to_disk][Action] Configuration saved")
|
||||
except Exception as e:
|
||||
logger.error(f"[_save_config_to_disk][Coherence:Failed] Failed to save: {e}")
|
||||
# [/DEF:_save_config_to_disk]
|
||||
|
||||
# [DEF:save:Function]
|
||||
# @PURPOSE: Saves the current configuration state to disk.
|
||||
def save(self):
|
||||
self._save_config_to_disk(self.config)
|
||||
# [/DEF:save]
|
||||
|
||||
# [DEF:get_config:Function]
|
||||
# @PURPOSE: Returns the current configuration.
|
||||
# @RETURN: AppConfig - The current configuration.
|
||||
def get_config(self) -> AppConfig:
|
||||
return self.config
|
||||
# [/DEF:get_config]
|
||||
|
||||
# [DEF:update_global_settings:Function]
|
||||
# @PURPOSE: Updates the global settings and persists the change.
|
||||
# @PRE: isinstance(settings, GlobalSettings)
|
||||
# @PARAM: settings (GlobalSettings) - The new global settings.
|
||||
def update_global_settings(self, settings: GlobalSettings):
|
||||
logger.info(f"[update_global_settings][Entry] Updating settings")
|
||||
|
||||
# 1. Runtime check of @PRE
|
||||
assert isinstance(settings, GlobalSettings), "settings must be an instance of GlobalSettings"
|
||||
|
||||
# 2. Logic implementation
|
||||
self.config.settings = settings
|
||||
self.save()
|
||||
|
||||
logger.info(f"[update_global_settings][Exit] Settings updated")
|
||||
# [/DEF:update_global_settings]
|
||||
|
||||
# [DEF:get_environments:Function]
|
||||
# @PURPOSE: Returns the list of configured environments.
|
||||
# @RETURN: List[Environment] - List of environments.
|
||||
def get_environments(self) -> List[Environment]:
|
||||
return self.config.environments
|
||||
# [/DEF:get_environments]
|
||||
|
||||
# [DEF:add_environment:Function]
|
||||
# @PURPOSE: Adds a new environment to the configuration.
|
||||
# @PRE: isinstance(env, Environment)
|
||||
# @PARAM: env (Environment) - The environment to add.
|
||||
def add_environment(self, env: Environment):
|
||||
logger.info(f"[add_environment][Entry] Adding environment {env.id}")
|
||||
|
||||
# 1. Runtime check of @PRE
|
||||
assert isinstance(env, Environment), "env must be an instance of Environment"
|
||||
|
||||
# 2. Logic implementation
|
||||
# Check for duplicate ID and remove if exists
|
||||
self.config.environments = [e for e in self.config.environments if e.id != env.id]
|
||||
self.config.environments.append(env)
|
||||
self.save()
|
||||
|
||||
logger.info(f"[add_environment][Exit] Environment added")
|
||||
# [/DEF:add_environment]
|
||||
|
||||
# [DEF:update_environment:Function]
|
||||
# @PURPOSE: Updates an existing environment.
|
||||
# @PRE: isinstance(env_id, str) and len(env_id) > 0 and isinstance(updated_env, Environment)
|
||||
# @PARAM: env_id (str) - The ID of the environment to update.
|
||||
# @PARAM: updated_env (Environment) - The updated environment data.
|
||||
# @RETURN: bool - True if updated, False otherwise.
|
||||
def update_environment(self, env_id: str, updated_env: Environment) -> bool:
|
||||
logger.info(f"[update_environment][Entry] Updating {env_id}")
|
||||
|
||||
# 1. Runtime check of @PRE
|
||||
assert env_id and isinstance(env_id, str), "env_id must be a non-empty string"
|
||||
assert isinstance(updated_env, Environment), "updated_env must be an instance of Environment"
|
||||
|
||||
# 2. Logic implementation
|
||||
for i, env in enumerate(self.config.environments):
|
||||
if env.id == env_id:
|
||||
# If password is masked, keep the old one
|
||||
if updated_env.password == "********":
|
||||
updated_env.password = env.password
|
||||
|
||||
self.config.environments[i] = updated_env
|
||||
self.save()
|
||||
logger.info(f"[update_environment][Coherence:OK] Updated {env_id}")
|
||||
return True
|
||||
|
||||
logger.warning(f"[update_environment][Coherence:Failed] Environment {env_id} not found")
|
||||
return False
|
||||
# [/DEF:update_environment]
|
||||
|
||||
# [DEF:delete_environment:Function]
|
||||
# @PURPOSE: Deletes an environment by ID.
|
||||
# @PRE: isinstance(env_id, str) and len(env_id) > 0
|
||||
# @PARAM: env_id (str) - The ID of the environment to delete.
|
||||
def delete_environment(self, env_id: str):
|
||||
logger.info(f"[delete_environment][Entry] Deleting {env_id}")
|
||||
|
||||
# 1. Runtime check of @PRE
|
||||
assert env_id and isinstance(env_id, str), "env_id must be a non-empty string"
|
||||
|
||||
# 2. Logic implementation
|
||||
original_count = len(self.config.environments)
|
||||
self.config.environments = [e for e in self.config.environments if e.id != env_id]
|
||||
|
||||
if len(self.config.environments) < original_count:
|
||||
self.save()
|
||||
logger.info(f"[delete_environment][Action] Deleted {env_id}")
|
||||
else:
|
||||
logger.warning(f"[delete_environment][Coherence:Failed] Environment {env_id} not found")
|
||||
# [/DEF:delete_environment]
|
||||
|
||||
# [/DEF:ConfigManager]
|
||||
|
||||
# [/DEF:ConfigManagerModule]
|
||||
36
backend/src/core/config_models.py
Executable file
36
backend/src/core/config_models.py
Executable file
@@ -0,0 +1,36 @@
|
||||
# [DEF:ConfigModels:Module]
|
||||
# @SEMANTICS: config, models, pydantic
|
||||
# @PURPOSE: Defines the data models for application configuration using Pydantic.
|
||||
# @LAYER: Core
|
||||
# @RELATION: READS_FROM -> config.json
|
||||
# @RELATION: USED_BY -> ConfigManager
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional
|
||||
|
||||
# [DEF:Environment:DataClass]
|
||||
# @PURPOSE: Represents a Superset environment configuration.
|
||||
class Environment(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
url: str
|
||||
username: str
|
||||
password: str # Will be masked in UI
|
||||
is_default: bool = False
|
||||
# [/DEF:Environment]
|
||||
|
||||
# [DEF:GlobalSettings:DataClass]
|
||||
# @PURPOSE: Represents global application settings.
|
||||
class GlobalSettings(BaseModel):
|
||||
backup_path: str
|
||||
default_environment_id: Optional[str] = None
|
||||
# [/DEF:GlobalSettings]
|
||||
|
||||
# [DEF:AppConfig:DataClass]
|
||||
# @PURPOSE: The root configuration model containing all application settings.
|
||||
class AppConfig(BaseModel):
|
||||
environments: List[Environment] = []
|
||||
settings: GlobalSettings
|
||||
# [/DEF:AppConfig]
|
||||
|
||||
# [/DEF:ConfigModels]
|
||||
182
backend/src/core/logger.py
Normal file → Executable file
182
backend/src/core/logger.py
Normal file → Executable file
@@ -1,92 +1,92 @@
|
||||
# [DEF:LoggerModule:Module]
|
||||
# @SEMANTICS: logging, websocket, streaming, handler
|
||||
# @PURPOSE: Configures the application's logging system, including a custom handler for buffering logs and streaming them over WebSockets.
|
||||
# @LAYER: Core
|
||||
# @RELATION: Used by the main application and other modules to log events. The WebSocketLogHandler is used by the WebSocket endpoint in app.py.
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, List, Optional
|
||||
from collections import deque
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Re-using LogEntry from task_manager for consistency
|
||||
# [DEF:LogEntry:Class]
|
||||
# @SEMANTICS: log, entry, record, pydantic
|
||||
# @PURPOSE: A Pydantic model representing a single, structured log entry. This is a re-definition for consistency, as it's also defined in task_manager.py.
|
||||
class LogEntry(BaseModel):
|
||||
timestamp: datetime = Field(default_factory=datetime.utcnow)
|
||||
level: str
|
||||
message: str
|
||||
context: Optional[Dict[str, Any]] = None
|
||||
|
||||
# [/DEF]
|
||||
|
||||
# [DEF:WebSocketLogHandler:Class]
|
||||
# @SEMANTICS: logging, handler, websocket, buffer
|
||||
# @PURPOSE: A custom logging handler that captures log records into a buffer. It is designed to be extended for real-time log streaming over WebSockets.
|
||||
class WebSocketLogHandler(logging.Handler):
|
||||
"""
|
||||
A logging handler that stores log records and can be extended to send them
|
||||
over WebSockets.
|
||||
"""
|
||||
def __init__(self, capacity: int = 1000):
|
||||
super().__init__()
|
||||
self.log_buffer: deque[LogEntry] = deque(maxlen=capacity)
|
||||
# In a real implementation, you'd have a way to manage active WebSocket connections
|
||||
# e.g., self.active_connections: Set[WebSocket] = set()
|
||||
|
||||
def emit(self, record: logging.LogRecord):
|
||||
try:
|
||||
log_entry = LogEntry(
|
||||
level=record.levelname,
|
||||
message=self.format(record),
|
||||
context={
|
||||
"name": record.name,
|
||||
"pathname": record.pathname,
|
||||
"lineno": record.lineno,
|
||||
"funcName": record.funcName,
|
||||
"process": record.process,
|
||||
"thread": record.thread,
|
||||
}
|
||||
)
|
||||
self.log_buffer.append(log_entry)
|
||||
# Here you would typically send the log_entry to all active WebSocket connections
|
||||
# for real-time streaming to the frontend.
|
||||
# Example: for ws in self.active_connections: await ws.send_json(log_entry.dict())
|
||||
except Exception:
|
||||
self.handleError(record)
|
||||
|
||||
def get_recent_logs(self) -> List[LogEntry]:
|
||||
"""
|
||||
Returns a list of recent log entries from the buffer.
|
||||
"""
|
||||
return list(self.log_buffer)
|
||||
|
||||
# [/DEF]
|
||||
|
||||
# [DEF:Logger:Global]
|
||||
# @SEMANTICS: logger, global, instance
|
||||
# @PURPOSE: The global logger instance for the application, configured with both a console handler and the custom WebSocket handler.
|
||||
logger = logging.getLogger("superset_tools_app")
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
# Create a formatter
|
||||
formatter = logging.Formatter(
|
||||
'[%(asctime)s][%(levelname)s][%(name)s] %(message)s'
|
||||
)
|
||||
|
||||
# Add console handler
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setFormatter(formatter)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# Add WebSocket log handler
|
||||
websocket_log_handler = WebSocketLogHandler()
|
||||
websocket_log_handler.setFormatter(formatter)
|
||||
logger.addHandler(websocket_log_handler)
|
||||
|
||||
# Example usage:
|
||||
# logger.info("Application started", extra={"context_key": "context_value"})
|
||||
# logger.error("An error occurred", exc_info=True)
|
||||
# [DEF:LoggerModule:Module]
|
||||
# @SEMANTICS: logging, websocket, streaming, handler
|
||||
# @PURPOSE: Configures the application's logging system, including a custom handler for buffering logs and streaming them over WebSockets.
|
||||
# @LAYER: Core
|
||||
# @RELATION: Used by the main application and other modules to log events. The WebSocketLogHandler is used by the WebSocket endpoint in app.py.
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, List, Optional
|
||||
from collections import deque
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Re-using LogEntry from task_manager for consistency
|
||||
# [DEF:LogEntry:Class]
|
||||
# @SEMANTICS: log, entry, record, pydantic
|
||||
# @PURPOSE: A Pydantic model representing a single, structured log entry. This is a re-definition for consistency, as it's also defined in task_manager.py.
|
||||
class LogEntry(BaseModel):
|
||||
timestamp: datetime = Field(default_factory=datetime.utcnow)
|
||||
level: str
|
||||
message: str
|
||||
context: Optional[Dict[str, Any]] = None
|
||||
|
||||
# [/DEF]
|
||||
|
||||
# [DEF:WebSocketLogHandler:Class]
|
||||
# @SEMANTICS: logging, handler, websocket, buffer
|
||||
# @PURPOSE: A custom logging handler that captures log records into a buffer. It is designed to be extended for real-time log streaming over WebSockets.
|
||||
class WebSocketLogHandler(logging.Handler):
|
||||
"""
|
||||
A logging handler that stores log records and can be extended to send them
|
||||
over WebSockets.
|
||||
"""
|
||||
def __init__(self, capacity: int = 1000):
|
||||
super().__init__()
|
||||
self.log_buffer: deque[LogEntry] = deque(maxlen=capacity)
|
||||
# In a real implementation, you'd have a way to manage active WebSocket connections
|
||||
# e.g., self.active_connections: Set[WebSocket] = set()
|
||||
|
||||
def emit(self, record: logging.LogRecord):
|
||||
try:
|
||||
log_entry = LogEntry(
|
||||
level=record.levelname,
|
||||
message=self.format(record),
|
||||
context={
|
||||
"name": record.name,
|
||||
"pathname": record.pathname,
|
||||
"lineno": record.lineno,
|
||||
"funcName": record.funcName,
|
||||
"process": record.process,
|
||||
"thread": record.thread,
|
||||
}
|
||||
)
|
||||
self.log_buffer.append(log_entry)
|
||||
# Here you would typically send the log_entry to all active WebSocket connections
|
||||
# for real-time streaming to the frontend.
|
||||
# Example: for ws in self.active_connections: await ws.send_json(log_entry.dict())
|
||||
except Exception:
|
||||
self.handleError(record)
|
||||
|
||||
def get_recent_logs(self) -> List[LogEntry]:
|
||||
"""
|
||||
Returns a list of recent log entries from the buffer.
|
||||
"""
|
||||
return list(self.log_buffer)
|
||||
|
||||
# [/DEF]
|
||||
|
||||
# [DEF:Logger:Global]
|
||||
# @SEMANTICS: logger, global, instance
|
||||
# @PURPOSE: The global logger instance for the application, configured with both a console handler and the custom WebSocket handler.
|
||||
logger = logging.getLogger("superset_tools_app")
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
# Create a formatter
|
||||
formatter = logging.Formatter(
|
||||
'[%(asctime)s][%(levelname)s][%(name)s] %(message)s'
|
||||
)
|
||||
|
||||
# Add console handler
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setFormatter(formatter)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# Add WebSocket log handler
|
||||
websocket_log_handler = WebSocketLogHandler()
|
||||
websocket_log_handler.setFormatter(formatter)
|
||||
logger.addHandler(websocket_log_handler)
|
||||
|
||||
# Example usage:
|
||||
# logger.info("Application started", extra={"context_key": "context_value"})
|
||||
# logger.error("An error occurred", exc_info=True)
|
||||
# [/DEF]
|
||||
140
backend/src/core/plugin_base.py
Normal file → Executable file
140
backend/src/core/plugin_base.py
Normal file → Executable file
@@ -1,71 +1,71 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# [DEF:PluginBase:Class]
|
||||
# @SEMANTICS: plugin, interface, base, abstract
|
||||
# @PURPOSE: Defines the abstract base class that all plugins must implement to be recognized by the system. It enforces a common structure for plugin metadata and execution.
|
||||
# @LAYER: Core
|
||||
# @RELATION: Used by PluginLoader to identify valid plugins.
|
||||
# @INVARIANT: All plugins MUST inherit from this class.
|
||||
class PluginBase(ABC):
|
||||
"""
|
||||
Base class for all plugins.
|
||||
Plugins must inherit from this class and implement the abstract methods.
|
||||
"""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def id(self) -> str:
|
||||
"""A unique identifier for the plugin."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def name(self) -> str:
|
||||
"""A human-readable name for the plugin."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def description(self) -> str:
|
||||
"""A brief description of what the plugin does."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def version(self) -> str:
|
||||
"""The version of the plugin."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_schema(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Returns the JSON schema for the plugin's input parameters.
|
||||
This schema will be used to generate the frontend form.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def execute(self, params: Dict[str, Any]):
|
||||
"""
|
||||
Executes the plugin's logic.
|
||||
The `params` argument will be validated against the schema returned by `get_schema()`.
|
||||
"""
|
||||
pass
|
||||
# [/DEF]
|
||||
|
||||
# [DEF:PluginConfig:Class]
|
||||
# @SEMANTICS: plugin, config, schema, pydantic
|
||||
# @PURPOSE: A Pydantic model used to represent the validated configuration and metadata of a loaded plugin. This object is what gets exposed to the API layer.
|
||||
# @LAYER: Core
|
||||
# @RELATION: Instantiated by PluginLoader after validating a PluginBase instance.
|
||||
class PluginConfig(BaseModel):
|
||||
"""Pydantic model for plugin configuration."""
|
||||
id: str = Field(..., description="Unique identifier for the plugin")
|
||||
name: str = Field(..., description="Human-readable name for the plugin")
|
||||
description: str = Field(..., description="Brief description of what the plugin does")
|
||||
version: str = Field(..., description="Version of the plugin")
|
||||
input_schema: Dict[str, Any] = Field(..., description="JSON schema for input parameters", alias="schema")
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# [DEF:PluginBase:Class]
|
||||
# @SEMANTICS: plugin, interface, base, abstract
|
||||
# @PURPOSE: Defines the abstract base class that all plugins must implement to be recognized by the system. It enforces a common structure for plugin metadata and execution.
|
||||
# @LAYER: Core
|
||||
# @RELATION: Used by PluginLoader to identify valid plugins.
|
||||
# @INVARIANT: All plugins MUST inherit from this class.
|
||||
class PluginBase(ABC):
|
||||
"""
|
||||
Base class for all plugins.
|
||||
Plugins must inherit from this class and implement the abstract methods.
|
||||
"""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def id(self) -> str:
|
||||
"""A unique identifier for the plugin."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def name(self) -> str:
|
||||
"""A human-readable name for the plugin."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def description(self) -> str:
|
||||
"""A brief description of what the plugin does."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def version(self) -> str:
|
||||
"""The version of the plugin."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_schema(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Returns the JSON schema for the plugin's input parameters.
|
||||
This schema will be used to generate the frontend form.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def execute(self, params: Dict[str, Any]):
|
||||
"""
|
||||
Executes the plugin's logic.
|
||||
The `params` argument will be validated against the schema returned by `get_schema()`.
|
||||
"""
|
||||
pass
|
||||
# [/DEF]
|
||||
|
||||
# [DEF:PluginConfig:Class]
|
||||
# @SEMANTICS: plugin, config, schema, pydantic
|
||||
# @PURPOSE: A Pydantic model used to represent the validated configuration and metadata of a loaded plugin. This object is what gets exposed to the API layer.
|
||||
# @LAYER: Core
|
||||
# @RELATION: Instantiated by PluginLoader after validating a PluginBase instance.
|
||||
class PluginConfig(BaseModel):
|
||||
"""Pydantic model for plugin configuration."""
|
||||
id: str = Field(..., description="Unique identifier for the plugin")
|
||||
name: str = Field(..., description="Human-readable name for the plugin")
|
||||
description: str = Field(..., description="Brief description of what the plugin does")
|
||||
version: str = Field(..., description="Version of the plugin")
|
||||
input_schema: Dict[str, Any] = Field(..., description="JSON schema for input parameters", alias="schema")
|
||||
# [/DEF]
|
||||
251
backend/src/core/plugin_loader.py
Normal file → Executable file
251
backend/src/core/plugin_loader.py
Normal file → Executable file
@@ -1,123 +1,130 @@
|
||||
import importlib.util
|
||||
import os
|
||||
import sys # Added this line
|
||||
from typing import Dict, Type, List, Optional
|
||||
from .plugin_base import PluginBase, PluginConfig
|
||||
from jsonschema import validate
|
||||
|
||||
# [DEF:PluginLoader:Class]
|
||||
# @SEMANTICS: plugin, loader, dynamic, import
|
||||
# @PURPOSE: Scans a specified directory for Python modules, dynamically loads them, and registers any classes that are valid implementations of the PluginBase interface.
|
||||
# @LAYER: Core
|
||||
# @RELATION: Depends on PluginBase. It is used by the main application to discover and manage available plugins.
|
||||
class PluginLoader:
|
||||
"""
|
||||
Scans a directory for Python modules, loads them, and identifies classes
|
||||
that inherit from PluginBase.
|
||||
"""
|
||||
|
||||
def __init__(self, plugin_dir: str):
|
||||
self.plugin_dir = plugin_dir
|
||||
self._plugins: Dict[str, PluginBase] = {}
|
||||
self._plugin_configs: Dict[str, PluginConfig] = {}
|
||||
self._load_plugins()
|
||||
|
||||
def _load_plugins(self):
|
||||
"""
|
||||
Scans the plugin directory, imports modules, and registers valid plugins.
|
||||
"""
|
||||
if not os.path.exists(self.plugin_dir):
|
||||
os.makedirs(self.plugin_dir)
|
||||
|
||||
# Add the plugin directory's parent to sys.path to enable relative imports within plugins
|
||||
# This assumes plugin_dir is something like 'backend/src/plugins'
|
||||
# and we want 'backend/src' to be on the path for 'from ..core...' imports
|
||||
plugin_parent_dir = os.path.abspath(os.path.join(self.plugin_dir, os.pardir))
|
||||
if plugin_parent_dir not in sys.path:
|
||||
sys.path.insert(0, plugin_parent_dir)
|
||||
|
||||
for filename in os.listdir(self.plugin_dir):
|
||||
if filename.endswith(".py") and filename != "__init__.py":
|
||||
module_name = filename[:-3]
|
||||
file_path = os.path.join(self.plugin_dir, filename)
|
||||
self._load_module(module_name, file_path)
|
||||
|
||||
def _load_module(self, module_name: str, file_path: str):
|
||||
"""
|
||||
Loads a single Python module and extracts PluginBase subclasses.
|
||||
"""
|
||||
package_name = f"src.plugins.{module_name}"
|
||||
spec = importlib.util.spec_from_file_location(package_name, file_path)
|
||||
if spec is None or spec.loader is None:
|
||||
print(f"Could not load module spec for {package_name}") # Replace with proper logging
|
||||
return
|
||||
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
try:
|
||||
spec.loader.exec_module(module)
|
||||
except Exception as e:
|
||||
print(f"Error loading plugin module {module_name}: {e}") # Replace with proper logging
|
||||
return
|
||||
|
||||
for attribute_name in dir(module):
|
||||
attribute = getattr(module, attribute_name)
|
||||
if (
|
||||
isinstance(attribute, type)
|
||||
and issubclass(attribute, PluginBase)
|
||||
and attribute is not PluginBase
|
||||
):
|
||||
try:
|
||||
plugin_instance = attribute()
|
||||
self._register_plugin(plugin_instance)
|
||||
except Exception as e:
|
||||
print(f"Error instantiating plugin {attribute_name} in {module_name}: {e}") # Replace with proper logging
|
||||
|
||||
def _register_plugin(self, plugin_instance: PluginBase):
|
||||
"""
|
||||
Registers a valid plugin instance.
|
||||
"""
|
||||
plugin_id = plugin_instance.id
|
||||
if plugin_id in self._plugins:
|
||||
print(f"Warning: Duplicate plugin ID '{plugin_id}' found. Skipping.") # Replace with proper logging
|
||||
return
|
||||
|
||||
try:
|
||||
schema = plugin_instance.get_schema()
|
||||
# Basic validation to ensure it's a dictionary
|
||||
if not isinstance(schema, dict):
|
||||
raise TypeError("get_schema() must return a dictionary.")
|
||||
|
||||
plugin_config = PluginConfig(
|
||||
id=plugin_instance.id,
|
||||
name=plugin_instance.name,
|
||||
description=plugin_instance.description,
|
||||
version=plugin_instance.version,
|
||||
schema=schema,
|
||||
)
|
||||
# The following line is commented out because it requires a schema to be passed to validate against.
|
||||
# The schema provided by the plugin is the one being validated, not the data.
|
||||
# validate(instance={}, schema=schema)
|
||||
self._plugins[plugin_id] = plugin_instance
|
||||
self._plugin_configs[plugin_id] = plugin_config
|
||||
print(f"Plugin '{plugin_instance.name}' (ID: {plugin_id}) loaded successfully.") # Replace with proper logging
|
||||
except Exception as e:
|
||||
print(f"Error validating plugin '{plugin_instance.name}' (ID: {plugin_id}): {e}") # Replace with proper logging
|
||||
|
||||
|
||||
def get_plugin(self, plugin_id: str) -> Optional[PluginBase]:
|
||||
"""
|
||||
Returns a loaded plugin instance by its ID.
|
||||
"""
|
||||
return self._plugins.get(plugin_id)
|
||||
|
||||
def get_all_plugin_configs(self) -> List[PluginConfig]:
|
||||
"""
|
||||
Returns a list of all loaded plugin configurations.
|
||||
"""
|
||||
return list(self._plugin_configs.values())
|
||||
|
||||
def has_plugin(self, plugin_id: str) -> bool:
|
||||
"""
|
||||
Checks if a plugin with the given ID is loaded.
|
||||
"""
|
||||
import importlib.util
|
||||
import os
|
||||
import sys # Added this line
|
||||
from typing import Dict, Type, List, Optional
|
||||
from .plugin_base import PluginBase, PluginConfig
|
||||
from jsonschema import validate
|
||||
|
||||
# [DEF:PluginLoader:Class]
|
||||
# @SEMANTICS: plugin, loader, dynamic, import
|
||||
# @PURPOSE: Scans a specified directory for Python modules, dynamically loads them, and registers any classes that are valid implementations of the PluginBase interface.
|
||||
# @LAYER: Core
|
||||
# @RELATION: Depends on PluginBase. It is used by the main application to discover and manage available plugins.
|
||||
class PluginLoader:
|
||||
"""
|
||||
Scans a directory for Python modules, loads them, and identifies classes
|
||||
that inherit from PluginBase.
|
||||
"""
|
||||
|
||||
def __init__(self, plugin_dir: str):
|
||||
self.plugin_dir = plugin_dir
|
||||
self._plugins: Dict[str, PluginBase] = {}
|
||||
self._plugin_configs: Dict[str, PluginConfig] = {}
|
||||
self._load_plugins()
|
||||
|
||||
def _load_plugins(self):
|
||||
"""
|
||||
Scans the plugin directory, imports modules, and registers valid plugins.
|
||||
"""
|
||||
if not os.path.exists(self.plugin_dir):
|
||||
os.makedirs(self.plugin_dir)
|
||||
|
||||
# Add the plugin directory's parent to sys.path to enable relative imports within plugins
|
||||
# This assumes plugin_dir is something like 'backend/src/plugins'
|
||||
# and we want 'backend/src' to be on the path for 'from ..core...' imports
|
||||
plugin_parent_dir = os.path.abspath(os.path.join(self.plugin_dir, os.pardir))
|
||||
if plugin_parent_dir not in sys.path:
|
||||
sys.path.insert(0, plugin_parent_dir)
|
||||
|
||||
for filename in os.listdir(self.plugin_dir):
|
||||
if filename.endswith(".py") and filename != "__init__.py":
|
||||
module_name = filename[:-3]
|
||||
file_path = os.path.join(self.plugin_dir, filename)
|
||||
self._load_module(module_name, file_path)
|
||||
|
||||
def _load_module(self, module_name: str, file_path: str):
|
||||
"""
|
||||
Loads a single Python module and extracts PluginBase subclasses.
|
||||
"""
|
||||
# Try to determine the correct package prefix based on how the app is running
|
||||
if "backend.src" in __name__:
|
||||
package_prefix = "backend.src.plugins"
|
||||
else:
|
||||
package_prefix = "src.plugins"
|
||||
|
||||
package_name = f"{package_prefix}.{module_name}"
|
||||
# print(f"DEBUG: Loading plugin {module_name} as {package_name}")
|
||||
spec = importlib.util.spec_from_file_location(package_name, file_path)
|
||||
if spec is None or spec.loader is None:
|
||||
print(f"Could not load module spec for {package_name}") # Replace with proper logging
|
||||
return
|
||||
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
try:
|
||||
spec.loader.exec_module(module)
|
||||
except Exception as e:
|
||||
print(f"Error loading plugin module {module_name}: {e}") # Replace with proper logging
|
||||
return
|
||||
|
||||
for attribute_name in dir(module):
|
||||
attribute = getattr(module, attribute_name)
|
||||
if (
|
||||
isinstance(attribute, type)
|
||||
and issubclass(attribute, PluginBase)
|
||||
and attribute is not PluginBase
|
||||
):
|
||||
try:
|
||||
plugin_instance = attribute()
|
||||
self._register_plugin(plugin_instance)
|
||||
except Exception as e:
|
||||
print(f"Error instantiating plugin {attribute_name} in {module_name}: {e}") # Replace with proper logging
|
||||
|
||||
def _register_plugin(self, plugin_instance: PluginBase):
|
||||
"""
|
||||
Registers a valid plugin instance.
|
||||
"""
|
||||
plugin_id = plugin_instance.id
|
||||
if plugin_id in self._plugins:
|
||||
print(f"Warning: Duplicate plugin ID '{plugin_id}' found. Skipping.") # Replace with proper logging
|
||||
return
|
||||
|
||||
try:
|
||||
schema = plugin_instance.get_schema()
|
||||
# Basic validation to ensure it's a dictionary
|
||||
if not isinstance(schema, dict):
|
||||
raise TypeError("get_schema() must return a dictionary.")
|
||||
|
||||
plugin_config = PluginConfig(
|
||||
id=plugin_instance.id,
|
||||
name=plugin_instance.name,
|
||||
description=plugin_instance.description,
|
||||
version=plugin_instance.version,
|
||||
schema=schema,
|
||||
)
|
||||
# The following line is commented out because it requires a schema to be passed to validate against.
|
||||
# The schema provided by the plugin is the one being validated, not the data.
|
||||
# validate(instance={}, schema=schema)
|
||||
self._plugins[plugin_id] = plugin_instance
|
||||
self._plugin_configs[plugin_id] = plugin_config
|
||||
print(f"Plugin '{plugin_instance.name}' (ID: {plugin_id}) loaded successfully.") # Replace with proper logging
|
||||
except Exception as e:
|
||||
print(f"Error validating plugin '{plugin_instance.name}' (ID: {plugin_id}): {e}") # Replace with proper logging
|
||||
|
||||
|
||||
def get_plugin(self, plugin_id: str) -> Optional[PluginBase]:
|
||||
"""
|
||||
Returns a loaded plugin instance by its ID.
|
||||
"""
|
||||
return self._plugins.get(plugin_id)
|
||||
|
||||
def get_all_plugin_configs(self) -> List[PluginConfig]:
|
||||
"""
|
||||
Returns a list of all loaded plugin configurations.
|
||||
"""
|
||||
return list(self._plugin_configs.values())
|
||||
|
||||
def has_plugin(self, plugin_id: str) -> bool:
|
||||
"""
|
||||
Checks if a plugin with the given ID is loaded.
|
||||
"""
|
||||
return plugin_id in self._plugins
|
||||
262
backend/src/core/task_manager.py
Normal file → Executable file
262
backend/src/core/task_manager.py
Normal file → Executable file
@@ -1,131 +1,131 @@
|
||||
# [DEF:TaskManagerModule:Module]
|
||||
# @SEMANTICS: task, manager, lifecycle, execution, state
|
||||
# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking. It uses a thread pool to run plugins asynchronously.
|
||||
# @LAYER: Core
|
||||
# @RELATION: Depends on PluginLoader to get plugin instances. It is used by the API layer to create and query tasks.
|
||||
import asyncio
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Dict, Any, List, Optional
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Assuming PluginBase and PluginConfig are defined in plugin_base.py
|
||||
# from .plugin_base import PluginBase, PluginConfig # Not needed here, TaskManager interacts with the PluginLoader
|
||||
|
||||
# [DEF:TaskStatus:Enum]
|
||||
# @SEMANTICS: task, status, state, enum
|
||||
# @PURPOSE: Defines the possible states a task can be in during its lifecycle.
|
||||
class TaskStatus(str, Enum):
|
||||
PENDING = "PENDING"
|
||||
RUNNING = "RUNNING"
|
||||
SUCCESS = "SUCCESS"
|
||||
FAILED = "FAILED"
|
||||
|
||||
# [/DEF]
|
||||
|
||||
# [DEF:LogEntry:Class]
|
||||
# @SEMANTICS: log, entry, record, pydantic
|
||||
# @PURPOSE: A Pydantic model representing a single, structured log entry associated with a task.
|
||||
class LogEntry(BaseModel):
|
||||
timestamp: datetime = Field(default_factory=datetime.utcnow)
|
||||
level: str
|
||||
message: str
|
||||
context: Optional[Dict[str, Any]] = None
|
||||
# [/DEF]
|
||||
|
||||
# [DEF:Task:Class]
|
||||
# @SEMANTICS: task, job, execution, state, pydantic
|
||||
# @PURPOSE: A Pydantic model representing a single execution instance of a plugin, including its status, parameters, and logs.
|
||||
class Task(BaseModel):
|
||||
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
||||
plugin_id: str
|
||||
status: TaskStatus = TaskStatus.PENDING
|
||||
started_at: Optional[datetime] = None
|
||||
finished_at: Optional[datetime] = None
|
||||
user_id: Optional[str] = None
|
||||
logs: List[LogEntry] = Field(default_factory=list)
|
||||
params: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
# [/DEF]
|
||||
|
||||
# [DEF:TaskManager:Class]
|
||||
# @SEMANTICS: task, manager, lifecycle, execution, state
|
||||
# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking.
|
||||
class TaskManager:
|
||||
"""
|
||||
Manages the lifecycle of tasks, including their creation, execution, and state tracking.
|
||||
"""
|
||||
def __init__(self, plugin_loader):
|
||||
self.plugin_loader = plugin_loader
|
||||
self.tasks: Dict[str, Task] = {}
|
||||
self.executor = ThreadPoolExecutor(max_workers=5) # For CPU-bound plugin execution
|
||||
self.loop = asyncio.get_event_loop()
|
||||
# [/DEF]
|
||||
|
||||
async def create_task(self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None) -> Task:
|
||||
"""
|
||||
Creates and queues a new task for execution.
|
||||
"""
|
||||
if not self.plugin_loader.has_plugin(plugin_id):
|
||||
raise ValueError(f"Plugin with ID '{plugin_id}' not found.")
|
||||
|
||||
plugin = self.plugin_loader.get_plugin(plugin_id)
|
||||
# Validate params against plugin schema (this will be done at a higher level, e.g., API route)
|
||||
# For now, a basic check
|
||||
if not isinstance(params, dict):
|
||||
raise ValueError("Task parameters must be a dictionary.")
|
||||
|
||||
task = Task(plugin_id=plugin_id, params=params, user_id=user_id)
|
||||
self.tasks[task.id] = task
|
||||
self.loop.create_task(self._run_task(task.id)) # Schedule task for execution
|
||||
return task
|
||||
|
||||
async def _run_task(self, task_id: str):
|
||||
"""
|
||||
Internal method to execute a task.
|
||||
"""
|
||||
task = self.tasks[task_id]
|
||||
plugin = self.plugin_loader.get_plugin(task.plugin_id)
|
||||
|
||||
task.status = TaskStatus.RUNNING
|
||||
task.started_at = datetime.utcnow()
|
||||
task.logs.append(LogEntry(level="INFO", message=f"Task started for plugin '{plugin.name}'"))
|
||||
|
||||
try:
|
||||
# Execute plugin in a separate thread to avoid blocking the event loop
|
||||
# if the plugin's execute method is synchronous and potentially CPU-bound.
|
||||
# If the plugin's execute method is already async, this can be simplified.
|
||||
await self.loop.run_in_executor(
|
||||
self.executor,
|
||||
lambda: asyncio.run(plugin.execute(task.params)) if asyncio.iscoroutinefunction(plugin.execute) else plugin.execute(task.params)
|
||||
)
|
||||
task.status = TaskStatus.SUCCESS
|
||||
task.logs.append(LogEntry(level="INFO", message=f"Task completed successfully for plugin '{plugin.name}'"))
|
||||
except Exception as e:
|
||||
task.status = TaskStatus.FAILED
|
||||
task.logs.append(LogEntry(level="ERROR", message=f"Task failed: {e}", context={"error_type": type(e).__name__}))
|
||||
finally:
|
||||
task.finished_at = datetime.utcnow()
|
||||
# In a real system, you might notify clients via WebSocket here
|
||||
|
||||
def get_task(self, task_id: str) -> Optional[Task]:
|
||||
"""
|
||||
Retrieves a task by its ID.
|
||||
"""
|
||||
return self.tasks.get(task_id)
|
||||
|
||||
def get_all_tasks(self) -> List[Task]:
|
||||
"""
|
||||
Retrieves all registered tasks.
|
||||
"""
|
||||
return list(self.tasks.values())
|
||||
|
||||
def get_task_logs(self, task_id: str) -> List[LogEntry]:
|
||||
"""
|
||||
Retrieves logs for a specific task.
|
||||
"""
|
||||
task = self.tasks.get(task_id)
|
||||
return task.logs if task else []
|
||||
# [DEF:TaskManagerModule:Module]
|
||||
# @SEMANTICS: task, manager, lifecycle, execution, state
|
||||
# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking. It uses a thread pool to run plugins asynchronously.
|
||||
# @LAYER: Core
|
||||
# @RELATION: Depends on PluginLoader to get plugin instances. It is used by the API layer to create and query tasks.
|
||||
import asyncio
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Dict, Any, List, Optional
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Assuming PluginBase and PluginConfig are defined in plugin_base.py
|
||||
# from .plugin_base import PluginBase, PluginConfig # Not needed here, TaskManager interacts with the PluginLoader
|
||||
|
||||
# [DEF:TaskStatus:Enum]
|
||||
# @SEMANTICS: task, status, state, enum
|
||||
# @PURPOSE: Defines the possible states a task can be in during its lifecycle.
|
||||
class TaskStatus(str, Enum):
|
||||
PENDING = "PENDING"
|
||||
RUNNING = "RUNNING"
|
||||
SUCCESS = "SUCCESS"
|
||||
FAILED = "FAILED"
|
||||
|
||||
# [/DEF]
|
||||
|
||||
# [DEF:LogEntry:Class]
|
||||
# @SEMANTICS: log, entry, record, pydantic
|
||||
# @PURPOSE: A Pydantic model representing a single, structured log entry associated with a task.
|
||||
class LogEntry(BaseModel):
|
||||
timestamp: datetime = Field(default_factory=datetime.utcnow)
|
||||
level: str
|
||||
message: str
|
||||
context: Optional[Dict[str, Any]] = None
|
||||
# [/DEF]
|
||||
|
||||
# [DEF:Task:Class]
|
||||
# @SEMANTICS: task, job, execution, state, pydantic
|
||||
# @PURPOSE: A Pydantic model representing a single execution instance of a plugin, including its status, parameters, and logs.
|
||||
class Task(BaseModel):
|
||||
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
||||
plugin_id: str
|
||||
status: TaskStatus = TaskStatus.PENDING
|
||||
started_at: Optional[datetime] = None
|
||||
finished_at: Optional[datetime] = None
|
||||
user_id: Optional[str] = None
|
||||
logs: List[LogEntry] = Field(default_factory=list)
|
||||
params: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
# [/DEF]
|
||||
|
||||
# [DEF:TaskManager:Class]
|
||||
# @SEMANTICS: task, manager, lifecycle, execution, state
|
||||
# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking.
|
||||
class TaskManager:
|
||||
"""
|
||||
Manages the lifecycle of tasks, including their creation, execution, and state tracking.
|
||||
"""
|
||||
def __init__(self, plugin_loader):
|
||||
self.plugin_loader = plugin_loader
|
||||
self.tasks: Dict[str, Task] = {}
|
||||
self.executor = ThreadPoolExecutor(max_workers=5) # For CPU-bound plugin execution
|
||||
self.loop = asyncio.get_event_loop()
|
||||
# [/DEF]
|
||||
|
||||
async def create_task(self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None) -> Task:
|
||||
"""
|
||||
Creates and queues a new task for execution.
|
||||
"""
|
||||
if not self.plugin_loader.has_plugin(plugin_id):
|
||||
raise ValueError(f"Plugin with ID '{plugin_id}' not found.")
|
||||
|
||||
plugin = self.plugin_loader.get_plugin(plugin_id)
|
||||
# Validate params against plugin schema (this will be done at a higher level, e.g., API route)
|
||||
# For now, a basic check
|
||||
if not isinstance(params, dict):
|
||||
raise ValueError("Task parameters must be a dictionary.")
|
||||
|
||||
task = Task(plugin_id=plugin_id, params=params, user_id=user_id)
|
||||
self.tasks[task.id] = task
|
||||
self.loop.create_task(self._run_task(task.id)) # Schedule task for execution
|
||||
return task
|
||||
|
||||
async def _run_task(self, task_id: str):
|
||||
"""
|
||||
Internal method to execute a task.
|
||||
"""
|
||||
task = self.tasks[task_id]
|
||||
plugin = self.plugin_loader.get_plugin(task.plugin_id)
|
||||
|
||||
task.status = TaskStatus.RUNNING
|
||||
task.started_at = datetime.utcnow()
|
||||
task.logs.append(LogEntry(level="INFO", message=f"Task started for plugin '{plugin.name}'"))
|
||||
|
||||
try:
|
||||
# Execute plugin in a separate thread to avoid blocking the event loop
|
||||
# if the plugin's execute method is synchronous and potentially CPU-bound.
|
||||
# If the plugin's execute method is already async, this can be simplified.
|
||||
await self.loop.run_in_executor(
|
||||
self.executor,
|
||||
lambda: asyncio.run(plugin.execute(task.params)) if asyncio.iscoroutinefunction(plugin.execute) else plugin.execute(task.params)
|
||||
)
|
||||
task.status = TaskStatus.SUCCESS
|
||||
task.logs.append(LogEntry(level="INFO", message=f"Task completed successfully for plugin '{plugin.name}'"))
|
||||
except Exception as e:
|
||||
task.status = TaskStatus.FAILED
|
||||
task.logs.append(LogEntry(level="ERROR", message=f"Task failed: {e}", context={"error_type": type(e).__name__}))
|
||||
finally:
|
||||
task.finished_at = datetime.utcnow()
|
||||
# In a real system, you might notify clients via WebSocket here
|
||||
|
||||
def get_task(self, task_id: str) -> Optional[Task]:
|
||||
"""
|
||||
Retrieves a task by its ID.
|
||||
"""
|
||||
return self.tasks.get(task_id)
|
||||
|
||||
def get_all_tasks(self) -> List[Task]:
|
||||
"""
|
||||
Retrieves all registered tasks.
|
||||
"""
|
||||
return list(self.tasks.values())
|
||||
|
||||
def get_task_logs(self, task_id: str) -> List[LogEntry]:
|
||||
"""
|
||||
Retrieves logs for a specific task.
|
||||
"""
|
||||
task = self.tasks.get(task_id)
|
||||
return task.logs if task else []
|
||||
|
||||
55
backend/src/dependencies.py
Normal file → Executable file
55
backend/src/dependencies.py
Normal file → Executable file
@@ -1,24 +1,33 @@
|
||||
# [DEF:Dependencies:Module]
|
||||
# @SEMANTICS: dependency, injection, singleton, factory
|
||||
# @PURPOSE: Manages the creation and provision of shared application dependencies, such as the PluginLoader and TaskManager, to avoid circular imports.
|
||||
# @LAYER: Core
|
||||
# @RELATION: Used by the main app and API routers to get access to shared instances.
|
||||
|
||||
from pathlib import Path
|
||||
from .core.plugin_loader import PluginLoader
|
||||
from .core.task_manager import TaskManager
|
||||
|
||||
# Initialize singletons
|
||||
# Use absolute path relative to this file to ensure plugins are found regardless of CWD
|
||||
plugin_dir = Path(__file__).parent / "plugins"
|
||||
plugin_loader = PluginLoader(plugin_dir=str(plugin_dir))
|
||||
task_manager = TaskManager(plugin_loader)
|
||||
|
||||
def get_plugin_loader() -> PluginLoader:
|
||||
"""Dependency injector for the PluginLoader."""
|
||||
return plugin_loader
|
||||
|
||||
def get_task_manager() -> TaskManager:
|
||||
"""Dependency injector for the TaskManager."""
|
||||
return task_manager
|
||||
# [DEF:Dependencies:Module]
|
||||
# @SEMANTICS: dependency, injection, singleton, factory
|
||||
# @PURPOSE: Manages the creation and provision of shared application dependencies, such as the PluginLoader and TaskManager, to avoid circular imports.
|
||||
# @LAYER: Core
|
||||
# @RELATION: Used by the main app and API routers to get access to shared instances.
|
||||
|
||||
from pathlib import Path
|
||||
from .core.plugin_loader import PluginLoader
|
||||
from .core.task_manager import TaskManager
|
||||
from .core.config_manager import ConfigManager
|
||||
|
||||
# Initialize singletons
|
||||
# Use absolute path relative to this file to ensure plugins are found regardless of CWD
|
||||
project_root = Path(__file__).parent.parent.parent
|
||||
config_path = project_root / "config.json"
|
||||
config_manager = ConfigManager(config_path=str(config_path))
|
||||
|
||||
def get_config_manager() -> ConfigManager:
|
||||
"""Dependency injector for the ConfigManager."""
|
||||
return config_manager
|
||||
|
||||
plugin_dir = Path(__file__).parent / "plugins"
|
||||
plugin_loader = PluginLoader(plugin_dir=str(plugin_dir))
|
||||
task_manager = TaskManager(plugin_loader)
|
||||
|
||||
def get_plugin_loader() -> PluginLoader:
|
||||
"""Dependency injector for the PluginLoader."""
|
||||
return plugin_loader
|
||||
|
||||
def get_task_manager() -> TaskManager:
|
||||
"""Dependency injector for the TaskManager."""
|
||||
return task_manager
|
||||
# [/DEF]
|
||||
249
backend/src/plugins/backup.py
Normal file → Executable file
249
backend/src/plugins/backup.py
Normal file → Executable file
@@ -1,121 +1,130 @@
|
||||
# [DEF:BackupPlugin:Module]
|
||||
# @SEMANTICS: backup, superset, automation, dashboard, plugin
|
||||
# @PURPOSE: A plugin that provides functionality to back up Superset dashboards.
|
||||
# @LAYER: App
|
||||
# @RELATION: IMPLEMENTS -> PluginBase
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.client
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.utils
|
||||
|
||||
from typing import Dict, Any
|
||||
from pathlib import Path
|
||||
from requests.exceptions import RequestException
|
||||
|
||||
from ..core.plugin_base import PluginBase
|
||||
from superset_tool.client import SupersetClient
|
||||
from superset_tool.exceptions import SupersetAPIError
|
||||
from superset_tool.utils.logger import SupersetLogger
|
||||
from superset_tool.utils.fileio import (
|
||||
save_and_unpack_dashboard,
|
||||
archive_exports,
|
||||
sanitize_filename,
|
||||
consolidate_archive_folders,
|
||||
remove_empty_directories,
|
||||
RetentionPolicy
|
||||
)
|
||||
from superset_tool.utils.init_clients import setup_clients
|
||||
|
||||
class BackupPlugin(PluginBase):
|
||||
"""
|
||||
A plugin to back up Superset dashboards.
|
||||
"""
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
return "superset-backup"
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "Superset Dashboard Backup"
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return "Backs up all dashboards from a Superset instance."
|
||||
|
||||
@property
|
||||
def version(self) -> str:
|
||||
return "1.0.0"
|
||||
|
||||
def get_schema(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"env": {
|
||||
"type": "string",
|
||||
"title": "Environment",
|
||||
"description": "The Superset environment to back up (e.g., 'dev', 'prod').",
|
||||
"enum": ["dev", "sbx", "prod", "preprod"],
|
||||
},
|
||||
"backup_path": {
|
||||
"type": "string",
|
||||
"title": "Backup Path",
|
||||
"description": "The root directory to save backups to.",
|
||||
"default": "P:\\Superset\\010 Бекапы"
|
||||
}
|
||||
},
|
||||
"required": ["env", "backup_path"],
|
||||
}
|
||||
|
||||
async def execute(self, params: Dict[str, Any]):
|
||||
env = params["env"]
|
||||
backup_path = Path(params["backup_path"])
|
||||
|
||||
logger = SupersetLogger(log_dir=backup_path / "Logs", console=True)
|
||||
logger.info(f"[BackupPlugin][Entry] Starting backup for {env}.")
|
||||
|
||||
try:
|
||||
clients = setup_clients(logger)
|
||||
client = clients[env]
|
||||
|
||||
dashboard_count, dashboard_meta = client.get_dashboards()
|
||||
logger.info(f"[BackupPlugin][Progress] Found {dashboard_count} dashboards to export in {env}.")
|
||||
|
||||
if dashboard_count == 0:
|
||||
logger.info("[BackupPlugin][Exit] No dashboards to back up.")
|
||||
return
|
||||
|
||||
for db in dashboard_meta:
|
||||
dashboard_id = db.get('id')
|
||||
dashboard_title = db.get('dashboard_title', 'Unknown Dashboard')
|
||||
if not dashboard_id:
|
||||
continue
|
||||
|
||||
try:
|
||||
dashboard_base_dir_name = sanitize_filename(f"{dashboard_title}")
|
||||
dashboard_dir = backup_path / env.upper() / dashboard_base_dir_name
|
||||
dashboard_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
zip_content, filename = client.export_dashboard(dashboard_id)
|
||||
|
||||
save_and_unpack_dashboard(
|
||||
zip_content=zip_content,
|
||||
original_filename=filename,
|
||||
output_dir=dashboard_dir,
|
||||
unpack=False,
|
||||
logger=logger
|
||||
)
|
||||
|
||||
archive_exports(str(dashboard_dir), policy=RetentionPolicy(), logger=logger)
|
||||
|
||||
except (SupersetAPIError, RequestException, IOError, OSError) as db_error:
|
||||
logger.error(f"[BackupPlugin][Failure] Failed to export dashboard {dashboard_title} (ID: {dashboard_id}): {db_error}", exc_info=True)
|
||||
continue
|
||||
|
||||
consolidate_archive_folders(backup_path / env.upper(), logger=logger)
|
||||
remove_empty_directories(str(backup_path / env.upper()), logger=logger)
|
||||
|
||||
logger.info(f"[BackupPlugin][CoherenceCheck:Passed] Backup logic completed for {env}.")
|
||||
|
||||
except (RequestException, IOError, KeyError) as e:
|
||||
logger.critical(f"[BackupPlugin][Failure] Fatal error during backup for {env}: {e}", exc_info=True)
|
||||
raise e
|
||||
# [DEF:BackupPlugin:Module]
|
||||
# @SEMANTICS: backup, superset, automation, dashboard, plugin
|
||||
# @PURPOSE: A plugin that provides functionality to back up Superset dashboards.
|
||||
# @LAYER: App
|
||||
# @RELATION: IMPLEMENTS -> PluginBase
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.client
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.utils
|
||||
|
||||
from typing import Dict, Any
|
||||
from pathlib import Path
|
||||
from requests.exceptions import RequestException
|
||||
|
||||
from ..core.plugin_base import PluginBase
|
||||
from superset_tool.client import SupersetClient
|
||||
from superset_tool.exceptions import SupersetAPIError
|
||||
from superset_tool.utils.logger import SupersetLogger
|
||||
from superset_tool.utils.fileio import (
|
||||
save_and_unpack_dashboard,
|
||||
archive_exports,
|
||||
sanitize_filename,
|
||||
consolidate_archive_folders,
|
||||
remove_empty_directories,
|
||||
RetentionPolicy
|
||||
)
|
||||
from superset_tool.utils.init_clients import setup_clients
|
||||
from ..dependencies import get_config_manager
|
||||
|
||||
class BackupPlugin(PluginBase):
|
||||
"""
|
||||
A plugin to back up Superset dashboards.
|
||||
"""
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
return "superset-backup"
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "Superset Dashboard Backup"
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return "Backs up all dashboards from a Superset instance."
|
||||
|
||||
@property
|
||||
def version(self) -> str:
|
||||
return "1.0.0"
|
||||
|
||||
def get_schema(self) -> Dict[str, Any]:
|
||||
config_manager = get_config_manager()
|
||||
envs = [e.name for e in config_manager.get_environments()]
|
||||
default_path = config_manager.get_config().settings.backup_path
|
||||
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"env": {
|
||||
"type": "string",
|
||||
"title": "Environment",
|
||||
"description": "The Superset environment to back up.",
|
||||
"enum": envs if envs else ["dev", "prod"],
|
||||
},
|
||||
"backup_path": {
|
||||
"type": "string",
|
||||
"title": "Backup Path",
|
||||
"description": "The root directory to save backups to.",
|
||||
"default": default_path
|
||||
}
|
||||
},
|
||||
"required": ["env", "backup_path"],
|
||||
}
|
||||
|
||||
async def execute(self, params: Dict[str, Any]):
|
||||
env = params["env"]
|
||||
backup_path = Path(params["backup_path"])
|
||||
|
||||
logger = SupersetLogger(log_dir=backup_path / "Logs", console=True)
|
||||
logger.info(f"[BackupPlugin][Entry] Starting backup for {env}.")
|
||||
|
||||
try:
|
||||
config_manager = get_config_manager()
|
||||
clients = setup_clients(logger, custom_envs=config_manager.get_environments())
|
||||
client = clients.get(env)
|
||||
|
||||
if not client:
|
||||
raise ValueError(f"Environment '{env}' not found in configuration.")
|
||||
|
||||
dashboard_count, dashboard_meta = client.get_dashboards()
|
||||
logger.info(f"[BackupPlugin][Progress] Found {dashboard_count} dashboards to export in {env}.")
|
||||
|
||||
if dashboard_count == 0:
|
||||
logger.info("[BackupPlugin][Exit] No dashboards to back up.")
|
||||
return
|
||||
|
||||
for db in dashboard_meta:
|
||||
dashboard_id = db.get('id')
|
||||
dashboard_title = db.get('dashboard_title', 'Unknown Dashboard')
|
||||
if not dashboard_id:
|
||||
continue
|
||||
|
||||
try:
|
||||
dashboard_base_dir_name = sanitize_filename(f"{dashboard_title}")
|
||||
dashboard_dir = backup_path / env.upper() / dashboard_base_dir_name
|
||||
dashboard_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
zip_content, filename = client.export_dashboard(dashboard_id)
|
||||
|
||||
save_and_unpack_dashboard(
|
||||
zip_content=zip_content,
|
||||
original_filename=filename,
|
||||
output_dir=dashboard_dir,
|
||||
unpack=False,
|
||||
logger=logger
|
||||
)
|
||||
|
||||
archive_exports(str(dashboard_dir), policy=RetentionPolicy(), logger=logger)
|
||||
|
||||
except (SupersetAPIError, RequestException, IOError, OSError) as db_error:
|
||||
logger.error(f"[BackupPlugin][Failure] Failed to export dashboard {dashboard_title} (ID: {dashboard_id}): {db_error}", exc_info=True)
|
||||
continue
|
||||
|
||||
consolidate_archive_folders(backup_path / env.upper(), logger=logger)
|
||||
remove_empty_directories(str(backup_path / env.upper()), logger=logger)
|
||||
|
||||
logger.info(f"[BackupPlugin][CoherenceCheck:Passed] Backup logic completed for {env}.")
|
||||
|
||||
except (RequestException, IOError, KeyError) as e:
|
||||
logger.critical(f"[BackupPlugin][Failure] Fatal error during backup for {env}: {e}", exc_info=True)
|
||||
raise e
|
||||
# [/DEF:BackupPlugin]
|
||||
306
backend/src/plugins/migration.py
Normal file → Executable file
306
backend/src/plugins/migration.py
Normal file → Executable file
@@ -1,150 +1,158 @@
|
||||
# [DEF:MigrationPlugin:Module]
|
||||
# @SEMANTICS: migration, superset, automation, dashboard, plugin
|
||||
# @PURPOSE: A plugin that provides functionality to migrate Superset dashboards between environments.
|
||||
# @LAYER: App
|
||||
# @RELATION: IMPLEMENTS -> PluginBase
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.client
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.utils
|
||||
|
||||
from typing import Dict, Any, List
|
||||
from pathlib import Path
|
||||
import zipfile
|
||||
import re
|
||||
|
||||
from ..core.plugin_base import PluginBase
|
||||
from superset_tool.client import SupersetClient
|
||||
from superset_tool.utils.init_clients import setup_clients
|
||||
from superset_tool.utils.fileio import create_temp_file, update_yamls, create_dashboard_export
|
||||
from superset_tool.utils.logger import SupersetLogger
|
||||
|
||||
class MigrationPlugin(PluginBase):
|
||||
"""
|
||||
A plugin to migrate Superset dashboards between environments.
|
||||
"""
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
return "superset-migration"
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "Superset Dashboard Migration"
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return "Migrates dashboards between Superset environments."
|
||||
|
||||
@property
|
||||
def version(self) -> str:
|
||||
return "1.0.0"
|
||||
|
||||
def get_schema(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"from_env": {
|
||||
"type": "string",
|
||||
"title": "Source Environment",
|
||||
"description": "The environment to migrate from.",
|
||||
"enum": ["dev", "sbx", "prod", "preprod"],
|
||||
},
|
||||
"to_env": {
|
||||
"type": "string",
|
||||
"title": "Target Environment",
|
||||
"description": "The environment to migrate to.",
|
||||
"enum": ["dev", "sbx", "prod", "preprod"],
|
||||
},
|
||||
"dashboard_regex": {
|
||||
"type": "string",
|
||||
"title": "Dashboard Regex",
|
||||
"description": "A regular expression to filter dashboards to migrate.",
|
||||
},
|
||||
"replace_db_config": {
|
||||
"type": "boolean",
|
||||
"title": "Replace DB Config",
|
||||
"description": "Whether to replace the database configuration.",
|
||||
"default": False,
|
||||
},
|
||||
"from_db_id": {
|
||||
"type": "integer",
|
||||
"title": "Source DB ID",
|
||||
"description": "The ID of the source database to replace (if replacing).",
|
||||
},
|
||||
"to_db_id": {
|
||||
"type": "integer",
|
||||
"title": "Target DB ID",
|
||||
"description": "The ID of the target database to replace with (if replacing).",
|
||||
},
|
||||
},
|
||||
"required": ["from_env", "to_env", "dashboard_regex"],
|
||||
}
|
||||
|
||||
async def execute(self, params: Dict[str, Any]):
|
||||
from_env = params["from_env"]
|
||||
to_env = params["to_env"]
|
||||
dashboard_regex = params["dashboard_regex"]
|
||||
replace_db_config = params.get("replace_db_config", False)
|
||||
from_db_id = params.get("from_db_id")
|
||||
to_db_id = params.get("to_db_id")
|
||||
|
||||
logger = SupersetLogger(log_dir=Path.cwd() / "logs", console=True)
|
||||
logger.info(f"[MigrationPlugin][Entry] Starting migration from {from_env} to {to_env}.")
|
||||
|
||||
try:
|
||||
all_clients = setup_clients(logger)
|
||||
from_c = all_clients[from_env]
|
||||
to_c = all_clients[to_env]
|
||||
|
||||
_, all_dashboards = from_c.get_dashboards()
|
||||
|
||||
regex_str = str(dashboard_regex)
|
||||
dashboards_to_migrate = [
|
||||
d for d in all_dashboards if re.search(regex_str, d["dashboard_title"], re.IGNORECASE)
|
||||
]
|
||||
|
||||
if not dashboards_to_migrate:
|
||||
logger.warning("[MigrationPlugin][State] No dashboards found matching the regex.")
|
||||
return
|
||||
|
||||
db_config_replacement = None
|
||||
if replace_db_config:
|
||||
if from_db_id is None or to_db_id is None:
|
||||
raise ValueError("Source and target database IDs are required when replacing database configuration.")
|
||||
from_db = from_c.get_database(int(from_db_id))
|
||||
to_db = to_c.get_database(int(to_db_id))
|
||||
old_result = from_db.get("result", {})
|
||||
new_result = to_db.get("result", {})
|
||||
db_config_replacement = {
|
||||
"old": {"database_name": old_result.get("database_name"), "uuid": old_result.get("uuid"), "id": str(from_db.get("id"))},
|
||||
"new": {"database_name": new_result.get("database_name"), "uuid": new_result.get("uuid"), "id": str(to_db.get("id"))}
|
||||
}
|
||||
|
||||
for dash in dashboards_to_migrate:
|
||||
dash_id, dash_slug, title = dash["id"], dash.get("slug"), dash["dashboard_title"]
|
||||
|
||||
try:
|
||||
exported_content, _ = from_c.export_dashboard(dash_id)
|
||||
with create_temp_file(content=exported_content, dry_run=True, suffix=".zip", logger=logger) as tmp_zip_path:
|
||||
if not db_config_replacement:
|
||||
to_c.import_dashboard(file_name=tmp_zip_path, dash_id=dash_id, dash_slug=dash_slug)
|
||||
else:
|
||||
with create_temp_file(suffix=".dir", logger=logger) as tmp_unpack_dir:
|
||||
with zipfile.ZipFile(tmp_zip_path, "r") as zip_ref:
|
||||
zip_ref.extractall(tmp_unpack_dir)
|
||||
|
||||
update_yamls(db_configs=[db_config_replacement], path=str(tmp_unpack_dir))
|
||||
|
||||
with create_temp_file(suffix=".zip", dry_run=True, logger=logger) as tmp_new_zip:
|
||||
create_dashboard_export(zip_path=tmp_new_zip, source_paths=[str(p) for p in Path(tmp_unpack_dir).glob("**/*")])
|
||||
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug)
|
||||
|
||||
logger.info(f"[MigrationPlugin][Success] Dashboard {title} imported.")
|
||||
except Exception as exc:
|
||||
logger.error(f"[MigrationPlugin][Failure] Failed to migrate dashboard {title}: {exc}", exc_info=True)
|
||||
|
||||
logger.info("[MigrationPlugin][Exit] Migration finished.")
|
||||
|
||||
except Exception as e:
|
||||
logger.critical(f"[MigrationPlugin][Failure] Fatal error during migration: {e}", exc_info=True)
|
||||
raise e
|
||||
# [DEF:MigrationPlugin:Module]
|
||||
# @SEMANTICS: migration, superset, automation, dashboard, plugin
|
||||
# @PURPOSE: A plugin that provides functionality to migrate Superset dashboards between environments.
|
||||
# @LAYER: App
|
||||
# @RELATION: IMPLEMENTS -> PluginBase
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.client
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.utils
|
||||
|
||||
from typing import Dict, Any, List
|
||||
from pathlib import Path
|
||||
import zipfile
|
||||
import re
|
||||
|
||||
from ..core.plugin_base import PluginBase
|
||||
from superset_tool.client import SupersetClient
|
||||
from superset_tool.utils.init_clients import setup_clients
|
||||
from superset_tool.utils.fileio import create_temp_file, update_yamls, create_dashboard_export
|
||||
from ..dependencies import get_config_manager
|
||||
from superset_tool.utils.logger import SupersetLogger
|
||||
|
||||
class MigrationPlugin(PluginBase):
|
||||
"""
|
||||
A plugin to migrate Superset dashboards between environments.
|
||||
"""
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
return "superset-migration"
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "Superset Dashboard Migration"
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return "Migrates dashboards between Superset environments."
|
||||
|
||||
@property
|
||||
def version(self) -> str:
|
||||
return "1.0.0"
|
||||
|
||||
def get_schema(self) -> Dict[str, Any]:
|
||||
config_manager = get_config_manager()
|
||||
envs = [e.name for e in config_manager.get_environments()]
|
||||
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"from_env": {
|
||||
"type": "string",
|
||||
"title": "Source Environment",
|
||||
"description": "The environment to migrate from.",
|
||||
"enum": envs if envs else ["dev", "prod"],
|
||||
},
|
||||
"to_env": {
|
||||
"type": "string",
|
||||
"title": "Target Environment",
|
||||
"description": "The environment to migrate to.",
|
||||
"enum": envs if envs else ["dev", "prod"],
|
||||
},
|
||||
"dashboard_regex": {
|
||||
"type": "string",
|
||||
"title": "Dashboard Regex",
|
||||
"description": "A regular expression to filter dashboards to migrate.",
|
||||
},
|
||||
"replace_db_config": {
|
||||
"type": "boolean",
|
||||
"title": "Replace DB Config",
|
||||
"description": "Whether to replace the database configuration.",
|
||||
"default": False,
|
||||
},
|
||||
"from_db_id": {
|
||||
"type": "integer",
|
||||
"title": "Source DB ID",
|
||||
"description": "The ID of the source database to replace (if replacing).",
|
||||
},
|
||||
"to_db_id": {
|
||||
"type": "integer",
|
||||
"title": "Target DB ID",
|
||||
"description": "The ID of the target database to replace with (if replacing).",
|
||||
},
|
||||
},
|
||||
"required": ["from_env", "to_env", "dashboard_regex"],
|
||||
}
|
||||
|
||||
async def execute(self, params: Dict[str, Any]):
|
||||
from_env = params["from_env"]
|
||||
to_env = params["to_env"]
|
||||
dashboard_regex = params["dashboard_regex"]
|
||||
replace_db_config = params.get("replace_db_config", False)
|
||||
from_db_id = params.get("from_db_id")
|
||||
to_db_id = params.get("to_db_id")
|
||||
|
||||
logger = SupersetLogger(log_dir=Path.cwd() / "logs", console=True)
|
||||
logger.info(f"[MigrationPlugin][Entry] Starting migration from {from_env} to {to_env}.")
|
||||
|
||||
try:
|
||||
config_manager = get_config_manager()
|
||||
all_clients = setup_clients(logger, custom_envs=config_manager.get_environments())
|
||||
from_c = all_clients.get(from_env)
|
||||
to_c = all_clients.get(to_env)
|
||||
|
||||
if not from_c or not to_c:
|
||||
raise ValueError(f"One or both environments ('{from_env}', '{to_env}') not found in configuration.")
|
||||
|
||||
_, all_dashboards = from_c.get_dashboards()
|
||||
|
||||
regex_str = str(dashboard_regex)
|
||||
dashboards_to_migrate = [
|
||||
d for d in all_dashboards if re.search(regex_str, d["dashboard_title"], re.IGNORECASE)
|
||||
]
|
||||
|
||||
if not dashboards_to_migrate:
|
||||
logger.warning("[MigrationPlugin][State] No dashboards found matching the regex.")
|
||||
return
|
||||
|
||||
db_config_replacement = None
|
||||
if replace_db_config:
|
||||
if from_db_id is None or to_db_id is None:
|
||||
raise ValueError("Source and target database IDs are required when replacing database configuration.")
|
||||
from_db = from_c.get_database(int(from_db_id))
|
||||
to_db = to_c.get_database(int(to_db_id))
|
||||
old_result = from_db.get("result", {})
|
||||
new_result = to_db.get("result", {})
|
||||
db_config_replacement = {
|
||||
"old": {"database_name": old_result.get("database_name"), "uuid": old_result.get("uuid"), "id": str(from_db.get("id"))},
|
||||
"new": {"database_name": new_result.get("database_name"), "uuid": new_result.get("uuid"), "id": str(to_db.get("id"))}
|
||||
}
|
||||
|
||||
for dash in dashboards_to_migrate:
|
||||
dash_id, dash_slug, title = dash["id"], dash.get("slug"), dash["dashboard_title"]
|
||||
|
||||
try:
|
||||
exported_content, _ = from_c.export_dashboard(dash_id)
|
||||
with create_temp_file(content=exported_content, dry_run=True, suffix=".zip", logger=logger) as tmp_zip_path:
|
||||
if not db_config_replacement:
|
||||
to_c.import_dashboard(file_name=tmp_zip_path, dash_id=dash_id, dash_slug=dash_slug)
|
||||
else:
|
||||
with create_temp_file(suffix=".dir", logger=logger) as tmp_unpack_dir:
|
||||
with zipfile.ZipFile(tmp_zip_path, "r") as zip_ref:
|
||||
zip_ref.extractall(tmp_unpack_dir)
|
||||
|
||||
update_yamls(db_configs=[db_config_replacement], path=str(tmp_unpack_dir))
|
||||
|
||||
with create_temp_file(suffix=".zip", dry_run=True, logger=logger) as tmp_new_zip:
|
||||
create_dashboard_export(zip_path=tmp_new_zip, source_paths=[str(p) for p in Path(tmp_unpack_dir).glob("**/*")])
|
||||
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug)
|
||||
|
||||
logger.info(f"[MigrationPlugin][Success] Dashboard {title} imported.")
|
||||
except Exception as exc:
|
||||
logger.error(f"[MigrationPlugin][Failure] Failed to migrate dashboard {title}: {exc}", exc_info=True)
|
||||
|
||||
logger.info("[MigrationPlugin][Exit] Migration finished.")
|
||||
|
||||
except Exception as e:
|
||||
logger.critical(f"[MigrationPlugin][Failure] Fatal error during migration: {e}", exc_info=True)
|
||||
raise e
|
||||
# [/DEF:MigrationPlugin]
|
||||
Reference in New Issue
Block a user