feat: implement project launch script run.sh and update README

This commit is contained in:
2025-12-20 22:05:18 +03:00
parent e4dc3159cd
commit 58831c536a
23 changed files with 964 additions and 28 deletions

189
backend/backend.log Normal file
View File

@@ -0,0 +1,189 @@
INFO: Will watch for changes in these directories: ['/home/user/ss-tools/backend']
INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit)
INFO: Started reloader process [7952] using StatReload
INFO: Started server process [7968]
INFO: Waiting for application startup.
INFO: Application startup complete.
Error loading plugin module backup: No module named 'yaml'
Error loading plugin module migration: No module named 'yaml'
INFO: 127.0.0.1:36934 - "HEAD /docs HTTP/1.1" 200 OK
INFO: 127.0.0.1:55006 - "GET /settings HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:55006 - "GET /settings/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:55010 - "GET /plugins HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:55010 - "GET /plugins/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:55010 - "GET /settings HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:55010 - "GET /settings/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:55010 - "GET /plugins HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:55010 - "GET /plugins/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:55010 - "GET /settings HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:55010 - "GET /settings/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:35508 - "GET /plugins HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:35508 - "GET /plugins/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:49820 - "GET /plugins HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:49820 - "GET /plugins/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:49822 - "GET /settings HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:49822 - "GET /settings/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:49822 - "GET /plugins HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:49822 - "GET /plugins/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:49908 - "GET /settings HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:49908 - "GET /settings/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:49922 - "OPTIONS /settings/environments HTTP/1.1" 200 OK
[2025-12-20 19:14:15,576][INFO][superset_tools_app] [ConfigManager.save_config][Coherence:OK] Configuration saved context={'path': '/home/user/ss-tools/config.json'}
INFO: 127.0.0.1:49922 - "POST /settings/environments HTTP/1.1" 200 OK
INFO: 127.0.0.1:49922 - "GET /settings HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:49922 - "GET /settings/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:49922 - "OPTIONS /settings/environments/7071dab6-881f-49a2-b850-c004b3fc11c0/test HTTP/1.1" 200 OK
INFO: 127.0.0.1:36930 - "POST /settings/environments/7071dab6-881f-49a2-b850-c004b3fc11c0/test HTTP/1.1" 500 Internal Server Error
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/uvicorn/protocols/http/h11_impl.py", line 403, in run_asgi
result = await app( # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
return await self.app(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/fastapi/applications.py", line 1135, in __call__
await super().__call__(scope, receive, send)
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/starlette/applications.py", line 107, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/starlette/middleware/errors.py", line 186, in __call__
raise exc
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/starlette/middleware/errors.py", line 164, in __call__
await self.app(scope, receive, _send)
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/starlette/middleware/cors.py", line 93, in __call__
await self.simple_response(scope, receive, send, request_headers=headers)
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/starlette/middleware/cors.py", line 144, in simple_response
await self.app(scope, receive, send)
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/starlette/middleware/exceptions.py", line 63, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
await self.app(scope, receive, send)
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/starlette/routing.py", line 716, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/starlette/routing.py", line 736, in app
await route.handle(scope, receive, send)
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/starlette/routing.py", line 290, in handle
await self.app(scope, receive, send)
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/fastapi/routing.py", line 118, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/fastapi/routing.py", line 104, in app
response = await f(request)
^^^^^^^^^^^^^^^^
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/fastapi/routing.py", line 428, in app
raw_response = await run_endpoint_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/fastapi/routing.py", line 314, in run_endpoint_function
return await dependant.call(**values)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/ss-tools/backend/src/api/routes/settings.py", line 103, in test_connection
import httpx
ModuleNotFoundError: No module named 'httpx'
INFO: 127.0.0.1:45776 - "POST /settings/environments/7071dab6-881f-49a2-b850-c004b3fc11c0/test HTTP/1.1" 200 OK
INFO: 127.0.0.1:45784 - "GET /plugins HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:45784 - "GET /plugins/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:41628 - "GET /settings HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:41628 - "GET /settings/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:41628 - "GET /plugins HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:41628 - "GET /plugins/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:60184 - "GET /settings HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:60184 - "GET /settings/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:60184 - "GET /plugins HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:60184 - "GET /plugins/ HTTP/1.1" 200 OK
INFO: 127.0.0.1:60184 - "GET /settings HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:60184 - "GET /settings/ HTTP/1.1" 200 OK
WARNING: StatReload detected changes in 'src/core/plugin_loader.py'. Reloading...
INFO: Shutting down
INFO: Waiting for application shutdown.
INFO: Application shutdown complete.
INFO: Finished server process [7968]
INFO: Started server process [12178]
INFO: Waiting for application startup.
INFO: Application startup complete.
WARNING: StatReload detected changes in 'src/dependencies.py'. Reloading...
INFO: Shutting down
INFO: Waiting for application shutdown.
INFO: Application shutdown complete.
INFO: Finished server process [12178]
INFO: Started server process [12451]
INFO: Waiting for application startup.
INFO: Application startup complete.
Plugin 'Superset Dashboard Backup' (ID: superset-backup) loaded successfully.
Plugin 'Superset Dashboard Migration' (ID: superset-migration) loaded successfully.
INFO: 127.0.0.1:37334 - "GET / HTTP/1.1" 200 OK
INFO: 127.0.0.1:37334 - "GET /favicon.ico HTTP/1.1" 404 Not Found
INFO: 127.0.0.1:39932 - "GET / HTTP/1.1" 200 OK
INFO: 127.0.0.1:39932 - "GET /favicon.ico HTTP/1.1" 404 Not Found
INFO: 127.0.0.1:39932 - "GET / HTTP/1.1" 200 OK
INFO: 127.0.0.1:39932 - "GET / HTTP/1.1" 200 OK
INFO: 127.0.0.1:54900 - "GET /plugins HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:49280 - "GET /plugins HTTP/1.1" 307 Temporary Redirect
INFO: 127.0.0.1:49280 - "GET /plugins/ HTTP/1.1" 200 OK
WARNING: StatReload detected changes in 'src/api/routes/plugins.py'. Reloading...
INFO: Shutting down
INFO: Waiting for application shutdown.
INFO: Application shutdown complete.
INFO: Finished server process [12451]
INFO: Started server process [15016]
INFO: Waiting for application startup.
INFO: Application startup complete.
Plugin 'Superset Dashboard Backup' (ID: superset-backup) loaded successfully.
Plugin 'Superset Dashboard Migration' (ID: superset-migration) loaded successfully.
INFO: 127.0.0.1:59340 - "GET /plugins HTTP/1.1" 307 Temporary Redirect
DEBUG: list_plugins called. Found 0 plugins.
INFO: 127.0.0.1:59340 - "GET /plugins/ HTTP/1.1" 200 OK
WARNING: StatReload detected changes in 'src/dependencies.py'. Reloading...
INFO: Shutting down
INFO: Waiting for application shutdown.
INFO: Application shutdown complete.
INFO: Finished server process [15016]
INFO: Started server process [15257]
INFO: Waiting for application startup.
INFO: Application startup complete.
Plugin 'Superset Dashboard Backup' (ID: superset-backup) loaded successfully.
Plugin 'Superset Dashboard Migration' (ID: superset-migration) loaded successfully.
DEBUG: dependencies.py initialized. PluginLoader ID: 139922613090976
DEBUG: dependencies.py initialized. PluginLoader ID: 139922627375088
INFO: 127.0.0.1:57464 - "GET /plugins HTTP/1.1" 307 Temporary Redirect
DEBUG: get_plugin_loader called. Returning PluginLoader ID: 139922627375088
DEBUG: list_plugins called. Found 0 plugins.
INFO: 127.0.0.1:57464 - "GET /plugins/ HTTP/1.1" 200 OK
WARNING: StatReload detected changes in 'src/core/plugin_loader.py'. Reloading...
INFO: Shutting down
INFO: Waiting for application shutdown.
INFO: Application shutdown complete.
INFO: Finished server process [15257]
INFO: Started server process [15533]
INFO: Waiting for application startup.
INFO: Application startup complete.
DEBUG: Loading plugin backup as src.plugins.backup
Plugin 'Superset Dashboard Backup' (ID: superset-backup) loaded successfully.
DEBUG: Loading plugin migration as src.plugins.migration
Plugin 'Superset Dashboard Migration' (ID: superset-migration) loaded successfully.
DEBUG: dependencies.py initialized. PluginLoader ID: 140371031142384
INFO: 127.0.0.1:46470 - "GET /plugins HTTP/1.1" 307 Temporary Redirect
DEBUG: get_plugin_loader called. Returning PluginLoader ID: 140371031142384
DEBUG: list_plugins called. Found 2 plugins.
DEBUG: Plugin: superset-backup
DEBUG: Plugin: superset-migration
INFO: 127.0.0.1:46470 - "GET /plugins/ HTTP/1.1" 200 OK
WARNING: StatReload detected changes in 'src/api/routes/settings.py'. Reloading...
INFO: Shutting down
INFO: Waiting for application shutdown.
INFO: Application shutdown complete.
INFO: Finished server process [15533]
INFO: Started server process [15827]
INFO: Waiting for application startup.
INFO: Application startup complete.
INFO: Shutting down
INFO: Waiting for application shutdown.
INFO: Application shutdown complete.
INFO: Finished server process [15827]
INFO: Stopping reloader process [7952]

View File

@@ -0,0 +1,35 @@
2025-12-20 19:55:11,325 - INFO - [BackupPlugin][Entry] Starting backup for superset.
2025-12-20 19:55:11,325 - INFO - [setup_clients][Enter] Starting Superset clients initialization.
2025-12-20 19:55:11,327 - CRITICAL - [setup_clients][Failure] Critical error during client initialization: 1 validation error for SupersetConfig
base_url
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
For further information visit https://errors.pydantic.dev/2.12/v/value_error
Traceback (most recent call last):
File "/home/user/ss-tools/superset_tool/utils/init_clients.py", line 43, in setup_clients
config = SupersetConfig(
^^^^^^^^^^^^^^^
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/pydantic/main.py", line 250, in __init__
validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pydantic_core._pydantic_core.ValidationError: 1 validation error for SupersetConfig
base_url
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
For further information visit https://errors.pydantic.dev/2.12/v/value_error
2025-12-20 21:01:49,905 - INFO - [BackupPlugin][Entry] Starting backup for superset.
2025-12-20 21:01:49,906 - INFO - [setup_clients][Enter] Starting Superset clients initialization.
2025-12-20 21:01:49,988 - INFO - [setup_clients][Action] Loading environments from ConfigManager
2025-12-20 21:01:49,990 - CRITICAL - [setup_clients][Failure] Critical error during client initialization: 1 validation error for SupersetConfig
base_url
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
For further information visit https://errors.pydantic.dev/2.12/v/value_error
Traceback (most recent call last):
File "/home/user/ss-tools/superset_tool/utils/init_clients.py", line 66, in setup_clients
config = SupersetConfig(
^^^^^^^^^^^^^^^
File "/home/user/ss-tools/venv/lib/python3.12/site-packages/pydantic/main.py", line 250, in __init__
validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pydantic_core._pydantic_core.ValidationError: 1 validation error for SupersetConfig
base_url
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
For further information visit https://errors.pydantic.dev/2.12/v/value_error

View File

@@ -66,10 +66,29 @@ async def get_environments(config_manager: ConfigManager = Depends(get_config_ma
# @RETURN: Environment - The added environment.
@router.post("/environments", response_model=Environment)
async def add_environment(
env: Environment,
env: Environment,
config_manager: ConfigManager = Depends(get_config_manager)
):
logger.info(f"[add_environment][Entry] Adding environment {env.id}")
# Validate connection before adding
try:
superset_config = SupersetConfig(
env=env.name,
base_url=env.url,
auth={
"provider": "db",
"username": env.username,
"password": env.password,
"refresh": "true"
}
)
client = SupersetClient(config=superset_config)
client.get_dashboards(query={"page_size": 1})
except Exception as e:
logger.error(f"[add_environment][Coherence:Failed] Connection validation failed: {e}")
raise HTTPException(status_code=400, detail=f"Connection validation failed: {e}")
config_manager.add_environment(env)
return env
# [/DEF:add_environment]
@@ -86,6 +105,32 @@ async def update_environment(
config_manager: ConfigManager = Depends(get_config_manager)
):
logger.info(f"[update_environment][Entry] Updating environment {id}")
# If password is masked, we need the real one for validation
env_to_validate = env.copy(deep=True)
if env_to_validate.password == "********":
old_env = next((e for e in config_manager.get_environments() if e.id == id), None)
if old_env:
env_to_validate.password = old_env.password
# Validate connection before updating
try:
superset_config = SupersetConfig(
env=env_to_validate.name,
base_url=env_to_validate.url,
auth={
"provider": "db",
"username": env_to_validate.username,
"password": env_to_validate.password,
"refresh": "true"
}
)
client = SupersetClient(config=superset_config)
client.get_dashboards(query={"page_size": 1})
except Exception as e:
logger.error(f"[update_environment][Coherence:Failed] Connection validation failed: {e}")
raise HTTPException(status_code=400, detail=f"Connection validation failed: {e}")
if config_manager.update_environment(id, env):
return env
raise HTTPException(status_code=404, detail=f"Environment {id} not found")
@@ -152,34 +197,22 @@ async def test_environment_connection(
# @PARAM: path (str) - The path to validate.
# @RETURN: dict - Validation result.
@router.post("/validate-path")
async def validate_backup_path(path_data: dict):
async def validate_backup_path(
path_data: dict,
config_manager: ConfigManager = Depends(get_config_manager)
):
path = path_data.get("path")
if not path:
raise HTTPException(status_code=400, detail="Path is required")
logger.info(f"[validate_backup_path][Entry] Validating path: {path}")
p = os.path.abspath(path)
exists = os.path.exists(p)
writable = os.access(p, os.W_OK) if exists else os.access(os.path.dirname(p), os.W_OK)
valid, message = config_manager.validate_path(path)
if not exists:
# Try to create it
try:
os.makedirs(p, exist_ok=True)
exists = True
writable = os.access(p, os.W_OK)
logger.info(f"[validate_backup_path][Action] Created directory: {p}")
except Exception as e:
logger.error(f"[validate_backup_path][Coherence:Failed] Failed to create directory: {e}")
return {"status": "error", "message": f"Path does not exist and could not be created: {e}"}
if not writable:
logger.warning(f"[validate_backup_path][Coherence:Failed] Path not writable: {p}")
return {"status": "error", "message": "Path is not writable"}
if not valid:
return {"status": "error", "message": message}
logger.info(f"[validate_backup_path][Coherence:OK] Path valid: {p}")
return {"status": "success", "message": "Path is valid and writable"}
return {"status": "success", "message": message}
# [/DEF:validate_backup_path]
# [/DEF:SettingsRouter]

View File

@@ -124,6 +124,24 @@ class ConfigManager:
logger.info(f"[update_global_settings][Exit] Settings updated")
# [/DEF:update_global_settings]
# [DEF:validate_path:Function]
# @PURPOSE: Validates if a path exists and is writable.
# @PARAM: path (str) - The path to validate.
# @RETURN: tuple (bool, str) - (is_valid, message)
def validate_path(self, path: str) -> tuple[bool, str]:
p = os.path.abspath(path)
if not os.path.exists(p):
try:
os.makedirs(p, exist_ok=True)
except Exception as e:
return False, f"Path does not exist and could not be created: {e}"
if not os.access(p, os.W_OK):
return False, "Path is not writable"
return True, "Path is valid and writable"
# [/DEF:validate_path]
# [DEF:get_environments:Function]
# @PURPOSE: Returns the list of configured environments.
# @RETURN: List[Environment] - List of environments.
@@ -131,6 +149,13 @@ class ConfigManager:
return self.config.environments
# [/DEF:get_environments]
# [DEF:has_environments:Function]
# @PURPOSE: Checks if at least one environment is configured.
# @RETURN: bool - True if at least one environment exists.
def has_environments(self) -> bool:
return len(self.config.environments) > 0
# [/DEF:has_environments]
# [DEF:add_environment:Function]
# @PURPOSE: Adds a new environment to the configuration.
# @PRE: isinstance(env, Environment)

View File

@@ -58,7 +58,7 @@ class BackupPlugin(PluginBase):
"type": "string",
"title": "Environment",
"description": "The Superset environment to back up.",
"enum": envs if envs else ["dev", "prod"],
"enum": envs if envs else [],
},
"backup_path": {
"type": "string",
@@ -79,6 +79,9 @@ class BackupPlugin(PluginBase):
try:
config_manager = get_config_manager()
if not config_manager.has_environments():
raise ValueError("No Superset environments configured. Please add an environment in Settings.")
clients = setup_clients(logger, custom_envs=config_manager.get_environments())
client = clients.get(env)