4 Commits

Author SHA1 Message Date
203ce446f4 ашч 2026-01-21 14:00:48 +03:00
c96d50a3f4 fix(backend): standardize superset client init and auth
- Update plugins (debug, mapper, search) to explicitly map environment config to SupersetConfig
- Add authenticate method to SupersetClient for explicit session management
- Add get_environment method to ConfigManager
- Fix navbar dropdown hover stability in frontend with invisible bridge
2026-01-20 19:31:17 +03:00
3bbe320949 TaskLog fix 2026-01-19 17:10:43 +03:00
2d2435642d bug fixs 2026-01-19 00:07:06 +03:00
14 changed files with 408 additions and 109 deletions

View File

@@ -1 +1 @@
{"mcpServers":{"tavily":{"command":"npx","args":["-y","tavily-mcp@0.2.3"],"env":{"TAVILY_API_KEY":"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"},"alwaysAllow":[]}}}
{"mcpServers":{}}

View File

@@ -0,0 +1,35 @@
#!/usr/bin/env python3
"""Script to delete tasks with RUNNING status from the database."""
from sqlalchemy.orm import Session
from src.core.database import TasksSessionLocal
from src.models.task import TaskRecord
def delete_running_tasks():
"""Delete all tasks with RUNNING status from the database."""
session: Session = TasksSessionLocal()
try:
# Find all task records with RUNNING status
running_tasks = session.query(TaskRecord).filter(TaskRecord.status == "RUNNING").all()
if not running_tasks:
print("No RUNNING tasks found.")
return
print(f"Found {len(running_tasks)} RUNNING tasks:")
for task in running_tasks:
print(f"- Task ID: {task.id}, Type: {task.type}")
# Delete the found tasks
session.query(TaskRecord).filter(TaskRecord.status == "RUNNING").delete(synchronize_session=False)
session.commit()
print(f"Successfully deleted {len(running_tasks)} RUNNING tasks.")
except Exception as e:
session.rollback()
print(f"Error deleting tasks: {e}")
finally:
session.close()
if __name__ == "__main__":
delete_running_tasks()

View File

@@ -86,7 +86,7 @@ async def log_requests(request: Request, call_next):
app.include_router(plugins.router, prefix="/api/plugins", tags=["Plugins"])
app.include_router(tasks.router, prefix="/api/tasks", tags=["Tasks"])
app.include_router(settings.router, prefix="/api/settings", tags=["Settings"])
app.include_router(connections.router, prefix="/api/settings/connections", tags=["Connections"])
app.include_router(connections.router, prefix="/api/connections", tags=["Connections"])
app.include_router(environments.router, prefix="/api/environments", tags=["Environments"])
app.include_router(mappings.router)
app.include_router(migration.router)
@@ -167,7 +167,8 @@ if frontend_path.exists():
with belief_scope("serve_spa", f"path={file_path}"):
# Don't serve SPA for API routes that fell through
if file_path.startswith("api/"):
raise HTTPException(status_code=404, detail="API endpoint not found")
logger.info(f"[DEBUG] API route fell through to serve_spa: {file_path}")
raise HTTPException(status_code=404, detail=f"API endpoint not found: {file_path}")
full_path = frontend_path / file_path
if full_path.is_file():

View File

@@ -186,6 +186,20 @@ class ConfigManager:
return len(self.config.environments) > 0
# [/DEF:has_environments:Function]
# [DEF:get_environment:Function]
# @PURPOSE: Returns a single environment by ID.
# @PRE: self.config is set and isinstance(env_id, str) and len(env_id) > 0.
# @POST: Returns Environment object if found, None otherwise.
# @PARAM: env_id (str) - The ID of the environment to retrieve.
# @RETURN: Optional[Environment] - The environment with the given ID, or None.
def get_environment(self, env_id: str) -> Optional[Environment]:
with belief_scope("get_environment"):
for env in self.config.environments:
if env.id == env_id:
return env
return None
# [/DEF:get_environment:Function]
# [DEF:add_environment:Function]
# @PURPOSE: Adds a new environment to the configuration.
# @PRE: isinstance(env, Environment)

View File

@@ -9,7 +9,7 @@
# [SECTION: IMPORTS]
from typing import List, Dict, Optional, Tuple
from backend.src.core.logger import belief_scope
from .logger import belief_scope
from superset_tool.client import SupersetClient as BaseSupersetClient
from superset_tool.models import SupersetConfig
# [/SECTION]
@@ -17,6 +17,14 @@ from superset_tool.models import SupersetConfig
# [DEF:SupersetClient:Class]
# @PURPOSE: Extended SupersetClient for migration-specific operations.
class SupersetClient(BaseSupersetClient):
# [DEF:authenticate:Function]
# @PURPOSE: Authenticates the client using the configured credentials.
# @PRE: self.network must be initialized with valid auth configuration.
# @POST: Client is authenticated and tokens are stored.
# @RETURN: Dict[str, str] - Authentication tokens.
def authenticate(self):
with belief_scope("SupersetClient.authenticate"):
return self.network.authenticate()
# [DEF:get_databases_summary:Function]
# @PURPOSE: Fetch a summary of databases including uuid, name, and engine.

View File

@@ -145,7 +145,19 @@ class DebugPlugin(PluginBase):
if not env_config:
raise ValueError(f"Environment '{name}' not found.")
client = SupersetClient(env_config)
# Map Environment model to SupersetConfig
from superset_tool.models import SupersetConfig
superset_config = SupersetConfig(
env=env_config.name,
base_url=env_config.url,
auth={
"provider": "db", # Defaulting to db provider
"username": env_config.username,
"password": env_config.password,
"refresh": "false"
}
)
client = SupersetClient(superset_config)
client.authenticate()
count, dbs = client.get_databases()
results[name] = {
@@ -176,7 +188,19 @@ class DebugPlugin(PluginBase):
if not env_config:
raise ValueError(f"Environment '{env_name}' not found.")
client = SupersetClient(env_config)
# Map Environment model to SupersetConfig
from superset_tool.models import SupersetConfig
superset_config = SupersetConfig(
env=env_config.name,
base_url=env_config.url,
auth={
"provider": "db", # Defaulting to db provider
"username": env_config.username,
"password": env_config.password,
"refresh": "false"
}
)
client = SupersetClient(superset_config)
client.authenticate()
dataset_response = client.get_dataset(dataset_id)

View File

@@ -137,13 +137,25 @@ class MapperPlugin(PluginBase):
# Get config and initialize client
from ..dependencies import get_config_manager
from superset_tool.models import SupersetConfig
config_manager = get_config_manager()
env_config = config_manager.get_environment(env_name)
if not env_config:
logger.error(f"[MapperPlugin.execute][State] Environment '{env_name}' not found.")
raise ValueError(f"Environment '{env_name}' not found in configuration.")
client = SupersetClient(env_config)
# Map Environment model to SupersetConfig
superset_config = SupersetConfig(
env=env_config.name,
base_url=env_config.url,
auth={
"provider": "db", # Defaulting to db provider
"username": env_config.username,
"password": env_config.password,
"refresh": "false"
}
)
client = SupersetClient(superset_config)
client.authenticate()
postgres_config = None

View File

@@ -106,13 +106,25 @@ class SearchPlugin(PluginBase):
# Get config and initialize client
from ..dependencies import get_config_manager
from superset_tool.models import SupersetConfig
config_manager = get_config_manager()
env_config = config_manager.get_environment(env_name)
if not env_config:
logger.error(f"[SearchPlugin.execute][State] Environment '{env_name}' not found.")
raise ValueError(f"Environment '{env_name}' not found in configuration.")
client = SupersetClient(env_config)
# Map Environment model to SupersetConfig
superset_config = SupersetConfig(
env=env_config.name,
base_url=env_config.url,
auth={
"provider": "db", # Defaulting to db provider
"username": env_config.username,
"password": env_config.password,
"refresh": "false"
}
)
client = SupersetClient(superset_config)
client.authenticate()
logger.info(f"[SearchPlugin.execute][Action] Searching for pattern: '{search_query}' in environment: {env_name}")

Binary file not shown.

99
backend/test_fix.py Normal file
View File

@@ -0,0 +1,99 @@
#!/usr/bin/env python3
"""Test script to verify the fixes for SupersetClient initialization."""
import sys
sys.path.insert(0, '.')
from src.core.config_manager import ConfigManager
from src.core.config_models import Environment
from src.plugins.search import SearchPlugin
from src.plugins.mapper import MapperPlugin
from src.plugins.debug import DebugPlugin
def test_config_manager():
"""Test ConfigManager methods."""
print("Testing ConfigManager...")
try:
config_manager = ConfigManager()
print(f" ConfigManager initialized")
# Test get_environment method
if hasattr(config_manager, 'get_environment'):
print(f" get_environment method exists")
# Add a test environment if none exists
if not config_manager.has_environments():
test_env = Environment(
id="test-env",
name="Test Environment",
url="http://localhost:8088",
username="admin",
password="admin"
)
config_manager.add_environment(test_env)
print(f" Added test environment: {test_env.name}")
# Test retrieving environment
envs = config_manager.get_environments()
if envs:
test_env_id = envs[0].id
env_config = config_manager.get_environment(test_env_id)
print(f" Successfully retrieved environment: {env_config.name}")
return True
else:
print(f" No environments available (add one in settings)")
return False
except Exception as e:
print(f" Error: {e}")
return False
def test_plugins():
"""Test plugin initialization."""
print("\nTesting plugins...")
plugins = [
("Search Plugin", SearchPlugin()),
("Mapper Plugin", MapperPlugin()),
("Debug Plugin", DebugPlugin())
]
all_ok = True
for name, plugin in plugins:
print(f"\nTesting {name}...")
try:
plugin_id = plugin.id
plugin_name = plugin.name
plugin_version = plugin.version
schema = plugin.get_schema()
print(f" ✓ ID: {plugin_id}")
print(f" ✓ Name: {plugin_name}")
print(f" ✓ Version: {plugin_version}")
print(f" ✓ Schema: {schema}")
except Exception as e:
print(f" ✗ Error: {e}")
all_ok = False
return all_ok
def main():
"""Main test function."""
print("=" * 50)
print("Superset Tools Fix Verification")
print("=" * 50)
config_ok = test_config_manager()
plugins_ok = test_plugins()
print("\n" + "=" * 50)
if config_ok and plugins_ok:
print("✅ All fixes verified successfully!")
else:
print("❌ Some tests failed")
sys.exit(1)
if __name__ == "__main__":
main()

View File

@@ -39,7 +39,7 @@
<button class="text-gray-600 hover:text-blue-600 font-medium pb-1 {$page.url.pathname.startsWith('/tools') ? 'text-blue-600 border-b-2 border-blue-600' : ''}">
Tools
</button>
<div class="absolute hidden group-hover:block bg-white shadow-lg rounded-md mt-1 py-2 w-48 z-10 border border-gray-100">
<div class="absolute hidden group-hover:block bg-white shadow-lg rounded-md mt-1 py-2 w-48 z-10 border border-gray-100 before:absolute before:-top-2 before:left-0 before:right-0 before:h-2 before:content-[''] right-0">
<a href="/tools/search" class="block px-4 py-2 text-sm text-gray-700 hover:bg-blue-50 hover:text-blue-600">Dataset Search</a>
<a href="/tools/mapper" class="block px-4 py-2 text-sm text-gray-700 hover:bg-blue-50 hover:text-blue-600">Dataset Mapper</a>
<a href="/tools/debug" class="block px-4 py-2 text-sm text-gray-700 hover:bg-blue-50 hover:text-blue-600">System Debug</a>
@@ -49,7 +49,7 @@
<button class="text-gray-600 hover:text-blue-600 font-medium pb-1 {$page.url.pathname.startsWith('/settings') ? 'text-blue-600 border-b-2 border-blue-600' : ''}">
Settings
</button>
<div class="absolute hidden group-hover:block bg-white shadow-lg rounded-md mt-1 py-2 w-48 z-10 border border-gray-100">
<div class="absolute hidden group-hover:block bg-white shadow-lg rounded-md mt-1 py-2 w-48 z-10 border border-gray-100 before:absolute before:-top-2 before:left-0 before:right-0 before:h-2 before:content-[''] right-0">
<a href="/settings" class="block px-4 py-2 text-sm text-gray-700 hover:bg-blue-50 hover:text-blue-600">General Settings</a>
<a href="/settings/connections" class="block px-4 py-2 text-sm text-gray-700 hover:bg-blue-50 hover:text-blue-600">Connections</a>
</div>

View File

@@ -1,15 +1,16 @@
<!-- [DEF:TaskLogViewer:Component] -->
<!--
@SEMANTICS: task, log, viewer, modal
@PURPOSE: Displays detailed logs for a specific task in a modal.
@SEMANTICS: task, log, viewer, modal, inline
@PURPOSE: Displays detailed logs for a specific task in a modal or inline.
@LAYER: UI
@RELATION: USES -> frontend/src/lib/api.js (inferred)
@RELATION: USES -> frontend/src/services/taskService.js
-->
<script>
import { createEventDispatcher, onMount, onDestroy } from 'svelte';
import { getTaskLogs } from '../services/taskService.js';
export let show = false;
export let inline = false;
export let taskId = null;
export let taskStatus = null; // To know if we should poll
@@ -22,19 +23,27 @@
let autoScroll = true;
let logContainer;
$: shouldShow = inline || show;
// [DEF:fetchLogs:Function]
// @PURPOSE: Fetches logs for the current task.
// @PRE: taskId must be set.
// @POST: logs array is updated with data from taskService.
/**
* @purpose Fetches logs for the current task.
* @pre taskId must be set.
* @post logs array is updated with data from taskService.
* @side_effect Updates logs, loading, and error state.
*/
async function fetchLogs() {
if (!taskId) return;
console.log(`[fetchLogs][Action] Fetching logs for task context={{'taskId': '${taskId}'}}`);
try {
logs = await getTaskLogs(taskId);
if (autoScroll) {
scrollToBottom();
}
console.log(`[fetchLogs][Coherence:OK] Logs fetched context={{'count': ${logs.length}}}`);
} catch (e) {
error = e.message;
console.error(`[fetchLogs][Coherence:Failed] Error fetching logs context={{'error': '${e.message}'}}`);
} finally {
loading = false;
}
@@ -42,9 +51,11 @@
// [/DEF:fetchLogs:Function]
// [DEF:scrollToBottom:Function]
// @PURPOSE: Scrolls the log container to the bottom.
// @PRE: logContainer element must be bound.
// @POST: logContainer scrollTop is set to scrollHeight.
/**
* @purpose Scrolls the log container to the bottom.
* @pre logContainer element must be bound.
* @post logContainer scrollTop is set to scrollHeight.
*/
function scrollToBottom() {
if (logContainer) {
setTimeout(() => {
@@ -55,9 +66,11 @@
// [/DEF:scrollToBottom:Function]
// [DEF:handleScroll:Function]
// @PURPOSE: Updates auto-scroll preference based on scroll position.
// @PRE: logContainer scroll event fired.
// @POST: autoScroll boolean is updated.
/**
* @purpose Updates auto-scroll preference based on scroll position.
* @pre logContainer scroll event fired.
* @post autoScroll boolean is updated.
*/
function handleScroll() {
if (!logContainer) return;
// If user scrolls up, disable auto-scroll
@@ -68,9 +81,11 @@
// [/DEF:handleScroll:Function]
// [DEF:close:Function]
// @PURPOSE: Closes the log viewer modal.
// @PRE: Modal is open.
// @POST: Modal is closed and close event is dispatched.
/**
* @purpose Closes the log viewer modal.
* @pre Modal is open.
* @post Modal is closed and close event is dispatched.
*/
function close() {
dispatch('close');
show = false;
@@ -78,9 +93,11 @@
// [/DEF:close:Function]
// [DEF:getLogLevelColor:Function]
// @PURPOSE: Returns the CSS color class for a given log level.
// @PRE: level string is provided.
// @POST: Returns tailwind color class string.
/**
* @purpose Returns the CSS color class for a given log level.
* @pre level string is provided.
* @post Returns tailwind color class string.
*/
function getLogLevelColor(level) {
switch (level) {
case 'INFO': return 'text-blue-600';
@@ -92,8 +109,10 @@
}
// [/DEF:getLogLevelColor:Function]
// React to changes in show/taskId
$: if (show && taskId) {
// React to changes in show/taskId/taskStatus
$: if (shouldShow && taskId) {
if (interval) clearInterval(interval);
logs = [];
loading = true;
error = "";
@@ -108,16 +127,59 @@
}
// [DEF:onDestroy:Function]
// @PURPOSE: Cleans up the polling interval.
// @PRE: Component is being destroyed.
// @POST: Polling interval is cleared.
/**
* @purpose Cleans up the polling interval.
* @pre Component is being destroyed.
* @post Polling interval is cleared.
*/
onDestroy(() => {
if (interval) clearInterval(interval);
});
// [/DEF:onDestroy:Function]
</script>
{#if show}
{#if shouldShow}
{#if inline}
<div class="flex flex-col h-full w-full p-4">
<div class="flex justify-between items-center mb-4">
<h3 class="text-lg font-medium text-gray-900">
Task Logs <span class="text-sm text-gray-500 font-normal">({taskId})</span>
</h3>
<button on:click={fetchLogs} class="text-sm text-indigo-600 hover:text-indigo-900">Refresh</button>
</div>
<div class="flex-1 border rounded-md bg-gray-50 p-4 overflow-y-auto font-mono text-sm"
bind:this={logContainer}
on:scroll={handleScroll}>
{#if loading && logs.length === 0}
<p class="text-gray-500 text-center">Loading logs...</p>
{:else if error}
<p class="text-red-500 text-center">{error}</p>
{:else if logs.length === 0}
<p class="text-gray-500 text-center">No logs available.</p>
{:else}
{#each logs as log}
<div class="mb-1 hover:bg-gray-100 p-1 rounded">
<span class="text-gray-400 text-xs mr-2">
{new Date(log.timestamp).toLocaleTimeString()}
</span>
<span class="font-bold text-xs mr-2 w-16 inline-block {getLogLevelColor(log.level)}">
[{log.level}]
</span>
<span class="text-gray-800 break-words">
{log.message}
</span>
{#if log.context}
<div class="ml-24 text-xs text-gray-500 mt-1 bg-gray-100 p-1 rounded overflow-x-auto">
<pre>{JSON.stringify(log.context, null, 2)}</pre>
</div>
{/if}
</div>
{/each}
{/if}
</div>
</div>
{:else}
<div class="fixed inset-0 z-50 overflow-y-auto" aria-labelledby="modal-title" role="dialog" aria-modal="true">
<div class="flex items-end justify-center min-h-screen pt-4 px-4 pb-20 text-center sm:block sm:p-0">
<!-- Background overlay -->
@@ -179,5 +241,6 @@
</div>
</div>
</div>
{/if}
{/if}
<!-- [/DEF:TaskLogViewer:Component] -->

View File

@@ -1,3 +1,11 @@
<!-- [DEF:TaskManagementPage:Component] -->
<!--
@SEMANTICS: tasks, management, history, logs
@PURPOSE: Page for managing and monitoring tasks.
@LAYER: Page
@RELATION: USES -> TaskList
@RELATION: USES -> TaskLogViewer
-->
<script>
import { onMount, onDestroy } from 'svelte';
import { getTasks, createTask, getEnvironmentsList } from '../../lib/api';
@@ -14,11 +22,13 @@
let selectedEnvId = '';
// [DEF:loadInitialData:Function]
/* @PURPOSE: Loads tasks and environments on page initialization.
@PRE: API must be reachable.
@POST: tasks and environments variables are populated.
/**
* @purpose Loads tasks and environments on page initialization.
* @pre API must be reachable.
* @post tasks and environments variables are populated.
*/
async function loadInitialData() {
console.log("[loadInitialData][Action] Loading initial tasks and environments");
try {
loading = true;
const [tasksData, envsData] = await Promise.all([
@@ -27,8 +37,9 @@
]);
tasks = tasksData;
environments = envsData;
console.log(`[loadInitialData][Coherence:OK] Data loaded context={{'tasks': ${tasks.length}, 'envs': ${environments.length}}}`);
} catch (error) {
console.error('Failed to load tasks data:', error);
console.error(`[loadInitialData][Coherence:Failed] Failed to load tasks data context={{'error': '${error.message}'}}`);
} finally {
loading = false;
}
@@ -36,9 +47,10 @@
// [/DEF:loadInitialData:Function]
// [DEF:refreshTasks:Function]
/* @PURPOSE: Periodically refreshes the task list.
@PRE: API must be reachable.
@POST: tasks variable is updated if data is valid.
/**
* @purpose Periodically refreshes the task list.
* @pre API must be reachable.
* @post tasks variable is updated if data is valid.
*/
async function refreshTasks() {
try {
@@ -48,25 +60,28 @@
tasks = data;
}
} catch (error) {
console.error('Failed to refresh tasks:', error);
console.error(`[refreshTasks][Coherence:Failed] Failed to refresh tasks context={{'error': '${error.message}'}}`);
}
}
// [/DEF:refreshTasks:Function]
// [DEF:handleSelectTask:Function]
/* @PURPOSE: Updates the selected task ID when a task is clicked.
@PRE: event.detail.id must be provided.
@POST: selectedTaskId is updated.
/**
* @purpose Updates the selected task ID when a task is clicked.
* @pre event.detail.id must be provided.
* @post selectedTaskId is updated.
*/
function handleSelectTask(event) {
selectedTaskId = event.detail.id;
console.log(`[handleSelectTask][Action] Task selected context={{'taskId': '${selectedTaskId}'}}`);
}
// [/DEF:handleSelectTask:Function]
// [DEF:handleRunBackup:Function]
/* @PURPOSE: Triggers a manual backup task for the selected environment.
@PRE: selectedEnvId must not be empty.
@POST: Backup task is created and task list is refreshed.
/**
* @purpose Triggers a manual backup task for the selected environment.
* @pre selectedEnvId must not be empty.
* @post Backup task is created and task list is refreshed.
*/
async function handleRunBackup() {
if (!selectedEnvId) {
@@ -74,14 +89,16 @@
return;
}
console.log(`[handleRunBackup][Action] Starting backup for env context={{'envId': '${selectedEnvId}'}}`);
try {
const task = await createTask('superset-backup', { environment_id: selectedEnvId });
addToast('Backup task started', 'success');
showBackupModal = false;
selectedTaskId = task.id;
await refreshTasks();
console.log(`[handleRunBackup][Coherence:OK] Backup task created context={{'taskId': '${task.id}'}}`);
} catch (error) {
console.error('Failed to start backup:', error);
console.error(`[handleRunBackup][Coherence:Failed] Failed to start backup context={{'error': '${error.message}'}}`);
}
}
// [/DEF:handleRunBackup:Function]
@@ -117,7 +134,11 @@
<h2 class="text-lg font-semibold mb-3 text-gray-700">Task Details & Logs</h2>
{#if selectedTaskId}
<div class="bg-white rounded-lg shadow-lg h-[600px] flex flex-col">
<TaskLogViewer taskId={selectedTaskId} />
<TaskLogViewer
taskId={selectedTaskId}
taskStatus={tasks.find(t => t.id === selectedTaskId)?.status}
inline={true}
/>
</div>
{:else}
<div class="bg-gray-50 border-2 border-dashed border-gray-300 rounded-lg h-[600px] flex items-center justify-center text-gray-500">
@@ -162,3 +183,5 @@
</div>
</div>
{/if}
<!-- [/DEF:TaskManagementPage:Component] -->

View File

@@ -20,7 +20,7 @@ from requests import Response
from superset_tool.models import SupersetConfig
from superset_tool.exceptions import ExportError, InvalidZipFormatError
from superset_tool.utils.fileio import get_filename_from_headers
from superset_tool.utils.logger import SupersetLogger
from superset_tool.utils.logger import SupersetLogger, belief_scope
from superset_tool.utils.network import APIClient
# [/SECTION]
@@ -29,6 +29,14 @@ from superset_tool.utils.network import APIClient
# @RELATION: CREATES_INSTANCE_OF -> APIClient
# @RELATION: USES -> SupersetConfig
class SupersetClient:
# [DEF:authenticate:Function]
# @PURPOSE: Authenticates the client using the configured credentials.
# @PRE: self.network must be initialized with valid auth configuration.
# @POST: Client is authenticated and tokens are stored.
# @RETURN: Dict[str, str] - Authentication tokens.
def authenticate(self):
with belief_scope("SupersetClient.authenticate"):
return self.network.authenticate()
# [DEF:__init__:Function]
# @PURPOSE: Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент.
# @PRE: `config` должен быть валидным объектом SupersetConfig.