diff --git a/backend/get_full_key.py b/backend/get_full_key.py new file mode 100644 index 00000000..ee784b0c --- /dev/null +++ b/backend/get_full_key.py @@ -0,0 +1 @@ +{"print(f'Length": {"else": "print('Provider not found')\ndb.close()"}} \ No newline at end of file diff --git a/backend/src/api/routes/llm.py b/backend/src/api/routes/llm.py index e4597242..43ea65c8 100644 --- a/backend/src/api/routes/llm.py +++ b/backend/src/api/routes/llm.py @@ -143,6 +143,15 @@ async def test_connection( raise HTTPException(status_code=404, detail="Provider not found") api_key = service.get_decrypted_api_key(provider_id) + + # Check if API key was successfully decrypted + if not api_key: + logger.error(f"[llm_routes][test_connection] Failed to decrypt API key for provider {provider_id}") + raise HTTPException( + status_code=500, + detail="Failed to decrypt API key. The provider may have been encrypted with a different encryption key. Please update the provider with a new API key." + ) + client = LLMClient( provider_type=LLMProviderType(db_provider.provider_type), api_key=api_key, @@ -173,6 +182,13 @@ async def test_provider_config( from ...plugins.llm_analysis.service import LLMClient logger.info(f"[llm_routes][test_provider_config][Action] Testing config for {config.name}") + # Check if API key is provided + if not config.api_key or config.api_key == "********": + raise HTTPException( + status_code=400, + detail="API key is required for testing connection" + ) + client = LLMClient( provider_type=config.provider_type, api_key=config.api_key, diff --git a/backend/src/app.py b/backend/src/app.py index a6f6a971..9d72664e 100755 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -18,6 +18,7 @@ import asyncio import os from .dependencies import get_task_manager, get_scheduler_service +from .core.utils.network import NetworkError from .core.logger import logger, belief_scope from .api.routes import plugins, tasks, settings, environments, mappings, migration, connections, git, storage, admin, llm from .api import auth @@ -77,13 +78,34 @@ app.add_middleware( # @POST: Logs request and response details. # @PARAM: request (Request) - The incoming request object. # @PARAM: call_next (Callable) - The next middleware or route handler. +@app.exception_handler(NetworkError) +async def network_error_handler(request: Request, exc: NetworkError): + with belief_scope("network_error_handler"): + logger.error(f"Network error: {exc}") + return HTTPException( + status_code=503, + detail="Environment unavailable. Please check if the Superset instance is running." + ) + @app.middleware("http") async def log_requests(request: Request, call_next): - with belief_scope("log_requests", f"{request.method} {request.url.path}"): - logger.info(f"[DEBUG] Incoming request: {request.method} {request.url.path}") + # Avoid spamming logs for polling endpoints + is_polling = request.url.path.endswith("/api/tasks") and request.method == "GET" + + if not is_polling: + logger.info(f"Incoming request: {request.method} {request.url.path}") + + try: response = await call_next(request) - logger.info(f"[DEBUG] Response status: {response.status_code} for {request.url.path}") + if not is_polling: + logger.info(f"Response status: {response.status_code} for {request.url.path}") return response + except NetworkError as e: + logger.error(f"Network error caught in middleware: {e}") + raise HTTPException( + status_code=503, + detail="Environment unavailable. Please check if the Superset instance is running." + ) # [/DEF:log_requests:Function] # Include API routes diff --git a/backend/src/core/task_manager/persistence.py b/backend/src/core/task_manager/persistence.py index 37eacd41..d4e420db 100644 --- a/backend/src/core/task_manager/persistence.py +++ b/backend/src/core/task_manager/persistence.py @@ -36,6 +36,7 @@ class TaskPersistenceService: # @PRE: isinstance(task, Task) # @POST: Task record created or updated in database. # @PARAM: task (Task) - The task object to persist. + # @SIDE_EFFECT: Writes to task_records table in tasks.db def persist_task(self, task: Task) -> None: with belief_scope("TaskPersistenceService.persist_task", f"task_id={task.id}"): session: Session = TasksSessionLocal() @@ -50,8 +51,19 @@ class TaskPersistenceService: record.environment_id = task.params.get("environment_id") or task.params.get("source_env_id") record.started_at = task.started_at record.finished_at = task.finished_at - record.params = task.params - record.result = task.result + + # Ensure params and result are JSON serializable + def json_serializable(obj): + if isinstance(obj, dict): + return {k: json_serializable(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [json_serializable(v) for v in obj] + elif isinstance(obj, datetime): + return obj.isoformat() + return obj + + record.params = json_serializable(task.params) + record.result = json_serializable(task.result) # Store logs as JSON, converting datetime to string record.logs = [] @@ -59,6 +71,9 @@ class TaskPersistenceService: log_dict = log.dict() if isinstance(log_dict.get('timestamp'), datetime): log_dict['timestamp'] = log_dict['timestamp'].isoformat() + # Also clean up any datetimes in context + if log_dict.get('context'): + log_dict['context'] = json_serializable(log_dict['context']) record.logs.append(log_dict) # Extract error if failed diff --git a/backend/src/core/utils/network.py b/backend/src/core/utils/network.py index 164e2ca4..d7913e0d 100644 --- a/backend/src/core/utils/network.py +++ b/backend/src/core/utils/network.py @@ -140,7 +140,16 @@ class APIClient: app_logger.info("[authenticate][Enter] Authenticating to %s", self.base_url) try: login_url = f"{self.base_url}/security/login" + # Log the payload keys and values (masking password) + masked_auth = {k: ("******" if k == "password" else v) for k, v in self.auth.items()} + app_logger.info(f"[authenticate][Debug] Login URL: {login_url}") + app_logger.info(f"[authenticate][Debug] Auth payload: {masked_auth}") + response = self.session.post(login_url, json=self.auth, timeout=self.request_settings["timeout"]) + + if response.status_code != 200: + app_logger.error(f"[authenticate][Error] Status: {response.status_code}, Response: {response.text}") + response.raise_for_status() access_token = response.json()["access_token"] @@ -153,6 +162,9 @@ class APIClient: app_logger.info("[authenticate][Exit] Authenticated successfully.") return self._tokens except requests.exceptions.HTTPError as e: + status_code = e.response.status_code if e.response is not None else None + if status_code in [502, 503, 504]: + raise NetworkError(f"Environment unavailable during authentication (Status {status_code})", status_code=status_code) from e raise AuthenticationError(f"Authentication failed: {e}") from e except (requests.exceptions.RequestException, KeyError) as e: raise NetworkError(f"Network or parsing error during authentication: {e}") from e @@ -209,6 +221,8 @@ class APIClient: def _handle_http_error(self, e: requests.exceptions.HTTPError, endpoint: str): with belief_scope("_handle_http_error"): status_code = e.response.status_code + if status_code == 502 or status_code == 503 or status_code == 504: + raise NetworkError(f"Environment unavailable (Status {status_code})", status_code=status_code) from e if status_code == 404: raise DashboardNotFoundError(endpoint) from e if status_code == 403: raise PermissionDeniedError() from e if status_code == 401: raise AuthenticationError() from e diff --git a/backend/src/dependencies.py b/backend/src/dependencies.py index 92c59665..f1aea063 100755 --- a/backend/src/dependencies.py +++ b/backend/src/dependencies.py @@ -35,8 +35,7 @@ init_db() # @RETURN: ConfigManager - The shared config manager instance. def get_config_manager() -> ConfigManager: """Dependency injector for the ConfigManager.""" - with belief_scope("get_config_manager"): - return config_manager + return config_manager # [/DEF:get_config_manager:Function] plugin_dir = Path(__file__).parent / "plugins" @@ -58,8 +57,7 @@ logger.info("SchedulerService initialized") # @RETURN: PluginLoader - The shared plugin loader instance. def get_plugin_loader() -> PluginLoader: """Dependency injector for the PluginLoader.""" - with belief_scope("get_plugin_loader"): - return plugin_loader + return plugin_loader # [/DEF:get_plugin_loader:Function] # [DEF:get_task_manager:Function] @@ -69,8 +67,7 @@ def get_plugin_loader() -> PluginLoader: # @RETURN: TaskManager - The shared task manager instance. def get_task_manager() -> TaskManager: """Dependency injector for the TaskManager.""" - with belief_scope("get_task_manager"): - return task_manager + return task_manager # [/DEF:get_task_manager:Function] # [DEF:get_scheduler_service:Function] @@ -80,8 +77,7 @@ def get_task_manager() -> TaskManager: # @RETURN: SchedulerService - The shared scheduler service instance. def get_scheduler_service() -> SchedulerService: """Dependency injector for the SchedulerService.""" - with belief_scope("get_scheduler_service"): - return scheduler_service + return scheduler_service # [/DEF:get_scheduler_service:Function] # [DEF:oauth2_scheme:Variable] @@ -98,25 +94,24 @@ oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login") # @PARAM: db (Session) - Auth database session. # @RETURN: User - The authenticated user. def get_current_user(token: str = Depends(oauth2_scheme), db = Depends(get_auth_db)): - with belief_scope("get_current_user"): - credentials_exception = HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Could not validate credentials", - headers={"WWW-Authenticate": "Bearer"}, - ) - try: - payload = decode_token(token) - username: str = payload.get("sub") - if username is None: - raise credentials_exception - except JWTError: + credentials_exception = HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Could not validate credentials", + headers={"WWW-Authenticate": "Bearer"}, + ) + try: + payload = decode_token(token) + username: str = payload.get("sub") + if username is None: raise credentials_exception - - repo = AuthRepository(db) - user = repo.get_user_by_username(username) - if user is None: - raise credentials_exception - return user + except JWTError: + raise credentials_exception + + repo = AuthRepository(db) + user = repo.get_user_by_username(username) + if user is None: + raise credentials_exception + return user # [/DEF:get_current_user:Function] # [DEF:has_permission:Function] @@ -129,24 +124,23 @@ def get_current_user(token: str = Depends(oauth2_scheme), db = Depends(get_auth_ # @RETURN: User - The authenticated user if permission granted. def has_permission(resource: str, action: str): def permission_checker(current_user: User = Depends(get_current_user)): - with belief_scope("has_permission", f"{resource}:{action}"): - # Union of all permissions across all roles - for role in current_user.roles: - for perm in role.permissions: - if perm.resource == resource and perm.action == action: - return current_user + # Union of all permissions across all roles + for role in current_user.roles: + for perm in role.permissions: + if perm.resource == resource and perm.action == action: + return current_user + + # Special case for Admin role (full access) + if any(role.name == "Admin" for role in current_user.roles): + return current_user + + from .core.auth.logger import log_security_event + log_security_event("PERMISSION_DENIED", current_user.username, {"resource": resource, "action": action}) - # Special case for Admin role (full access) - if any(role.name == "Admin" for role in current_user.roles): - return current_user - - from .core.auth.logger import log_security_event - log_security_event("PERMISSION_DENIED", current_user.username, {"resource": resource, "action": action}) - - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=f"Permission denied for {resource}:{action}" - ) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Permission denied for {resource}:{action}" + ) return permission_checker # [/DEF:has_permission:Function] diff --git a/backend/src/plugins/llm_analysis/__init__.py b/backend/src/plugins/llm_analysis/__init__.py index c97b5658..3a514fcd 100644 --- a/backend/src/plugins/llm_analysis/__init__.py +++ b/backend/src/plugins/llm_analysis/__init__.py @@ -1,6 +1,7 @@ # [DEF:backend/src/plugins/llm_analysis/__init__.py:Module] # @TIER: TRIVIAL # @PURPOSE: Initialize the LLM Analysis plugin package. +# @LAYER: Domain """ LLM Analysis Plugin for automated dashboard validation and dataset documentation. @@ -8,4 +9,4 @@ LLM Analysis Plugin for automated dashboard validation and dataset documentation from .plugin import DashboardValidationPlugin, DocumentationPlugin -# [/DEF:backend/src/plugins/llm_analysis/__init__.py] \ No newline at end of file +# [/DEF:backend/src/plugins/llm_analysis/__init__.py:Module] diff --git a/backend/src/plugins/llm_analysis/models.py b/backend/src/plugins/llm_analysis/models.py index 81a7db4c..b33024c4 100644 --- a/backend/src/plugins/llm_analysis/models.py +++ b/backend/src/plugins/llm_analysis/models.py @@ -24,7 +24,7 @@ class LLMProviderConfig(BaseModel): provider_type: LLMProviderType name: str base_url: str - api_key: str + api_key: Optional[str] = None default_model: str is_active: bool = True # [/DEF:LLMProviderConfig:Class] @@ -58,4 +58,4 @@ class ValidationResult(BaseModel): raw_response: Optional[str] = None # [/DEF:ValidationResult:Class] -# [/DEF:backend/src/plugins/llm_analysis/models.py] \ No newline at end of file +# [/DEF:backend/src/plugins/llm_analysis/models.py:Module] diff --git a/backend/src/plugins/llm_analysis/plugin.py b/backend/src/plugins/llm_analysis/plugin.py index 8bb152ae..fab7bd5e 100644 --- a/backend/src/plugins/llm_analysis/plugin.py +++ b/backend/src/plugins/llm_analysis/plugin.py @@ -1,9 +1,9 @@ -# [DEF:backend.src.plugins.llm_analysis.plugin:Module] +# [DEF:backend/src/plugins/llm_analysis/plugin.py:Module] # @TIER: STANDARD # @SEMANTICS: plugin, llm, analysis, documentation # @PURPOSE: Implements DashboardValidationPlugin and DocumentationPlugin. # @LAYER: Domain -# @RELATION: INHERITS_FROM -> backend.src.core.plugin_base.PluginBase +# @RELATION: INHERITS -> backend.src.core.plugin_base.PluginBase # @RELATION: CALLS -> backend.src.plugins.llm_analysis.service.ScreenshotService # @RELATION: CALLS -> backend.src.plugins.llm_analysis.service.LLMClient # @RELATION: CALLS -> backend.src.services.llm_provider.LLMProviderService @@ -12,6 +12,7 @@ from typing import Dict, Any, Optional, List import os import json +import logging from datetime import datetime, timedelta from ...core.plugin_base import PluginBase from ...core.logger import belief_scope, logger @@ -54,6 +55,11 @@ class DashboardValidationPlugin(PluginBase): "required": ["dashboard_id", "environment_id", "provider_id"] } + # [DEF:DashboardValidationPlugin.execute:Function] + # @PURPOSE: Executes the dashboard validation task. + # @PRE: params contains dashboard_id, environment_id, and provider_id. + # @POST: Returns a dictionary with validation results and persists them to the database. + # @SIDE_EFFECT: Captures a screenshot, calls LLM API, and writes to the database. async def execute(self, params: Dict[str, Any]): with belief_scope("execute", f"plugin_id={self.id}"): logger.info(f"Executing {self.name} with params: {params}") @@ -88,12 +94,35 @@ class DashboardValidationPlugin(PluginBase): if not db_provider: raise ValueError(f"LLM Provider {provider_id} not found") + logger.info(f"[DashboardValidationPlugin.execute] Retrieved provider config:") + logger.info(f"[DashboardValidationPlugin.execute] Provider ID: {db_provider.id}") + logger.info(f"[DashboardValidationPlugin.execute] Provider Name: {db_provider.name}") + logger.info(f"[DashboardValidationPlugin.execute] Provider Type: {db_provider.provider_type}") + logger.info(f"[DashboardValidationPlugin.execute] Base URL: {db_provider.base_url}") + logger.info(f"[DashboardValidationPlugin.execute] Default Model: {db_provider.default_model}") + logger.info(f"[DashboardValidationPlugin.execute] Is Active: {db_provider.is_active}") + api_key = llm_service.get_decrypted_api_key(provider_id) + logger.info(f"[DashboardValidationPlugin.execute] API Key decrypted (first 8 chars): {api_key[:8] if api_key and len(api_key) > 8 else 'EMPTY_OR_NONE'}...") + logger.info(f"[DashboardValidationPlugin.execute] API Key Length: {len(api_key) if api_key else 0}") + + # Check if API key was successfully decrypted + if not api_key: + raise ValueError( + f"Failed to decrypt API key for provider {provider_id}. " + f"The provider may have been encrypted with a different encryption key. " + f"Please update the provider with a new API key through the UI." + ) # 3. Capture Screenshot screenshot_service = ScreenshotService(env) - os.makedirs("ss-tools-storage/screenshots", exist_ok=True) - screenshot_path = f"ss-tools-storage/screenshots/{dashboard_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png" + + storage_root = config_mgr.get_config().settings.storage.root_path + screenshots_dir = os.path.join(storage_root, "screenshots") + os.makedirs(screenshots_dir, exist_ok=True) + + filename = f"{dashboard_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png" + screenshot_path = os.path.join(screenshots_dir, filename) await screenshot_service.capture_dashboard(dashboard_id, screenshot_path) @@ -109,8 +138,8 @@ class DashboardValidationPlugin(PluginBase): # Note: We filter by dashboard_id matching the object query_params = { "filters": [ - {"col": "dashboard_id", "op": "eq", "value": dashboard_id}, - {"col": "dttm", "op": "gt", "value": start_time} + {"col": "dashboard_id", "opr": "eq", "value": dashboard_id}, + {"col": "dttm", "opr": "gt", "value": start_time} ], "order_column": "dttm", "order_direction": "desc", @@ -149,11 +178,11 @@ class DashboardValidationPlugin(PluginBase): analysis = await llm_client.analyze_dashboard(screenshot_path, logs) # Log analysis summary to task logs for better visibility - logger.info(f"[ANALYSIS_SUMMARY] Status: {analysis['status']}") - logger.info(f"[ANALYSIS_SUMMARY] Summary: {analysis['summary']}") + task_log("INFO", f"[ANALYSIS_SUMMARY] Status: {analysis['status']}") + task_log("INFO", f"[ANALYSIS_SUMMARY] Summary: {analysis['summary']}") if analysis.get("issues"): for i, issue in enumerate(analysis["issues"]): - logger.info(f"[ANALYSIS_ISSUE][{i+1}] {issue.get('severity')}: {issue.get('message')} (Location: {issue.get('location', 'N/A')})") + task_log("INFO", f"[ANALYSIS_ISSUE][{i+1}] {issue.get('severity')}: {issue.get('message')} (Location: {issue.get('location', 'N/A')})") # 6. Persist Result validation_result = ValidationResult( @@ -178,7 +207,7 @@ class DashboardValidationPlugin(PluginBase): # 7. Notification on failure (US1 / FR-015) if validation_result.status == ValidationStatus.FAIL: - logger.warning(f"Dashboard {dashboard_id} validation FAILED. Summary: {validation_result.summary}") + task_log("WARNING", f"Dashboard {dashboard_id} validation FAILED. Summary: {validation_result.summary}") # Placeholder for Email/Pulse notification dispatch # In a real implementation, we would call a NotificationService here # with a payload containing the summary and a link to the report. @@ -190,6 +219,7 @@ class DashboardValidationPlugin(PluginBase): finally: db.close() + # [/DEF:DashboardValidationPlugin.execute:Function] # [/DEF:DashboardValidationPlugin:Class] # [DEF:DocumentationPlugin:Class] @@ -223,12 +253,7 @@ class DocumentationPlugin(PluginBase): "required": ["dataset_id", "environment_id", "provider_id"] } - # [DEF:execute:Function] - # @PURPOSE: Executes the dashboard validation task. - # @PRE: params contains dashboard_id, environment_id, and provider_id. - # @POST: Returns a dictionary with validation results and persists them to the database. - # @SIDE_EFFECT: Captures a screenshot, calls LLM API, and writes to the database. - # [DEF:execute:Function] + # [DEF:DocumentationPlugin.execute:Function] # @PURPOSE: Executes the dataset documentation task. # @PRE: params contains dataset_id, environment_id, and provider_id. # @POST: Returns generated documentation and updates the dataset in Superset. @@ -256,7 +281,25 @@ class DocumentationPlugin(PluginBase): if not db_provider: raise ValueError(f"LLM Provider {provider_id} not found") + logger.info(f"[DocumentationPlugin.execute] Retrieved provider config:") + logger.info(f"[DocumentationPlugin.execute] Provider ID: {db_provider.id}") + logger.info(f"[DocumentationPlugin.execute] Provider Name: {db_provider.name}") + logger.info(f"[DocumentationPlugin.execute] Provider Type: {db_provider.provider_type}") + logger.info(f"[DocumentationPlugin.execute] Base URL: {db_provider.base_url}") + logger.info(f"[DocumentationPlugin.execute] Default Model: {db_provider.default_model}") + logger.info(f"[DocumentationPlugin.execute] Is Active: {db_provider.is_active}") + api_key = llm_service.get_decrypted_api_key(provider_id) + logger.info(f"[DocumentationPlugin.execute] API Key decrypted (first 8 chars): {api_key[:8] if api_key and len(api_key) > 8 else 'EMPTY_OR_NONE'}...") + logger.info(f"[DocumentationPlugin.execute] API Key Length: {len(api_key) if api_key else 0}") + + # Check if API key was successfully decrypted + if not api_key: + raise ValueError( + f"Failed to decrypt API key for provider {provider_id}. " + f"The provider may have been encrypted with a different encryption key. " + f"Please update the provider with a new API key through the UI." + ) # 3. Fetch Metadata (US2 / T024) from ...core.superset_client import SupersetClient @@ -328,6 +371,7 @@ class DocumentationPlugin(PluginBase): finally: db.close() + # [/DEF:DocumentationPlugin.execute:Function] # [/DEF:DocumentationPlugin:Class] -# [/DEF:backend.src.plugins.llm_analysis.plugin:Module] \ No newline at end of file +# [/DEF:backend/src/plugins/llm_analysis/plugin.py:Module] diff --git a/backend/src/plugins/llm_analysis/scheduler.py b/backend/src/plugins/llm_analysis/scheduler.py index 4568f06a..795734a0 100644 --- a/backend/src/plugins/llm_analysis/scheduler.py +++ b/backend/src/plugins/llm_analysis/scheduler.py @@ -14,6 +14,7 @@ from ...core.logger import belief_scope, logger # @PARAM: dashboard_id (str) - ID of the dashboard to validate. # @PARAM: cron_expression (str) - Standard cron expression for scheduling. # @PARAM: params (Dict[str, Any]) - Task parameters (environment_id, provider_id). +# @SIDE_EFFECT: Adds a job to the scheduler service. def schedule_dashboard_validation(dashboard_id: str, cron_expression: str, params: Dict[str, Any]): with belief_scope("schedule_dashboard_validation", f"dashboard_id={dashboard_id}"): scheduler = get_scheduler_service() @@ -39,6 +40,10 @@ def schedule_dashboard_validation(dashboard_id: str, cron_expression: str, param ) logger.info(f"Scheduled validation for dashboard {dashboard_id} with cron {cron_expression}") +# [DEF:_parse_cron:Function] +# @PURPOSE: Basic cron parser placeholder. +# @PARAM: cron (str) - Cron expression. +# @RETURN: Dict[str, str] - Parsed cron parts. def _parse_cron(cron: str) -> Dict[str, str]: # Basic cron parser placeholder parts = cron.split() @@ -51,6 +56,5 @@ def _parse_cron(cron: str) -> Dict[str, str]: "month": parts[3], "day_of_week": parts[4] } -# [/DEF:schedule_dashboard_validation:Function] -# [/DEF:backend/src/plugins/llm_analysis/scheduler.py] \ No newline at end of file +# [/DEF:backend/src/plugins/llm_analysis/scheduler.py:Module] diff --git a/backend/src/plugins/llm_analysis/service.py b/backend/src/plugins/llm_analysis/service.py index c792aecb..e1f605b2 100644 --- a/backend/src/plugins/llm_analysis/service.py +++ b/backend/src/plugins/llm_analysis/service.py @@ -1,4 +1,4 @@ -# [DEF:backend.src.plugins.llm_analysis.service:Module] +# [DEF:backend/src/plugins/llm_analysis/service.py:Module] # @TIER: STANDARD # @SEMANTICS: service, llm, screenshot, playwright, openai # @PURPOSE: Services for LLM interaction and dashboard screenshots. @@ -6,12 +6,17 @@ # @RELATION: DEPENDS_ON -> playwright # @RELATION: DEPENDS_ON -> openai # @RELATION: DEPENDS_ON -> tenacity +# @INVARIANT: Screenshots must be 1920px width and capture full page height. import asyncio +import base64 +import json +import io from typing import List, Optional, Dict, Any +from PIL import Image from playwright.async_api import async_playwright -from openai import AsyncOpenAI, RateLimitError -from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type +from openai import AsyncOpenAI, RateLimitError, AuthenticationError as OpenAIAuthenticationError +from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception from .models import LLMProviderType, ValidationResult, ValidationStatus, DetectedIssue from ...core.logger import belief_scope, logger from ...core.config_models import Environment @@ -19,142 +24,559 @@ from ...core.config_models import Environment # [DEF:ScreenshotService:Class] # @PURPOSE: Handles capturing screenshots of Superset dashboards. class ScreenshotService: + # [DEF:ScreenshotService.__init__:Function] + # @PURPOSE: Initializes the ScreenshotService with environment configuration. # @PRE: env is a valid Environment object. def __init__(self, env: Environment): self.env = env + # [/DEF:ScreenshotService.__init__:Function] - # [DEF:capture_dashboard:Function] - # @PURPOSE: Captures a screenshot of a dashboard using Playwright. - # @PARAM: dashboard_id (str) - ID of the dashboard. - # @PARAM: output_path (str) - Path to save the screenshot. - # @RETURN: bool - True if successful. + # [DEF:ScreenshotService.capture_dashboard:Function] + # @PURPOSE: Captures a full-page screenshot of a dashboard using Playwright and CDP. + # @PRE: dashboard_id is a valid string, output_path is a writable path. + # @POST: Returns True if screenshot is saved successfully. + # @SIDE_EFFECT: Launches a browser, performs UI login, switches tabs, and writes a PNG file. + # @UX_STATE: [Navigating] -> Loading dashboard UI + # @UX_STATE: [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading + # @UX_STATE: [CalculatingHeight] -> Determining dashboard dimensions + # @UX_STATE: [Capturing] -> Executing CDP screenshot async def capture_dashboard(self, dashboard_id: str, output_path: str) -> bool: with belief_scope("capture_dashboard", f"dashboard_id={dashboard_id}"): logger.info(f"Capturing screenshot for dashboard {dashboard_id}") async with async_playwright() as p: - browser = await p.chromium.launch(headless=True) - context = await browser.new_context(viewport={'width': 1280, 'height': 720}) - page = await context.new_page() + browser = await p.chromium.launch( + headless=True, + args=[ + "--disable-blink-features=AutomationControlled", + "--disable-infobars", + "--no-sandbox" + ] + ) + # Set a realistic user agent to avoid 403 Forbidden from OpenResty/WAF + user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + # Construct base UI URL from environment (strip /api/v1 suffix) + base_ui_url = self.env.url.rstrip("/") + if base_ui_url.endswith("/api/v1"): + base_ui_url = base_ui_url[:-len("/api/v1")] - # 1. Authenticate via API to get tokens - from ...core.superset_client import SupersetClient - client = SupersetClient(self.env) - try: - tokens = client.authenticate() - access_token = tokens.get("access_token") - - # Set JWT in localStorage if possible, or use as cookie - # Superset UI uses session cookies, but we can try to set the Authorization header - # or inject the token into the session. - # For now, we'll use the token to set a cookie if we can determine the name, - # but the most reliable way for Playwright is often still the UI login - # UNLESS we use the API to set a session cookie. - logger.info("API Authentication successful") - except Exception as e: - logger.warning(f"API Authentication failed: {e}. Falling back to UI login.") - - # 2. Navigate to dashboard - dashboard_url = f"{self.env.url}/superset/dashboard/{dashboard_id}/" - logger.info(f"Navigating to {dashboard_url}") - - # We still go to the URL first - await page.goto(dashboard_url) - await page.wait_for_load_state("networkidle") - - # 3. Check if we are redirected to login - if "/login" in page.url: - logger.info(f"Redirected to login: {page.url}. Filling credentials from Environment.") - - # More exhaustive list of selectors for various Superset versions/themes - selectors = { - "username": ['input[name="username"]', 'input#username', 'input[placeholder*="Username"]'], - "password": ['input[name="password"]', 'input#password', 'input[placeholder*="Password"]'], - "submit": ['button[type="submit"]', 'button#submit', '.btn-primary'] + # Create browser context with realistic headers + context = await browser.new_context( + viewport={'width': 1280, 'height': 720}, + user_agent=user_agent, + extra_http_headers={ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1" } - - try: - # Find and fill username - u_selector = None - for s in selectors["username"]: - if await page.locator(s).count() > 0: - u_selector = s - break - - if not u_selector: - raise RuntimeError("Could not find username input field") - - await page.fill(u_selector, self.env.username) - - # Find and fill password - p_selector = None - for s in selectors["password"]: - if await page.locator(s).count() > 0: - p_selector = s - break - - if not p_selector: - raise RuntimeError("Could not find password input field") - - await page.fill(p_selector, self.env.password) - - # Click submit - s_selector = selectors["submit"][0] - for s in selectors["submit"]: - if await page.locator(s).count() > 0: - s_selector = s - break - - await page.click(s_selector) - await page.wait_for_load_state("networkidle") - - # Re-verify we are at the dashboard - if "/login" in page.url: - # Check for error messages on page - error_msg = await page.locator(".alert-danger, .error-message").text_content() if await page.locator(".alert-danger, .error-message").count() > 0 else "Unknown error" - raise RuntimeError(f"Login failed after submission: {error_msg}") + ) + logger.info("Browser context created successfully") - if "/superset/dashboard" not in page.url: - logger.info(f"Redirecting back to dashboard after login: {dashboard_url}") - await page.goto(dashboard_url) - await page.wait_for_load_state("networkidle") - - except Exception as e: - page_title = await page.title() - logger.error(f"UI Login failed. Page title: {page_title}, URL: {page.url}, Error: {str(e)}") + page = await context.new_page() + # Bypass navigator.webdriver detection + await page.add_init_script("delete Object.getPrototypeOf(navigator).webdriver") + + # 1. Navigate to login page and authenticate + login_url = f"{base_ui_url.rstrip('/')}/login/" + logger.info(f"[DEBUG] Navigating to login page: {login_url}") + + response = await page.goto(login_url, wait_until="networkidle", timeout=60000) + if response: + logger.info(f"[DEBUG] Login page response status: {response.status}") + + # Wait for login form to be ready + await page.wait_for_load_state("domcontentloaded") + + # More exhaustive list of selectors for various Superset versions/themes + selectors = { + "username": ['input[name="username"]', 'input#username', 'input[placeholder*="Username"]', 'input[type="text"]'], + "password": ['input[name="password"]', 'input#password', 'input[placeholder*="Password"]', 'input[type="password"]'], + "submit": ['button[type="submit"]', 'button#submit', '.btn-primary', 'input[type="submit"]'] + } + logger.info(f"[DEBUG] Attempting to find login form elements...") + + try: + # Find and fill username + u_selector = None + for s in selectors["username"]: + count = await page.locator(s).count() + logger.info(f"[DEBUG] Selector '{s}': {count} elements found") + if count > 0: + u_selector = s + break + + if not u_selector: + # Log all input fields on the page for debugging + all_inputs = await page.locator('input').all() + logger.info(f"[DEBUG] Found {len(all_inputs)} input fields on page") + for i, inp in enumerate(all_inputs[:5]): # Log first 5 + inp_type = await inp.get_attribute('type') + inp_name = await inp.get_attribute('name') + inp_id = await inp.get_attribute('id') + logger.info(f"[DEBUG] Input {i}: type={inp_type}, name={inp_name}, id={inp_id}") + raise RuntimeError("Could not find username input field on login page") + + logger.info(f"[DEBUG] Filling username field with selector: {u_selector}") + await page.fill(u_selector, self.env.username) + + # Find and fill password + p_selector = None + for s in selectors["password"]: + if await page.locator(s).count() > 0: + p_selector = s + break + + if not p_selector: + raise RuntimeError("Could not find password input field on login page") + + logger.info(f"[DEBUG] Filling password field with selector: {p_selector}") + await page.fill(p_selector, self.env.password) + + # Click submit + s_selector = selectors["submit"][0] + for s in selectors["submit"]: + if await page.locator(s).count() > 0: + s_selector = s + break + + logger.info(f"[DEBUG] Clicking submit button with selector: {s_selector}") + await page.click(s_selector) + + # Wait for navigation after login + await page.wait_for_load_state("networkidle", timeout=30000) + + # Check if login was successful + if "/login" in page.url: + # Check for error messages on page + error_msg = await page.locator(".alert-danger, .error-message").text_content() if await page.locator(".alert-danger, .error-message").count() > 0 else "Unknown error" + logger.error(f"[DEBUG] Login failed. Still on login page. Error: {error_msg}") debug_path = output_path.replace(".png", "_debug_failed_login.png") await page.screenshot(path=debug_path) - raise RuntimeError(f"Login failed: {str(e)}. Debug screenshot saved to {debug_path}") - # Wait a bit more for charts to render - await asyncio.sleep(5) + raise RuntimeError(f"Login failed: {error_msg}. Debug screenshot saved to {debug_path}") + + logger.info(f"[DEBUG] Login successful. Current URL: {page.url}") + + # Check cookies after successful login + page_cookies = await context.cookies() + logger.info(f"[DEBUG] Cookies after login: {len(page_cookies)}") + for c in page_cookies: + logger.info(f"[DEBUG] Cookie: name={c['name']}, domain={c['domain']}, value={c.get('value', '')[:20]}...") + + except Exception as e: + page_title = await page.title() + logger.error(f"UI Login failed. Page title: {page_title}, URL: {page.url}, Error: {str(e)}") + debug_path = output_path.replace(".png", "_debug_failed_login.png") + await page.screenshot(path=debug_path) + raise RuntimeError(f"Login failed: {str(e)}. Debug screenshot saved to {debug_path}") + + # 2. Navigate to dashboard + # @UX_STATE: [Navigating] -> Loading dashboard UI + dashboard_url = f"{base_ui_url.rstrip('/')}/superset/dashboard/{dashboard_id}/?standalone=true" + + if base_ui_url.startswith("https://") and dashboard_url.startswith("http://"): + dashboard_url = dashboard_url.replace("http://", "https://") + + logger.info(f"[DEBUG] Navigating to dashboard: {dashboard_url}") + + # Use networkidle to ensure all initial assets are loaded + response = await page.goto(dashboard_url, wait_until="networkidle", timeout=60000) + + if response: + logger.info(f"[DEBUG] Dashboard navigation response status: {response.status}, URL: {response.url}") + + try: + # Wait for the dashboard grid to be present + await page.wait_for_selector('.dashboard-component, .dashboard-header, [data-test="dashboard-grid"]', timeout=30000) + logger.info(f"[DEBUG] Dashboard container loaded") + + # Wait for charts to finish loading (Superset uses loading spinners/skeletons) + # We wait until loading indicators disappear or a timeout occurs + try: + # Wait for loading indicators to disappear + await page.wait_for_selector('.loading, .ant-skeleton, .spinner', state="hidden", timeout=60000) + logger.info(f"[DEBUG] Loading indicators hidden") + except: + logger.warning(f"[DEBUG] Timeout waiting for loading indicators to hide") + + # Wait for charts to actually render their content (e.g., ECharts, NVD3) + # We look for common chart containers that should have content + try: + await page.wait_for_selector('.chart-container canvas, .slice_container svg, .superset-chart-canvas, .grid-content .chart-container', timeout=60000) + logger.info(f"[DEBUG] Chart content detected") + except: + logger.warning(f"[DEBUG] Timeout waiting for chart content") + + # Additional check: wait for all chart containers to have non-empty content + logger.info(f"[DEBUG] Waiting for all charts to have rendered content...") + await page.wait_for_function("""() => { + const charts = document.querySelectorAll('.chart-container, .slice_container'); + if (charts.length === 0) return true; // No charts to wait for + + // Check if all charts have rendered content (canvas, svg, or non-empty div) + return Array.from(charts).every(chart => { + const hasCanvas = chart.querySelector('canvas') !== null; + const hasSvg = chart.querySelector('svg') !== null; + const hasContent = chart.innerText.trim().length > 0 || chart.children.length > 0; + return hasCanvas || hasSvg || hasContent; + }); + }""", timeout=60000) + logger.info(f"[DEBUG] All charts have rendered content") + + # Scroll to bottom and back to top to trigger lazy loading of all charts + logger.info(f"[DEBUG] Scrolling to trigger lazy loading...") + await page.evaluate("""async () => { + const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); + for (let i = 0; i < document.body.scrollHeight; i += 500) { + window.scrollTo(0, i); + await delay(100); + } + window.scrollTo(0, 0); + await delay(500); + }""") + + except Exception as e: + logger.warning(f"[DEBUG] Dashboard content wait failed: {e}, proceeding anyway after delay") + + # Final stabilization delay - increased for complex dashboards + logger.info(f"[DEBUG] Final stabilization delay...") + await asyncio.sleep(15) + + # Logic to handle tabs and full-page capture + try: + # 1. Handle Tabs (Recursive switching) + # @UX_STATE: [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading + processed_tabs = set() + + async def switch_tabs(depth=0): + if depth > 3: return # Limit recursion depth + + tab_selectors = [ + '.ant-tabs-nav-list .ant-tabs-tab', + '.dashboard-component-tabs .ant-tabs-tab', + '[data-test="dashboard-component-tabs"] .ant-tabs-tab' + ] + + found_tabs = [] + for selector in tab_selectors: + found_tabs = await page.locator(selector).all() + if found_tabs: break + + if found_tabs: + logger.info(f"[DEBUG][TabSwitching] Found {len(found_tabs)} tabs at depth {depth}") + for i, tab in enumerate(found_tabs): + try: + tab_text = (await tab.inner_text()).strip() + tab_id = f"{depth}_{i}_{tab_text}" + + if tab_id in processed_tabs: + continue + + if await tab.is_visible(): + logger.info(f"[DEBUG][TabSwitching] Switching to tab: {tab_text}") + processed_tabs.add(tab_id) + + is_active = "ant-tabs-tab-active" in (await tab.get_attribute("class") or "") + if not is_active: + await tab.click() + await asyncio.sleep(2) # Wait for content to render + + await switch_tabs(depth + 1) + except Exception as tab_e: + logger.warning(f"[DEBUG][TabSwitching] Failed to process tab {i}: {tab_e}") + + try: + first_tab = found_tabs[0] + if "ant-tabs-tab-active" not in (await first_tab.get_attribute("class") or ""): + await first_tab.click() + await asyncio.sleep(1) + except: pass + + await switch_tabs() + + # 2. Calculate full height for screenshot + # @UX_STATE: [CalculatingHeight] -> Determining dashboard dimensions + full_height = await page.evaluate("""() => { + const body = document.body; + const html = document.documentElement; + const dashboardContent = document.querySelector('.dashboard-content'); + + return Math.max( + body.scrollHeight, body.offsetHeight, + html.clientHeight, html.scrollHeight, html.offsetHeight, + dashboardContent ? dashboardContent.scrollHeight + 100 : 0 + ); + }""") + logger.info(f"[DEBUG] Calculated full height: {full_height}") + + # DIAGNOSTIC: Count chart elements before resize + chart_count_before = await page.evaluate("""() => { + return { + chartContainers: document.querySelectorAll('.chart-container, .slice_container').length, + canvasElements: document.querySelectorAll('canvas').length, + svgElements: document.querySelectorAll('.chart-container svg, .slice_container svg').length, + visibleCharts: document.querySelectorAll('.chart-container:visible, .slice_container:visible').length + }; + }""") + logger.info(f"[DIAGNOSTIC] Chart elements BEFORE viewport resize: {chart_count_before}") + + # DIAGNOSTIC: Capture pre-resize screenshot for comparison + pre_resize_path = output_path.replace(".png", "_preresize.png") + try: + await page.screenshot(path=pre_resize_path, full_page=False, timeout=10000) + import os + pre_resize_size = os.path.getsize(pre_resize_path) if os.path.exists(pre_resize_path) else 0 + logger.info(f"[DIAGNOSTIC] Pre-resize screenshot saved: {pre_resize_path} ({pre_resize_size} bytes)") + except Exception as pre_e: + logger.warning(f"[DIAGNOSTIC] Failed to capture pre-resize screenshot: {pre_e}") + + logger.info(f"[DIAGNOSTIC] Resizing viewport from current to 1920x{int(full_height)}") + await page.set_viewport_size({"width": 1920, "height": int(full_height)}) + + # DIAGNOSTIC: Increased wait time and log timing + logger.info("[DIAGNOSTIC] Waiting 10 seconds after viewport resize for re-render...") + await asyncio.sleep(10) + logger.info("[DIAGNOSTIC] Wait completed") + + # DIAGNOSTIC: Count chart elements after resize and wait + chart_count_after = await page.evaluate("""() => { + return { + chartContainers: document.querySelectorAll('.chart-container, .slice_container').length, + canvasElements: document.querySelectorAll('canvas').length, + svgElements: document.querySelectorAll('.chart-container svg, .slice_container svg').length, + visibleCharts: document.querySelectorAll('.chart-container:visible, .slice_container:visible').length + }; + }""") + logger.info(f"[DIAGNOSTIC] Chart elements AFTER viewport resize + wait: {chart_count_after}") + + # DIAGNOSTIC: Check if any charts have error states + chart_errors = await page.evaluate("""() => { + const errors = []; + document.querySelectorAll('.chart-container, .slice_container').forEach((chart, i) => { + const errorEl = chart.querySelector('.error, .alert-danger, .ant-alert-error'); + if (errorEl) { + errors.push({index: i, text: errorEl.innerText.substring(0, 100)}); + } + }); + return errors; + }""") + if chart_errors: + logger.warning(f"[DIAGNOSTIC] Charts with error states detected: {chart_errors}") + else: + logger.info("[DIAGNOSTIC] No chart error states detected") + + # 3. Take screenshot using CDP to bypass Playwright's font loading wait + # @UX_STATE: [Capturing] -> Executing CDP screenshot + logger.info("[DEBUG] Attempting full-page screenshot via CDP...") + cdp = await page.context.new_cdp_session(page) + + screenshot_data = await cdp.send("Page.captureScreenshot", { + "format": "png", + "fromSurface": True, + "captureBeyondViewport": True + }) + + image_data = base64.b64decode(screenshot_data["data"]) + + with open(output_path, 'wb') as f: + f.write(image_data) + + # DIAGNOSTIC: Verify screenshot file + import os + final_size = os.path.getsize(output_path) if os.path.exists(output_path) else 0 + logger.info(f"[DIAGNOSTIC] Final screenshot saved: {output_path}") + logger.info(f"[DIAGNOSTIC] Final screenshot size: {final_size} bytes ({final_size / 1024:.2f} KB)") + + # DIAGNOSTIC: Get image dimensions + try: + with Image.open(output_path) as final_img: + logger.info(f"[DIAGNOSTIC] Final screenshot dimensions: {final_img.width}x{final_img.height}") + except Exception as img_err: + logger.warning(f"[DIAGNOSTIC] Could not read final image dimensions: {img_err}") + + logger.info(f"Full-page screenshot saved to {output_path} (via CDP)") + except Exception as e: + logger.error(f"[DEBUG] Full-page/Tab capture failed: {e}") + try: + await page.screenshot(path=output_path, full_page=True, timeout=10000) + except Exception as e2: + logger.error(f"[DEBUG] Fallback screenshot also failed: {e2}") + await page.screenshot(path=output_path, timeout=5000) - await page.screenshot(path=output_path, full_page=True) await browser.close() - logger.info(f"Screenshot saved to {output_path}") return True + # [/DEF:ScreenshotService.capture_dashboard:Function] # [/DEF:ScreenshotService:Class] # [DEF:LLMClient:Class] # @PURPOSE: Wrapper for LLM provider APIs. class LLMClient: + # [DEF:LLMClient.__init__:Function] + # @PURPOSE: Initializes the LLMClient with provider settings. + # @PRE: api_key, base_url, and default_model are non-empty strings. def __init__(self, provider_type: LLMProviderType, api_key: str, base_url: str, default_model: str): self.provider_type = provider_type self.api_key = api_key self.base_url = base_url self.default_model = default_model + + # DEBUG: Log initialization parameters (without exposing full API key) + logger.info(f"[LLMClient.__init__] Initializing LLM client:") + logger.info(f"[LLMClient.__init__] Provider Type: {provider_type}") + logger.info(f"[LLMClient.__init__] Base URL: {base_url}") + logger.info(f"[LLMClient.__init__] Default Model: {default_model}") + logger.info(f"[LLMClient.__init__] API Key (first 8 chars): {api_key[:8] if api_key and len(api_key) > 8 else 'EMPTY_OR_NONE'}...") + logger.info(f"[LLMClient.__init__] API Key Length: {len(api_key) if api_key else 0}") + self.client = AsyncOpenAI(api_key=api_key, base_url=base_url) + # [/DEF:LLMClient.__init__:Function] - # [DEF:analyze_dashboard:Function] - # @PURPOSE: Sends dashboard data to LLM for analysis. + # [DEF:LLMClient.get_json_completion:Function] + # @PURPOSE: Helper to handle LLM calls with JSON mode and fallback parsing. + # @PRE: messages is a list of valid message dictionaries. + # @POST: Returns a parsed JSON dictionary. + # @SIDE_EFFECT: Calls external LLM API. + def _should_retry(exception: Exception) -> bool: + """Custom retry predicate that excludes authentication errors.""" + # Don't retry on authentication errors + if isinstance(exception, OpenAIAuthenticationError): + return False + # Retry on rate limit errors and other exceptions + return isinstance(exception, (RateLimitError, Exception)) + @retry( stop=stop_after_attempt(5), wait=wait_exponential(multiplier=2, min=5, max=60), - retry=retry_if_exception_type((Exception, RateLimitError)) + retry=retry_if_exception(_should_retry), + reraise=True ) + async def get_json_completion(self, messages: List[Dict[str, Any]]) -> Dict[str, Any]: + with belief_scope("get_json_completion"): + response = None + try: + try: + logger.info(f"[get_json_completion] Attempting LLM call with JSON mode for model: {self.default_model}") + logger.info(f"[get_json_completion] Base URL being used: {self.base_url}") + logger.info(f"[get_json_completion] Number of messages: {len(messages)}") + logger.info(f"[get_json_completion] API Key present: {bool(self.api_key and len(self.api_key) > 0)}") + + response = await self.client.chat.completions.create( + model=self.default_model, + messages=messages, + response_format={"type": "json_object"} + ) + except Exception as e: + if "JSON mode is not enabled" in str(e) or "400" in str(e): + logger.warning(f"[get_json_completion] JSON mode failed or not supported: {str(e)}. Falling back to plain text response.") + response = await self.client.chat.completions.create( + model=self.default_model, + messages=messages + ) + else: + raise e + + logger.debug(f"[get_json_completion] LLM Response: {response}") + except OpenAIAuthenticationError as e: + logger.error(f"[get_json_completion] Authentication error: {str(e)}") + # Do not retry on auth errors - re-raise to stop retry + raise + except RateLimitError as e: + logger.warning(f"[get_json_completion] Rate limit hit: {str(e)}") + + # Extract retry_delay from error metadata if available + retry_delay = 5.0 # Default fallback + try: + # Based on logs, the raw response is in e.body or e.response.json() + # The logs show 'metadata': {'raw': '...'} which suggests a proxy or specific client wrapper + # Let's try to find the 'retryDelay' in the error message or response + import re + + # Try to find "retryDelay": "XXs" in the string representation of the error + error_str = str(e) + match = re.search(r'"retryDelay":\s*"(\d+)s"', error_str) + if match: + retry_delay = float(match.group(1)) + else: + # Try to parse from response if it's a standard OpenAI-like error with body + if hasattr(e, 'body') and isinstance(e.body, dict): + # Some providers put it in details + details = e.body.get('error', {}).get('details', []) + for detail in details: + if detail.get('@type') == 'type.googleapis.com/google.rpc.RetryInfo': + delay_str = detail.get('retryDelay', '5s') + retry_delay = float(delay_str.rstrip('s')) + break + except Exception as parse_e: + logger.debug(f"[get_json_completion] Failed to parse retry delay: {parse_e}") + + # Add a small safety margin (0.5s) as requested + wait_time = retry_delay + 0.5 + logger.info(f"[get_json_completion] Waiting for {wait_time}s before retry...") + await asyncio.sleep(wait_time) + raise + except Exception as e: + logger.error(f"[get_json_completion] LLM call failed: {str(e)}") + raise + + if not response or not hasattr(response, 'choices') or not response.choices: + raise RuntimeError(f"Invalid LLM response: {response}") + + content = response.choices[0].message.content + logger.debug(f"[get_json_completion] Raw content to parse: {content}") + + try: + return json.loads(content) + except json.JSONDecodeError: + logger.warning("[get_json_completion] Failed to parse JSON directly, attempting to extract from code blocks") + if "```json" in content: + json_str = content.split("```json")[1].split("```")[0].strip() + return json.loads(json_str) + elif "```" in content: + json_str = content.split("```")[1].split("```")[0].strip() + return json.loads(json_str) + else: + raise + # [/DEF:LLMClient.get_json_completion:Function] + + # [DEF:LLMClient.analyze_dashboard:Function] + # @PURPOSE: Sends dashboard data (screenshot + logs) to LLM for health analysis. + # @PRE: screenshot_path exists, logs is a list of strings. + # @POST: Returns a structured analysis dictionary (status, summary, issues). + # @SIDE_EFFECT: Reads screenshot file and calls external LLM API. async def analyze_dashboard(self, screenshot_path: str, logs: List[str]) -> Dict[str, Any]: with belief_scope("analyze_dashboard"): - import base64 - with open(screenshot_path, "rb") as image_file: - base64_image = base64.b64encode(image_file.read()).decode('utf-8') + # Optimize image to reduce token count (US1 / T023) + # Gemini/Gemma models have limits on input tokens, and large images contribute significantly. + try: + with Image.open(screenshot_path) as img: + # Convert to RGB if necessary + if img.mode in ("RGBA", "P"): + img = img.convert("RGB") + + # Resize if too large (max 1024px width while maintaining aspect ratio) + # We reduce width further to 1024px to stay within token limits for long dashboards + max_width = 1024 + if img.width > max_width or img.height > 2048: + # Calculate scaling factor to fit within 1024x2048 + scale = min(max_width / img.width, 2048 / img.height) + if scale < 1.0: + new_width = int(img.width * scale) + new_height = int(img.height * scale) + img = img.resize((new_width, new_height), Image.Resampling.LANCZOS) + logger.info(f"[analyze_dashboard] Resized image from {img.width}x{img.height} to {new_width}x{new_height}") + + # Compress and convert to base64 + buffer = io.BytesIO() + # Lower quality to 60% to further reduce payload size + img.save(buffer, format="JPEG", quality=60, optimize=True) + base_64_image = base64.b64encode(buffer.getvalue()).decode('utf-8') + logger.info(f"[analyze_dashboard] Optimized image size: {len(buffer.getvalue()) / 1024:.2f} KB") + except Exception as img_e: + logger.warning(f"[analyze_dashboard] Image optimization failed: {img_e}. Using raw image.") + with open(screenshot_path, "rb") as image_file: + base_64_image = base64.b64encode(image_file.read()).decode('utf-8') log_text = "\n".join(logs) prompt = f""" @@ -177,48 +599,31 @@ class LLMClient: }} """ - logger.debug(f"[analyze_dashboard] Calling LLM with model: {self.default_model}") - try: - response = await self.client.chat.completions.create( - model=self.default_model, - messages=[ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, { - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{base64_image}" - } - } - ] + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{base_64_image}" + } } - ], - response_format={"type": "json_object"} - ) - logger.debug(f"[analyze_dashboard] LLM Response: {response}") - except RateLimitError as e: - logger.warning(f"[analyze_dashboard] Rate limit hit: {str(e)}") - raise # tenacity will handle retry - except Exception as e: - logger.error(f"[analyze_dashboard] LLM call failed: {str(e)}") - raise + ] + } + ] - if not response or not hasattr(response, 'choices') or not response.choices: - error_info = getattr(response, 'error', 'No choices in response') - logger.error(f"[analyze_dashboard] Invalid LLM response. Error info: {error_info}") + try: + return await self.get_json_completion(messages) + except Exception as e: + logger.error(f"[analyze_dashboard] Failed to get analysis: {str(e)}") return { "status": "FAIL", - "summary": f"Failed to get response from LLM: {error_info}", + "summary": f"Failed to get response from LLM: {str(e)}", "issues": [{"severity": "FAIL", "message": "LLM provider returned empty or invalid response"}] } - - import json - result = json.loads(response.choices[0].message.content) - return result - # [/DEF:analyze_dashboard:Function] - + # [/DEF:LLMClient.analyze_dashboard:Function] # [/DEF:LLMClient:Class] -# [/DEF:backend.src.plugins.llm_analysis.service:Module] \ No newline at end of file +# [/DEF:backend/src/plugins/llm_analysis/service.py:Module] diff --git a/backend/src/services/llm_provider.py b/backend/src/services/llm_provider.py index d05e0c5a..e747cf8b 100644 --- a/backend/src/services/llm_provider.py +++ b/backend/src/services/llm_provider.py @@ -19,7 +19,7 @@ import os class EncryptionManager: # @INVARIANT: Uses a secret key from environment or a default one (fallback only for dev). def __init__(self): - self.key = os.getenv("ENCRYPTION_KEY", "7_u-l7-B-j9f5_V5z-5-5-5-5-5-5-5-5-5-5-5-5-5=").encode() + self.key = os.getenv("ENCRYPTION_KEY", "REMOVED_HISTORICAL_SECRET_DO_NOT_USE").encode() self.fernet = Fernet(self.key) def encrypt(self, data: str) -> str: @@ -80,7 +80,8 @@ class LLMProviderService: db_provider.provider_type = config.provider_type.value db_provider.name = config.name db_provider.base_url = config.base_url - if config.api_key != "********": + # Only update API key if provided (not None and not empty) + if config.api_key is not None and config.api_key != "": db_provider.api_key = self.encryption.encrypt(config.api_key) db_provider.default_model = config.default_model db_provider.is_active = config.is_active @@ -108,8 +109,19 @@ class LLMProviderService: with belief_scope("get_decrypted_api_key"): db_provider = self.get_provider(provider_id) if not db_provider: + logger.warning(f"[get_decrypted_api_key] Provider {provider_id} not found in database") + return None + + logger.info(f"[get_decrypted_api_key] Provider found: {db_provider.id}") + logger.info(f"[get_decrypted_api_key] Encrypted API key length: {len(db_provider.api_key) if db_provider.api_key else 0}") + + try: + decrypted_key = self.encryption.decrypt(db_provider.api_key) + logger.info(f"[get_decrypted_api_key] Decryption successful, key length: {len(decrypted_key) if decrypted_key else 0}") + return decrypted_key + except Exception as e: + logger.error(f"[get_decrypted_api_key] Decryption failed: {str(e)}") return None - return self.encryption.decrypt(db_provider.api_key) # [/DEF:get_decrypted_api_key:Function] # [/DEF:LLMProviderService:Class] diff --git a/backend/ss-tools-storage/screenshots/4_20260129_174429.png b/backend/ss-tools-storage/screenshots/4_20260129_174429.png deleted file mode 100644 index d02775e4..00000000 Binary files a/backend/ss-tools-storage/screenshots/4_20260129_174429.png and /dev/null differ diff --git a/docs/architecture_decision_superset_migration.md b/docs/architecture_decision_superset_migration.md new file mode 100644 index 00000000..dee2f1c0 --- /dev/null +++ b/docs/architecture_decision_superset_migration.md @@ -0,0 +1,38 @@ +# Архитектурное решение: Standalone сервис vs Интеграция в Superset + +## Рекомендация: Оставить отдельным сервисом + +Хотя интеграция непосредственно в Superset может показаться привлекательной для создания "единого интерфейса", **настоятельно рекомендуется оставить `ss-tools` отдельным сервисом** по следующим архитектурным и техническим причинам. + +### 1. Архитектурный паттерн: Оркестратор против Инстанса +* **Проблема:** `ss-tools` действует как **"Менеджер менеджеров"**. Его основная ценность заключается в подключении и перемещении данных *между* различными окружениями Superset (Dev → Stage → Prod). +* **Почему разделение лучше:** Если встроить эту логику в Superset, придется сделать один конкретный инстанс (например, "Dev") "главным контроллером" для остальных. Если этот инстанс упадет, вы потеряете возможности управления. Отдельный инструмент находится *над* окружениями, рассматривая их все как равные endpoints. + +### 2. Несовместимость технологического стека +* **Superset:** Бэкенд на **Flask** (Python) с фронтендом на **React**. +* **ss-tools:** Бэкенд на **FastAPI** (Python) с фронтендом на **SvelteKit**. +* **Цена миграции:** Перенос в Superset потребует **полного переписывания** всего фронтенда с Svelte на React, чтобы он мог работать как плагин или расширение Superset. Также потребуется рефакторинг внедрения зависимостей (Dependency Injection) из FastAPI, чтобы соответствовать структуре Flask-AppBuilder в Superset. + +### 3. Цикл релизов и гибкость +* **Superset:** Имеет сложный и медленный цикл релизов. Внедрение кастомных фич в основной репозиторий (upstream) затруднительно, а поддержка собственного "форка" Superset — это кошмар сопровождения (конфликты слияния при каждом обновлении). +* **ss-tools:** Как отдельный микросервис, вы можете развертывать новые функции (например, новые LLM плагины или Git-процессы) мгновенно, не беспокоясь о том, что сломаете основную BI-платформу. + +### 4. Безопасность и изоляция +* **Права доступа:** `ss-tools` имеет свои собственные страницы администрирования (`AdminRolesPage` и `AdminUsersPage`), управляющие правами специально для задач *деплоя* и *резервного копирования*. Это задачи DevOps, а не бизнес-аналитики (BI). Смешивание привилегий развертывания с привилегиями просмотра данных в одной системе увеличивает "радиус поражения" в случае проблем с безопасностью. + +### Техническая реализуемость (если все же решиться) +Это технически возможно, но процесс будет выглядеть так: +1. **Frontend:** Придется переписать все компоненты Svelte (`DashboardGrid`, `MappingTable` и т.д.) как компоненты React, чтобы они могли жить внутри `superset-frontend`. +2. **Backend:** Вероятно, придется реализовать это как серверное **расширение Superset** (используя Flask Blueprints), создав новые API endpoints, которые будет вызывать фронтенд на React. +3. **Database:** Придется добавить таблицы проекта (`task_records`, `connection_configs`) в базу данных метаданных Superset через миграции Alembic. + +### Итоговая таблица + +| Характеристика | Отдельный сервис (Текущий) | Интеграция в Superset | +| :--- | :--- | :--- | +| **Скорость разработки** | 🚀 Высокая (Независимая) | 🐢 Низкая (Связана с монолитом) | +| **Технологический стек** | Современный (FastAPI + SvelteKit) | Legacy/Строгий (Flask + React) | +| **Управление мульти-окружением** | ✅ Нативно (Спроектирован для этого) | ⚠️ Неудобно (Один инстанс управляет всеми) | +| **Поддержка (Maintenance)** | ✅ Низкая (Обновление в любое время) | ❌ Высокая (Rebase при каждом обновлении Superset) | + +**Вывод:** Проект уже хорошо спроектирован как специализированный инструмент DevOps (слой `DevOps/Tooling` в вашей семантической карте). Слияние его с Superset, скорее всего, приведет к техническому долгу и регрессии функциональности. diff --git a/frontend/src/components/llm/ProviderConfig.svelte b/frontend/src/components/llm/ProviderConfig.svelte index 1e68e23e..c285dcf8 100644 --- a/frontend/src/components/llm/ProviderConfig.svelte +++ b/frontend/src/components/llm/ProviderConfig.svelte @@ -75,8 +75,15 @@ const method = editingProvider ? 'PUT' : 'POST'; const endpoint = editingProvider ? `/llm/providers/${editingProvider.id}` : '/llm/providers'; + // When editing, only include api_key if user entered a new one + const submitData = { ...formData }; + if (editingProvider && !submitData.api_key) { + // If editing and api_key is empty, don't send it (backend will keep existing) + delete submitData.api_key; + } + try { - await requestApi(endpoint, method, formData); + await requestApi(endpoint, method, submitData); showForm = false; resetForm(); onSave(); diff --git a/semantics/semantic_map.json b/semantics/semantic_map.json index 633021ef..710e5bd2 100644 --- a/semantics/semantic_map.json +++ b/semantics/semantic_map.json @@ -1,6 +1,6 @@ { "project_root": ".", - "generated_at": "2026-01-28T16:48:53.121048", + "generated_at": "2026-02-02T19:29:13.109054", "modules": [ { "name": "generate_semantic_map", @@ -2085,7 +2085,7 @@ "type": "Component", "tier": "STANDARD", "start_line": 1, - "end_line": 166, + "end_line": 161, "tags": { "TIER": "STANDARD", "SEMANTICS": "login, auth, ui, form", @@ -2100,8 +2100,8 @@ "name": "handleLogin", "type": "Function", "tier": "STANDARD", - "start_line": 23, - "end_line": 80, + "start_line": 24, + "end_line": 75, "tags": { "PURPOSE": "Submits the local login form to the backend.", "PRE": "Username and password are not empty.", @@ -2115,12 +2115,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 23 + "line_number": 24 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 23 + "line_number": 24 } ], "score": 0.8 @@ -2130,8 +2130,8 @@ "name": "handleADFSLogin", "type": "Function", "tier": "STANDARD", - "start_line": 82, - "end_line": 89, + "start_line": 77, + "end_line": 84, "tags": { "PURPOSE": "Redirects the user to the ADFS login endpoint." }, @@ -2143,32 +2143,32 @@ { "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", "severity": "WARNING", - "line_number": 82 + "line_number": 77 }, { "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", "severity": "WARNING", - "line_number": 82 + "line_number": 77 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 82 + "line_number": 77 }, { "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", "severity": "WARNING", - "line_number": 82 + "line_number": 77 }, { "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", "severity": "WARNING", - "line_number": 82 + "line_number": 77 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 82 + "line_number": 77 } ], "score": 0.26666666666666655 @@ -2184,12 +2184,12 @@ { "store": "app", "type": "READS_FROM", - "line": 16 + "line": 17 }, { "store": "auth", "type": "READS_FROM", - "line": 92 + "line": 87 } ] }, @@ -2866,12 +2866,76 @@ } ] }, + { + "name": "LLMSettingsPage", + "type": "Component", + "tier": "STANDARD", + "start_line": 1, + "end_line": 48, + "tags": { + "TIER": "STANDARD", + "PURPOSE": "Admin settings page for LLM provider configuration.", + "LAYER": "UI", + "RELATION": "CALLS -> frontend/src/components/llm/ProviderConfig.svelte" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @SEMANTICS (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 1 + } + ], + "score": 0.85 + } + }, + { + "name": "+page", + "type": "Module", + "tier": "TRIVIAL", + "start_line": 1, + "end_line": 48, + "tags": { + "PURPOSE": "Auto-generated module for frontend/src/routes/admin/settings/llm/+page.svelte", + "TIER": "TRIVIAL", + "LAYER": "Unknown" + }, + "relations": [], + "children": [ + { + "name": "fetchProviders", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 17, + "end_line": 17, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + } + ], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, { "name": "MigrationDashboard", "type": "Component", "tier": "STANDARD", "start_line": 1, - "end_line": 390, + "end_line": 391, "tags": { "SEMANTICS": "migration, dashboard, environment, selection, database-replacement", "PURPOSE": "Main dashboard for configuring and starting migrations.", @@ -3127,7 +3191,7 @@ "type": "Component", "tier": "STANDARD", "start_line": 306, - "end_line": 319, + "end_line": 320, "tags": {}, "relations": [], "children": [], @@ -6091,7 +6155,7 @@ "type": "Component", "tier": "STANDARD", "start_line": 1, - "end_line": 258, + "end_line": 319, "tags": { "SEMANTICS": "dashboard, grid, selection, pagination", "PURPOSE": "Displays a grid of dashboards with selection and pagination.", @@ -6101,12 +6165,60 @@ }, "relations": [], "children": [ + { + "name": "handleValidate", + "type": "Function", + "tier": "STANDARD", + "start_line": 43, + "end_line": 86, + "tags": { + "PURPOSE": "Triggers dashboard validation task." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 43 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 43 + }, + { + "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 43 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 43 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 43 + }, + { + "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 43 + } + ], + "score": 0.26666666666666655 + } + }, { "name": "handleSort", "type": "Function", "tier": "STANDARD", - "start_line": 71, - "end_line": 83, + "start_line": 120, + "end_line": 132, "tags": { "PURPOSE": "Toggles sort direction or changes sort column.", "PRE": "column name is provided.", @@ -6120,12 +6232,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 71 + "line_number": 120 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 71 + "line_number": 120 } ], "score": 0.8 @@ -6135,8 +6247,8 @@ "name": "handleSelectionChange", "type": "Function", "tier": "STANDARD", - "start_line": 85, - "end_line": 99, + "start_line": 134, + "end_line": 148, "tags": { "PURPOSE": "Handles individual checkbox changes.", "PRE": "dashboard ID and checked status provided.", @@ -6150,12 +6262,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 85 + "line_number": 134 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 85 + "line_number": 134 } ], "score": 0.8 @@ -6165,8 +6277,8 @@ "name": "handleSelectAll", "type": "Function", "tier": "STANDARD", - "start_line": 101, - "end_line": 119, + "start_line": 150, + "end_line": 168, "tags": { "PURPOSE": "Handles select all checkbox.", "PRE": "checked status provided.", @@ -6180,12 +6292,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 101 + "line_number": 150 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 101 + "line_number": 150 } ], "score": 0.8 @@ -6195,8 +6307,8 @@ "name": "goToPage", "type": "Function", "tier": "STANDARD", - "start_line": 121, - "end_line": 130, + "start_line": 170, + "end_line": 179, "tags": { "PURPOSE": "Changes current page.", "PRE": "page index is provided.", @@ -6210,12 +6322,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 121 + "line_number": 170 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 121 + "line_number": 170 } ], "score": 0.8 @@ -6225,8 +6337,8 @@ "name": "openGit", "type": "Function", "tier": "STANDARD", - "start_line": 132, - "end_line": 141, + "start_line": 181, + "end_line": 190, "tags": { "PURPOSE": "Opens the Git management modal for a dashboard." }, @@ -6238,32 +6350,32 @@ { "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", "severity": "WARNING", - "line_number": 132 + "line_number": 181 }, { "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", "severity": "WARNING", - "line_number": 132 + "line_number": 181 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 132 + "line_number": 181 }, { "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", "severity": "WARNING", - "line_number": 132 + "line_number": 181 }, { "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", "severity": "WARNING", - "line_number": 132 + "line_number": 181 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 132 + "line_number": 181 } ], "score": 0.26666666666666655 @@ -6291,6 +6403,11 @@ "name": "selectedIds", "type": "number[] ", "default": "[]" + }, + { + "name": "environmentId", + "type": "string ", + "default": "\"ss1\"" } ], "events": [ @@ -6300,47 +6417,52 @@ { "store": "t", "type": "WRITES_TO", - "line": 151 + "line": 200 }, { "store": "t", "type": "READS_FROM", - "line": 170 + "line": 219 }, { "store": "t", "type": "READS_FROM", - "line": 173 + "line": 222 }, { "store": "t", "type": "READS_FROM", - "line": 176 + "line": 225 }, { "store": "t", "type": "WRITES_TO", - "line": 178 + "line": 227 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 228 }, { "store": "t", "type": "READS_FROM", - "line": 206 + "line": 267 }, { "store": "t", "type": "READS_FROM", - "line": 218 + "line": 279 }, { "store": "t", "type": "READS_FROM", - "line": 230 + "line": 291 }, { "store": "t", "type": "READS_FROM", - "line": 238 + "line": 299 } ] }, @@ -6349,7 +6471,7 @@ "type": "Component", "tier": "STANDARD", "start_line": 1, - "end_line": 79, + "end_line": 80, "tags": { "SEMANTICS": "navbar, navigation, header, layout", "PURPOSE": "Main navigation bar for the application.", @@ -6390,6 +6512,11 @@ "type": "READS_FROM", "line": 11 }, + { + "store": "lib", + "type": "READS_FROM", + "line": 12 + }, { "store": "app", "type": "READS_FROM", @@ -6475,15 +6602,20 @@ "type": "WRITES_TO", "line": 60 }, + { + "store": "t", + "type": "WRITES_TO", + "line": 61 + }, { "store": "auth", "type": "READS_FROM", - "line": 66 + "line": 67 }, { "store": "auth", "type": "WRITES_TO", - "line": 68 + "line": 69 } ] }, @@ -6492,7 +6624,7 @@ "type": "Module", "tier": "TRIVIAL", "start_line": 1, - "end_line": 79, + "end_line": 80, "tags": { "PURPOSE": "Auto-generated module for frontend/src/components/Navbar.svelte", "TIER": "TRIVIAL", @@ -6530,7 +6662,7 @@ "type": "Component", "tier": "STANDARD", "start_line": 1, - "end_line": 215, + "end_line": 199, "tags": { "SEMANTICS": "task, history, list, status, monitoring", "PURPOSE": "Displays a list of recent tasks with their status and allows selecting them for viewing logs.", @@ -6543,8 +6675,8 @@ "name": "fetchTasks", "type": "Function", "tier": "STANDARD", - "start_line": 18, - "end_line": 50, + "start_line": 19, + "end_line": 47, "tags": { "PURPOSE": "Fetches the list of recent tasks from the API.", "PRE": "None.", @@ -6558,12 +6690,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 18 + "line_number": 19 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 18 + "line_number": 19 } ], "score": 0.8 @@ -6573,8 +6705,8 @@ "name": "clearTasks", "type": "Function", "tier": "STANDARD", - "start_line": 52, - "end_line": 73, + "start_line": 49, + "end_line": 65, "tags": { "PURPOSE": "Clears tasks from the history, optionally filtered by status.", "PRE": "User confirms deletion via prompt.", @@ -6588,12 +6720,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 52 + "line_number": 49 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 52 + "line_number": 49 } ], "score": 0.8 @@ -6603,8 +6735,8 @@ "name": "selectTask", "type": "Function", "tier": "STANDARD", - "start_line": 75, - "end_line": 97, + "start_line": 67, + "end_line": 81, "tags": { "PURPOSE": "Selects a task and fetches its full details.", "PRE": "task object is provided.", @@ -6618,12 +6750,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 75 + "line_number": 67 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 75 + "line_number": 67 } ], "score": 0.8 @@ -6633,8 +6765,8 @@ "name": "getStatusColor", "type": "Function", "tier": "STANDARD", - "start_line": 99, - "end_line": 113, + "start_line": 83, + "end_line": 97, "tags": { "PURPOSE": "Returns the CSS color class for a given task status.", "PRE": "status string is provided.", @@ -6648,12 +6780,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 99 + "line_number": 83 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 99 + "line_number": 83 } ], "score": 0.8 @@ -6663,8 +6795,8 @@ "name": "onMount", "type": "Function", "tier": "STANDARD", - "start_line": 115, - "end_line": 123, + "start_line": 99, + "end_line": 107, "tags": { "PURPOSE": "Initializes the component by fetching tasks and starting polling.", "PRE": "Component is mounting.", @@ -6678,12 +6810,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 115 + "line_number": 99 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 115 + "line_number": 99 } ], "score": 0.8 @@ -6693,8 +6825,8 @@ "name": "onDestroy", "type": "Function", "tier": "STANDARD", - "start_line": 125, - "end_line": 132, + "start_line": 109, + "end_line": 116, "tags": { "PURPOSE": "Cleans up the polling interval when the component is destroyed.", "PRE": "Component is being destroyed.", @@ -6708,12 +6840,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 125 + "line_number": 109 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 125 + "line_number": 109 } ], "score": 0.8 @@ -6735,27 +6867,27 @@ { "store": "selectedTask", "type": "READS_FROM", - "line": 38 + "line": 35 }, { "store": "selectedTask", "type": "WRITES_TO", - "line": 39 + "line": 36 }, { "store": "selectedTask", "type": "WRITES_TO", - "line": 40 + "line": 37 }, { "store": "selectedTask", "type": "WRITES_TO", - "line": 177 + "line": 161 }, { "store": "selectedTask", "type": "WRITES_TO", - "line": 177 + "line": 161 } ] }, @@ -6799,7 +6931,7 @@ "type": "Component", "tier": "STANDARD", "start_line": 1, - "end_line": 395, + "end_line": 380, "tags": { "SEMANTICS": "task, runner, logs, websocket", "PURPOSE": "Connects to a WebSocket to display real-time logs for a running task.", @@ -6845,7 +6977,7 @@ "type": "Function", "tier": "STANDARD", "start_line": 134, - "end_line": 156, + "end_line": 154, "tags": { "PURPOSE": "Fetches the list of databases in the target environment.", "PRE": "task must be selected and have a target environment parameter.", @@ -6874,8 +7006,8 @@ "name": "handleMappingResolve", "type": "Function", "tier": "STANDARD", - "start_line": 158, - "end_line": 201, + "start_line": 156, + "end_line": 190, "tags": { "PURPOSE": "Handles the resolution of a missing database mapping.", "PRE": "event.detail contains sourceDbUuid, targetDbUuid, and targetDbName.", @@ -6889,12 +7021,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 158 + "line_number": 156 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 158 + "line_number": 156 } ], "score": 0.8 @@ -6904,8 +7036,8 @@ "name": "handlePasswordResume", "type": "Function", "tier": "STANDARD", - "start_line": 203, - "end_line": 225, + "start_line": 192, + "end_line": 210, "tags": { "PURPOSE": "Handles the submission of database passwords to resume a task.", "PRE": "event.detail contains passwords dictionary.", @@ -6919,12 +7051,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 203 + "line_number": 192 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 203 + "line_number": 192 } ], "score": 0.8 @@ -6934,8 +7066,8 @@ "name": "startDataTimeout", "type": "Function", "tier": "STANDARD", - "start_line": 227, - "end_line": 239, + "start_line": 212, + "end_line": 224, "tags": { "PURPOSE": "Starts a timeout to detect when the log stream has stalled.", "PRE": "None.", @@ -6949,12 +7081,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 227 + "line_number": 212 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 227 + "line_number": 212 } ], "score": 0.8 @@ -6964,8 +7096,8 @@ "name": "resetDataTimeout", "type": "Function", "tier": "STANDARD", - "start_line": 241, - "end_line": 250, + "start_line": 226, + "end_line": 235, "tags": { "PURPOSE": "Resets the data stall timeout.", "PRE": "dataTimeout must be active.", @@ -6979,12 +7111,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 241 + "line_number": 226 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 241 + "line_number": 226 } ], "score": 0.8 @@ -6994,8 +7126,8 @@ "name": "onMount", "type": "Function", "tier": "STANDARD", - "start_line": 252, - "end_line": 279, + "start_line": 237, + "end_line": 264, "tags": { "PURPOSE": "Initializes the component and subscribes to task selection changes.", "PRE": "Svelte component is mounting.", @@ -7009,12 +7141,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 252 + "line_number": 237 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 252 + "line_number": 237 } ], "score": 0.8 @@ -7024,8 +7156,8 @@ "name": "onDestroy", "type": "Function", "tier": "STANDARD", - "start_line": 281, - "end_line": 295, + "start_line": 266, + "end_line": 280, "tags": { "PURPOSE": "Close WebSocket connection when the component is destroyed.", "PRE": "Component is being destroyed.", @@ -7055,52 +7187,52 @@ { "store": "selectedTask", "type": "READS_FROM", - "line": 300 + "line": 285 }, { "store": "selectedTask", "type": "WRITES_TO", - "line": 302 + "line": 287 }, { "store": "selectedTask", "type": "WRITES_TO", - "line": 335 + "line": 320 }, { "store": "selectedTask", "type": "WRITES_TO", - "line": 336 + "line": 321 }, { "store": "selectedTask", "type": "WRITES_TO", - "line": 337 + "line": 322 }, { "store": "selectedTask", "type": "WRITES_TO", - "line": 337 + "line": 322 }, { "store": "selectedTask", "type": "WRITES_TO", - "line": 338 + "line": 323 }, { "store": "selectedTask", "type": "WRITES_TO", - "line": 342 + "line": 327 }, { "store": "taskLogs", "type": "READS_FROM", - "line": 349 + "line": 334 }, { "store": "taskLogs", "type": "READS_FROM", - "line": 352 + "line": 337 } ] }, @@ -7420,7 +7552,7 @@ "type": "Component", "tier": "STANDARD", "start_line": 1, - "end_line": 61, + "end_line": 50, "tags": { "SEMANTICS": "auth, guard, route, protection", "PURPOSE": "Wraps content to ensure only authenticated users can access it.", @@ -7445,37 +7577,32 @@ { "store": "app", "type": "READS_FROM", - "line": 15 + "line": 16 }, { "store": "auth", "type": "READS_FROM", - "line": 23 + "line": 24 }, { "store": "auth", "type": "READS_FROM", - "line": 23 + "line": 24 }, { "store": "auth", "type": "READS_FROM", - "line": 28 + "line": 36 }, { "store": "auth", "type": "READS_FROM", - "line": 47 + "line": 42 }, { "store": "auth", "type": "READS_FROM", - "line": 53 - }, - { - "store": "auth", - "type": "READS_FROM", - "line": 57 + "line": 46 } ] }, @@ -8034,7 +8161,7 @@ "type": "Component", "tier": "STANDARD", "start_line": 1, - "end_line": 183, + "end_line": 250, "tags": { "SEMANTICS": "mapper, tool, dataset, postgresql, excel", "PURPOSE": "UI component for mapping dataset column verbose names using the MapperPlugin.", @@ -8047,8 +8174,8 @@ "name": "fetchData", "type": "Function", "tier": "STANDARD", - "start_line": 31, - "end_line": 44, + "start_line": 35, + "end_line": 47, "tags": { "PURPOSE": "Fetches environments and saved connections.", "PRE": "None.", @@ -8062,12 +8189,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 31 + "line_number": 35 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 31 + "line_number": 35 } ], "score": 0.8 @@ -8077,8 +8204,8 @@ "name": "handleRunMapper", "type": "Function", "tier": "STANDARD", - "start_line": 46, - "end_line": 87, + "start_line": 49, + "end_line": 90, "tags": { "PURPOSE": "Triggers the MapperPlugin task.", "PRE": "selectedEnv and datasetId are set; source-specific fields are valid.", @@ -8092,12 +8219,42 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 46 + "line_number": 49 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 46 + "line_number": 49 + } + ], + "score": 0.8 + } + }, + { + "name": "handleGenerateDocs", + "type": "Function", + "tier": "STANDARD", + "start_line": 92, + "end_line": 127, + "tags": { + "PURPOSE": "Triggers the LLM Documentation task.", + "PRE": "selectedEnv and datasetId are set.", + "POST": "Documentation task is started." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 92 + }, + { + "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 92 } ], "score": 0.8 @@ -8119,100 +8276,143 @@ { "store": "t", "type": "READS_FROM", - "line": 41 + "line": 44 }, { "store": "t", "type": "READS_FROM", - "line": 52 + "line": 55 }, { "store": "t", "type": "READS_FROM", - "line": 57 + "line": 60 }, { "store": "t", "type": "READS_FROM", - "line": 62 + "line": 65 }, { "store": "t", "type": "READS_FROM", - "line": 80 - }, - { - "store": "t", - "type": "WRITES_TO", - "line": 94 - }, - { - "store": "t", - "type": "WRITES_TO", - "line": 99 + "line": 83 }, { "store": "t", "type": "READS_FROM", - "line": 102 + "line": 98 }, { "store": "t", "type": "WRITES_TO", - "line": 109 + "line": 144 }, { "store": "t", "type": "WRITES_TO", - "line": 117 - }, - { - "store": "t", - "type": "WRITES_TO", - "line": 121 - }, - { - "store": "t", - "type": "WRITES_TO", - "line": 125 - }, - { - "store": "t", - "type": "WRITES_TO", - "line": 134 + "line": 149 }, { "store": "t", "type": "READS_FROM", - "line": 137 - }, - { - "store": "t", - "type": "WRITES_TO", - "line": 145 - }, - { - "store": "t", - "type": "WRITES_TO", "line": 152 }, { "store": "t", "type": "WRITES_TO", - "line": 162 + "line": 159 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 167 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 171 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 175 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 184 }, { "store": "t", "type": "READS_FROM", - "line": 176 + "line": 187 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 195 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 202 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 212 }, { "store": "t", "type": "READS_FROM", - "line": 176 + "line": 237 + }, + { + "store": "t", + "type": "READS_FROM", + "line": 237 } ] }, + { + "name": "MapperTool", + "type": "Module", + "tier": "TRIVIAL", + "start_line": 1, + "end_line": 250, + "tags": { + "PURPOSE": "Auto-generated module for frontend/src/components/tools/MapperTool.svelte", + "TIER": "TRIVIAL", + "LAYER": "Unknown" + }, + "relations": [], + "children": [ + { + "name": "handleApplyDoc", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 129, + "end_line": 129, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + } + ], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, { "name": "DebugTool", "type": "Component", @@ -8682,7 +8882,7 @@ "type": "Component", "tier": "STANDARD", "start_line": 1, - "end_line": 175, + "end_line": 211, "tags": { "SEMANTICS": "git, commit, modal, version_control, diff", "PURPOSE": "\u041c\u043e\u0434\u0430\u043b\u044c\u043d\u043e\u0435 \u043e\u043a\u043d\u043e \u0434\u043b\u044f \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u044f \u043a\u043e\u043c\u043c\u0438\u0442\u0430 \u0441 \u043f\u0440\u043e\u0441\u043c\u043e\u0442\u0440\u043e\u043c \u0438\u0437\u043c\u0435\u043d\u0435\u043d\u0438\u0439 (diff).", @@ -8691,12 +8891,60 @@ }, "relations": [], "children": [ + { + "name": "handleGenerateMessage", + "type": "Function", + "tier": "STANDARD", + "start_line": 36, + "end_line": 55, + "tags": { + "PURPOSE": "Generates a commit message using LLM." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 36 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 36 + }, + { + "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 36 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 36 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 36 + }, + { + "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 36 + } + ], + "score": 0.26666666666666655 + } + }, { "name": "loadStatus", "type": "Function", "tier": "STANDARD", - "start_line": 34, - "end_line": 61, + "start_line": 57, + "end_line": 84, "tags": { "PURPOSE": "\u0417\u0430\u0433\u0440\u0443\u0436\u0430\u0435\u0442 \u0442\u0435\u043a\u0443\u0449\u0438\u0439 \u0441\u0442\u0430\u0442\u0443\u0441 \u0440\u0435\u043f\u043e\u0437\u0438\u0442\u043e\u0440\u0438\u044f \u0438 diff.", "PRE": "dashboardId \u0434\u043e\u043b\u0436\u0435\u043d \u0431\u044b\u0442\u044c \u0432\u0430\u043b\u0438\u0434\u043d\u044b\u043c." @@ -8709,22 +8957,22 @@ { "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", "severity": "WARNING", - "line_number": 34 + "line_number": 57 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 34 + "line_number": 57 }, { "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", "severity": "WARNING", - "line_number": 34 + "line_number": 57 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 34 + "line_number": 57 } ], "score": 0.5333333333333333 @@ -8734,8 +8982,8 @@ "name": "handleCommit", "type": "Function", "tier": "STANDARD", - "start_line": 63, - "end_line": 87, + "start_line": 86, + "end_line": 110, "tags": { "PURPOSE": "\u0421\u043e\u0437\u0434\u0430\u0435\u0442 \u043a\u043e\u043c\u043c\u0438\u0442 \u0441 \u0443\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u043c \u0441\u043e\u043e\u0431\u0449\u0435\u043d\u0438\u0435\u043c.", "PRE": "message \u043d\u0435 \u0434\u043e\u043b\u0436\u043d\u043e \u0431\u044b\u0442\u044c \u043f\u0443\u0441\u0442\u044b\u043c.", @@ -8749,12 +8997,12 @@ { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 63 + "line_number": 86 }, { "message": "Missing Belief State Logging: Function should use console.log with [ID][STATE] (required for STANDARD tier)", "severity": "WARNING", - "line_number": 63 + "line_number": 86 } ], "score": 0.8 @@ -9268,6 +9516,414 @@ } ] }, + { + "name": "DocPreview", + "type": "Component", + "tier": "STANDARD", + "start_line": 1, + "end_line": 79, + "tags": { + "TIER": "STANDARD", + "PURPOSE": "UI component for previewing generated dataset documentation before saving.", + "LAYER": "UI", + "RELATION": "DEPENDS_ON -> backend/src/plugins/llm_analysis/plugin.py", + "TYPE": "{Object} */" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @SEMANTICS (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 1 + } + ], + "score": 0.85 + }, + "props": [ + { + "name": "documentation", + "type": "any", + "default": "null" + }, + { + "name": "onSave", + "type": "any", + "default": "async (doc) => {}" + }, + { + "name": "onCancel", + "type": "any", + "default": "() => {}" + } + ], + "data_flow": [ + { + "store": "t", + "type": "WRITES_TO", + "line": 34 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 37 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 40 + }, + { + "store": "t", + "type": "READS_FROM", + "line": 65 + }, + { + "store": "t", + "type": "READS_FROM", + "line": 72 + }, + { + "store": "t", + "type": "READS_FROM", + "line": 72 + } + ] + }, + { + "name": "DocPreview", + "type": "Module", + "tier": "TRIVIAL", + "start_line": 1, + "end_line": 79, + "tags": { + "PURPOSE": "Auto-generated module for frontend/src/components/llm/DocPreview.svelte", + "TIER": "TRIVIAL", + "LAYER": "Unknown" + }, + "relations": [], + "children": [ + { + "name": "handleSave", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 19, + "end_line": 19, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + } + ], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "ProviderConfig", + "type": "Component", + "tier": "STANDARD", + "start_line": 1, + "end_line": 219, + "tags": { + "TIER": "STANDARD", + "PURPOSE": "UI form for managing LLM provider configurations.", + "LAYER": "UI", + "RELATION": "DEPENDS_ON -> backend/src/api/routes/llm.py", + "TYPE": "{Array} */" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @SEMANTICS (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 1 + } + ], + "score": 0.85 + }, + "props": [ + { + "name": "providers", + "type": "any", + "default": "[]" + }, + { + "name": "onSave", + "type": "any", + "default": "() => {}" + } + ], + "data_flow": [ + { + "store": "t", + "type": "WRITES_TO", + "line": 55 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 62 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 64 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 67 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 103 + }, + { + "store": "t", + "type": "READS_FROM", + "line": 108 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 115 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 115 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 119 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 124 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 133 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 138 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 143 + }, + { + "store": "t", + "type": "WRITES_TO", + "line": 149 + }, + { + "store": "t", + "type": "READS_FROM", + "line": 164 + }, + { + "store": "t", + "type": "READS_FROM", + "line": 171 + }, + { + "store": "t", + "type": "READS_FROM", + "line": 171 + }, + { + "store": "t", + "type": "READS_FROM", + "line": 177 + }, + { + "store": "t", + "type": "READS_FROM", + "line": 191 + }, + { + "store": "t", + "type": "READS_FROM", + "line": 201 + }, + { + "store": "t", + "type": "READS_FROM", + "line": 213 + } + ] + }, + { + "name": "ProviderConfig", + "type": "Module", + "tier": "TRIVIAL", + "start_line": 1, + "end_line": 219, + "tags": { + "PURPOSE": "Auto-generated module for frontend/src/components/llm/ProviderConfig.svelte", + "TIER": "TRIVIAL", + "LAYER": "Unknown" + }, + "relations": [], + "children": [ + { + "name": "resetForm", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 33, + "end_line": 33, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "handleEdit", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 46, + "end_line": 46, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "testConnection", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 52, + "end_line": 52, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "handleSubmit", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 73, + "end_line": 73, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "toggleActive", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 88, + "end_line": 88, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + } + ], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "ValidationReport", + "type": "Module", + "tier": "TRIVIAL", + "start_line": 1, + "end_line": 75, + "tags": { + "PURPOSE": "Auto-generated module for frontend/src/components/llm/ValidationReport.svelte", + "TIER": "TRIVIAL", + "LAYER": "Unknown" + }, + "relations": [], + "children": [ + { + "name": "getStatusColor", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 8, + "end_line": 8, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + } + ], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, { "name": "backend.delete_running_tasks", "type": "Module", @@ -9329,7 +9985,7 @@ "type": "Module", "tier": "STANDARD", "start_line": 1, - "end_line": 198, + "end_line": 216, "tags": { "SEMANTICS": "app, main, entrypoint, fastapi", "PURPOSE": "The main entry point for the FastAPI application. It initializes the app, configures CORS, sets up dependencies, includes API routers, and defines the WebSocket endpoint for log streaming.", @@ -9342,8 +9998,8 @@ "name": "App", "type": "Global", "tier": "STANDARD", - "start_line": 26, - "end_line": 34, + "start_line": 27, + "end_line": 35, "tags": { "SEMANTICS": "app, fastapi, instance", "PURPOSE": "The global FastAPI application instance." @@ -9360,8 +10016,8 @@ "name": "startup_event", "type": "Function", "tier": "STANDARD", - "start_line": 36, - "end_line": 46, + "start_line": 37, + "end_line": 47, "tags": { "PURPOSE": "Handles application startup tasks, such as starting the scheduler.", "PRE": "None.", @@ -9379,8 +10035,8 @@ "name": "shutdown_event", "type": "Function", "tier": "STANDARD", - "start_line": 48, - "end_line": 58, + "start_line": 49, + "end_line": 59, "tags": { "PURPOSE": "Handles application shutdown tasks, such as stopping the scheduler.", "PRE": "None.", @@ -9398,8 +10054,8 @@ "name": "log_requests", "type": "Function", "tier": "STANDARD", - "start_line": 74, - "end_line": 87, + "start_line": 75, + "end_line": 104, "tags": { "PURPOSE": "Middleware to log incoming HTTP requests and their response status.", "PRE": "request is a FastAPI Request object.", @@ -9418,8 +10074,8 @@ "name": "websocket_endpoint", "type": "Function", "tier": "STANDARD", - "start_line": 102, - "end_line": 159, + "start_line": 120, + "end_line": 177, "tags": { "PURPOSE": "Provides a WebSocket endpoint for real-time log streaming of a task.", "PRE": "task_id must be a valid task ID.", @@ -9437,8 +10093,8 @@ "name": "StaticFiles", "type": "Mount", "tier": "STANDARD", - "start_line": 161, - "end_line": 197, + "start_line": 179, + "end_line": 215, "tags": { "SEMANTICS": "static, frontend, spa", "PURPOSE": "Mounts the frontend build directory to serve static assets." @@ -9449,8 +10105,8 @@ "name": "serve_spa", "type": "Function", "tier": "STANDARD", - "start_line": 169, - "end_line": 186, + "start_line": 187, + "end_line": 204, "tags": { "PURPOSE": "Serves frontend static files or index.html for SPA routing.", "PRE": "file_path is requested by the client.", @@ -9468,8 +10124,8 @@ "name": "read_root", "type": "Function", "tier": "STANDARD", - "start_line": 188, - "end_line": 196, + "start_line": 206, + "end_line": 214, "tags": { "PURPOSE": "A simple root endpoint to confirm that the API is running when frontend is missing.", "PRE": "None.", @@ -9489,6 +10145,24 @@ "issues": [], "score": 1.0 } + }, + { + "name": "network_error_handler", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 82, + "end_line": 82, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } } ], "compliance": { @@ -11384,7 +12058,7 @@ "type": "Module", "tier": "STANDARD", "start_line": 1, - "end_line": 128, + "end_line": 141, "tags": { "SEMANTICS": "database, sqlite, sqlalchemy, session, persistence", "PURPOSE": "Configures the SQLite database connection and session management.", @@ -11406,12 +12080,29 @@ } ], "children": [ + { + "name": "BASE_DIR", + "type": "Variable", + "tier": "STANDARD", + "start_line": 28, + "end_line": 31, + "tags": { + "PURPOSE": "Base directory for the backend (where .db files should reside)." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, { "name": "DATABASE_URL", "type": "Constant", "tier": "STANDARD", - "start_line": 26, - "end_line": 29, + "start_line": 33, + "end_line": 36, "tags": { "PURPOSE": "URL for the main mappings database." }, @@ -11427,8 +12118,8 @@ "name": "TASKS_DATABASE_URL", "type": "Constant", "tier": "STANDARD", - "start_line": 31, - "end_line": 34, + "start_line": 38, + "end_line": 41, "tags": { "PURPOSE": "URL for the tasks execution database." }, @@ -11444,8 +12135,8 @@ "name": "AUTH_DATABASE_URL", "type": "Constant", "tier": "STANDARD", - "start_line": 36, - "end_line": 39, + "start_line": 43, + "end_line": 52, "tags": { "PURPOSE": "URL for the authentication database." }, @@ -11461,8 +12152,8 @@ "name": "engine", "type": "Variable", "tier": "STANDARD", - "start_line": 41, - "end_line": 44, + "start_line": 54, + "end_line": 57, "tags": { "PURPOSE": "SQLAlchemy engine for mappings database." }, @@ -11478,8 +12169,8 @@ "name": "tasks_engine", "type": "Variable", "tier": "STANDARD", - "start_line": 46, - "end_line": 49, + "start_line": 59, + "end_line": 62, "tags": { "PURPOSE": "SQLAlchemy engine for tasks database." }, @@ -11495,8 +12186,8 @@ "name": "auth_engine", "type": "Variable", "tier": "STANDARD", - "start_line": 51, - "end_line": 54, + "start_line": 64, + "end_line": 67, "tags": { "PURPOSE": "SQLAlchemy engine for authentication database." }, @@ -11512,8 +12203,8 @@ "name": "SessionLocal", "type": "Class", "tier": "STANDARD", - "start_line": 56, - "end_line": 60, + "start_line": 69, + "end_line": 73, "tags": { "PURPOSE": "A session factory for the main mappings database.", "PRE": "engine is initialized." @@ -11526,12 +12217,12 @@ { "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", "severity": "WARNING", - "line_number": 56 + "line_number": 69 }, { "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", "severity": "WARNING", - "line_number": 56 + "line_number": 69 } ], "score": 0.7000000000000001 @@ -11541,8 +12232,8 @@ "name": "TasksSessionLocal", "type": "Class", "tier": "STANDARD", - "start_line": 62, - "end_line": 66, + "start_line": 75, + "end_line": 79, "tags": { "PURPOSE": "A session factory for the tasks execution database.", "PRE": "tasks_engine is initialized." @@ -11555,12 +12246,12 @@ { "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", "severity": "WARNING", - "line_number": 62 + "line_number": 75 }, { "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", "severity": "WARNING", - "line_number": 62 + "line_number": 75 } ], "score": 0.7000000000000001 @@ -11570,8 +12261,8 @@ "name": "AuthSessionLocal", "type": "Class", "tier": "STANDARD", - "start_line": 68, - "end_line": 72, + "start_line": 81, + "end_line": 85, "tags": { "PURPOSE": "A session factory for the authentication database.", "PRE": "auth_engine is initialized." @@ -11584,12 +12275,12 @@ { "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", "severity": "WARNING", - "line_number": 68 + "line_number": 81 }, { "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", "severity": "WARNING", - "line_number": 68 + "line_number": 81 } ], "score": 0.7000000000000001 @@ -11599,8 +12290,8 @@ "name": "init_db", "type": "Function", "tier": "STANDARD", - "start_line": 74, - "end_line": 84, + "start_line": 87, + "end_line": 97, "tags": { "PURPOSE": "Initializes the database by creating all tables.", "PRE": "engine, tasks_engine and auth_engine are initialized.", @@ -11619,8 +12310,8 @@ "name": "get_db", "type": "Function", "tier": "STANDARD", - "start_line": 86, - "end_line": 98, + "start_line": 99, + "end_line": 111, "tags": { "PURPOSE": "Dependency for getting a database session.", "PRE": "SessionLocal is initialized.", @@ -11639,8 +12330,8 @@ "name": "get_tasks_db", "type": "Function", "tier": "STANDARD", - "start_line": 100, - "end_line": 112, + "start_line": 113, + "end_line": 125, "tags": { "PURPOSE": "Dependency for getting a tasks database session.", "PRE": "TasksSessionLocal is initialized.", @@ -11659,8 +12350,8 @@ "name": "get_auth_db", "type": "Function", "tier": "STANDARD", - "start_line": 114, - "end_line": 126, + "start_line": 127, + "end_line": 139, "tags": { "PURPOSE": "Dependency for getting an authentication database session.", "PRE": "AuthSessionLocal is initialized.", @@ -13653,7 +14344,7 @@ "type": "Module", "tier": "STANDARD", "start_line": 1, - "end_line": 326, + "end_line": 340, "tags": { "SEMANTICS": "network, http, client, api, requests, session, authentication", "PURPOSE": "\u0418\u043d\u043a\u0430\u043f\u0441\u0443\u043b\u0438\u0440\u0443\u0435\u0442 \u043d\u0438\u0437\u043a\u043e\u0443\u0440\u043e\u0432\u043d\u0435\u0432\u0443\u044e HTTP-\u043b\u043e\u0433\u0438\u043a\u0443 \u0434\u043b\u044f \u0432\u0437\u0430\u0438\u043c\u043e\u0434\u0435\u0439\u0441\u0442\u0432\u0438\u044f \u0441 Superset API, \u0432\u043a\u043b\u044e\u0447\u0430\u044f \u0430\u0443\u0442\u0435\u043d\u0442\u0438\u0444\u0438\u043a\u0430\u0446\u0438\u044e, \u0443\u043f\u0440\u0430\u0432\u043b\u0435\u043d\u0438\u0435 \u0441\u0435\u0441\u0441\u0438\u0435\u0439, retry-\u043b\u043e\u0433\u0438\u043a\u0443 \u0438 \u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u0443 \u043e\u0448\u0438\u0431\u043e\u043a.", @@ -13916,7 +14607,7 @@ "type": "Class", "tier": "STANDARD", "start_line": 89, - "end_line": 324, + "end_line": 338, "tags": { "PURPOSE": "\u0418\u043d\u043a\u0430\u043f\u0441\u0443\u043b\u0438\u0440\u0443\u0435\u0442 HTTP-\u043b\u043e\u0433\u0438\u043a\u0443 \u0434\u043b\u044f \u0440\u0430\u0431\u043e\u0442\u044b \u0441 API, \u0432\u043a\u043b\u044e\u0447\u0430\u044f \u0441\u0435\u0441\u0441\u0438\u0438, \u0430\u0443\u0442\u0435\u043d\u0442\u0438\u0444\u0438\u043a\u0430\u0446\u0438\u044e, \u0438 \u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u0443 \u0437\u0430\u043f\u0440\u043e\u0441\u043e\u0432." }, @@ -13967,7 +14658,7 @@ "type": "Function", "tier": "STANDARD", "start_line": 132, - "end_line": 159, + "end_line": 171, "tags": { "PURPOSE": "\u0412\u044b\u043f\u043e\u043b\u043d\u044f\u0435\u0442 \u0430\u0443\u0442\u0435\u043d\u0442\u0438\u0444\u0438\u043a\u0430\u0446\u0438\u044e \u0432 Superset API \u0438 \u043f\u043e\u043b\u0443\u0447\u0430\u0435\u0442 access \u0438 CSRF \u0442\u043e\u043a\u0435\u043d\u044b.", "PRE": "self.auth and self.base_url must be valid.", @@ -13987,8 +14678,8 @@ "name": "headers", "type": "Function", "tier": "STANDARD", - "start_line": 162, - "end_line": 175, + "start_line": 174, + "end_line": 187, "tags": { "PURPOSE": "\u0412\u043e\u0437\u0432\u0440\u0430\u0449\u0430\u0435\u0442 HTTP-\u0437\u0430\u0433\u043e\u043b\u043e\u0432\u043a\u0438 \u0434\u043b\u044f \u0430\u0443\u0442\u0435\u043d\u0442\u0438\u0444\u0438\u0446\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u044b\u0445 \u0437\u0430\u043f\u0440\u043e\u0441\u043e\u0432.", "PRE": "APIClient is initialized and authenticated or can be authenticated.", @@ -14006,8 +14697,8 @@ "name": "request", "type": "Function", "tier": "STANDARD", - "start_line": 177, - "end_line": 201, + "start_line": 189, + "end_line": 213, "tags": { "PURPOSE": "\u0412\u044b\u043f\u043e\u043b\u043d\u044f\u0435\u0442 \u0443\u043d\u0438\u0432\u0435\u0440\u0441\u0430\u043b\u044c\u043d\u044b\u0439 HTTP-\u0437\u0430\u043f\u0440\u043e\u0441 \u043a API.", "PARAM": "raw_response (bool) - \u0412\u043e\u0437\u0432\u0440\u0430\u0449\u0430\u0442\u044c \u043b\u0438 \u0441\u044b\u0440\u043e\u0439 \u043e\u0442\u0432\u0435\u0442.", @@ -14028,8 +14719,8 @@ "name": "_handle_http_error", "type": "Function", "tier": "STANDARD", - "start_line": 203, - "end_line": 216, + "start_line": 215, + "end_line": 230, "tags": { "PURPOSE": "(Helper) \u041f\u0440\u0435\u043e\u0431\u0440\u0430\u0437\u0443\u0435\u0442 HTTP \u043e\u0448\u0438\u0431\u043a\u0438 \u0432 \u043a\u0430\u0441\u0442\u043e\u043c\u043d\u044b\u0435 \u0438\u0441\u043a\u043b\u044e\u0447\u0435\u043d\u0438\u044f.", "PARAM": "endpoint (str) - \u042d\u043d\u0434\u043f\u043e\u0438\u043d\u0442.", @@ -14048,8 +14739,8 @@ "name": "_handle_network_error", "type": "Function", "tier": "STANDARD", - "start_line": 218, - "end_line": 230, + "start_line": 232, + "end_line": 244, "tags": { "PURPOSE": "(Helper) \u041f\u0440\u0435\u043e\u0431\u0440\u0430\u0437\u0443\u0435\u0442 \u0441\u0435\u0442\u0435\u0432\u044b\u0435 \u043e\u0448\u0438\u0431\u043a\u0438 \u0432 `NetworkError`.", "PARAM": "url (str) - URL.", @@ -14068,8 +14759,8 @@ "name": "upload_file", "type": "Function", "tier": "STANDARD", - "start_line": 232, - "end_line": 259, + "start_line": 246, + "end_line": 273, "tags": { "PURPOSE": "\u0417\u0430\u0433\u0440\u0443\u0436\u0430\u0435\u0442 \u0444\u0430\u0439\u043b \u043d\u0430 \u0441\u0435\u0440\u0432\u0435\u0440 \u0447\u0435\u0440\u0435\u0437 multipart/form-data.", "PARAM": "timeout (Optional[int]) - \u0422\u0430\u0439\u043c\u0430\u0443\u0442.", @@ -14090,8 +14781,8 @@ "name": "_perform_upload", "type": "Function", "tier": "STANDARD", - "start_line": 261, - "end_line": 287, + "start_line": 275, + "end_line": 301, "tags": { "PURPOSE": "(Helper) \u0412\u044b\u043f\u043e\u043b\u043d\u044f\u0435\u0442 POST \u0437\u0430\u043f\u0440\u043e\u0441 \u0441 \u0444\u0430\u0439\u043b\u043e\u043c.", "PARAM": "timeout (Optional[int]) - \u0422\u0430\u0439\u043c\u0430\u0443\u0442.", @@ -14111,8 +14802,8 @@ "name": "fetch_paginated_count", "type": "Function", "tier": "STANDARD", - "start_line": 289, - "end_line": 301, + "start_line": 303, + "end_line": 315, "tags": { "PURPOSE": "\u041f\u043e\u043b\u0443\u0447\u0430\u0435\u0442 \u043e\u0431\u0449\u0435\u0435 \u043a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u043e \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u043e\u0432 \u0434\u043b\u044f \u043f\u0430\u0433\u0438\u043d\u0430\u0446\u0438\u0438.", "PARAM": "count_field (str) - \u041f\u043e\u043b\u0435 \u0441 \u043a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u043e\u043c.", @@ -14132,8 +14823,8 @@ "name": "fetch_paginated_data", "type": "Function", "tier": "STANDARD", - "start_line": 303, - "end_line": 322, + "start_line": 317, + "end_line": 336, "tags": { "PURPOSE": "\u0410\u0432\u0442\u043e\u043c\u0430\u0442\u0438\u0447\u0435\u0441\u043a\u0438 \u0441\u043e\u0431\u0438\u0440\u0430\u0435\u0442 \u0434\u0430\u043d\u043d\u044b\u0435 \u0441\u043e \u0432\u0441\u0435\u0445 \u0441\u0442\u0440\u0430\u043d\u0438\u0446 \u043f\u0430\u0433\u0438\u043d\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u043e\u0433\u043e \u044d\u043d\u0434\u043f\u043e\u0438\u043d\u0442\u0430.", "PARAM": "pagination_options (Dict[str, Any]) - \u041e\u043f\u0446\u0438\u0438 \u043f\u0430\u0433\u0438\u043d\u0430\u0446\u0438\u0438.", @@ -15413,12 +16104,179 @@ "score": 0.85 } }, + { + "name": "router", + "type": "Global", + "tier": "STANDARD", + "start_line": 17, + "end_line": 20, + "tags": { + "PURPOSE": "APIRouter instance for LLM routes." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "get_providers", + "type": "Function", + "tier": "STANDARD", + "start_line": 22, + "end_line": 48, + "tags": { + "PURPOSE": "Retrieve all LLM provider configurations.", + "PRE": "User is authenticated.", + "POST": "Returns list of LLMProviderConfig." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 22 + } + ], + "score": 0.9 + } + }, + { + "name": "create_provider", + "type": "Function", + "tier": "STANDARD", + "start_line": 50, + "end_line": 74, + "tags": { + "PURPOSE": "Create a new LLM provider configuration.", + "PRE": "User is authenticated and has admin permissions.", + "POST": "Returns the created LLMProviderConfig." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 50 + } + ], + "score": 0.9 + } + }, + { + "name": "update_provider", + "type": "Function", + "tier": "STANDARD", + "start_line": 76, + "end_line": 104, + "tags": { + "PURPOSE": "Update an existing LLM provider configuration.", + "PRE": "User is authenticated and has admin permissions.", + "POST": "Returns the updated LLMProviderConfig." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 76 + } + ], + "score": 0.9 + } + }, + { + "name": "delete_provider", + "type": "Function", + "tier": "STANDARD", + "start_line": 106, + "end_line": 123, + "tags": { + "PURPOSE": "Delete an LLM provider configuration.", + "PRE": "User is authenticated and has admin permissions.", + "POST": "Returns success status." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 106 + } + ], + "score": 0.9 + } + }, + { + "name": "test_connection", + "type": "Function", + "tier": "STANDARD", + "start_line": 125, + "end_line": 159, + "tags": { + "PURPOSE": "Test connection to an LLM provider.", + "PRE": "User is authenticated.", + "POST": "Returns success status and message." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 125 + } + ], + "score": 0.9 + } + }, + { + "name": "test_provider_config", + "type": "Function", + "tier": "STANDARD", + "start_line": 161, + "end_line": 189, + "tags": { + "PURPOSE": "Test connection with a provided configuration (not yet saved).", + "PRE": "User is authenticated.", + "POST": "Returns success status and message." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 161 + } + ], + "score": 0.9 + } + }, { "name": "backend.src.api.routes.git", "type": "Module", "tier": "STANDARD", "start_line": 1, - "end_line": 400, + "end_line": 455, "tags": { "SEMANTICS": "git, routes, api, fastapi, repository, deployment", "PURPOSE": "Provides FastAPI endpoints for Git integration operations.", @@ -15784,6 +16642,25 @@ "issues": [], "score": 1.0 } + }, + { + "name": "generate_commit_message", + "type": "Function", + "tier": "STANDARD", + "start_line": 400, + "end_line": 453, + "tags": { + "PURPOSE": "Generate a suggested commit message using LLM.", + "PRE": "Repository for `dashboard_id` is initialized.", + "POST": "Returns a suggested commit message string." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } } ], "compliance": { @@ -17531,7 +18408,7 @@ "type": "Module", "tier": "STANDARD", "start_line": 1, - "end_line": 194, + "end_line": 211, "tags": { "SEMANTICS": "api, router, tasks, create, list, get", "PURPOSE": "Defines the FastAPI router for task-related endpoints, allowing clients to create, list, and get the status of tasks.", @@ -17545,7 +18422,7 @@ "type": "Function", "tier": "STANDARD", "start_line": 27, - "end_line": 51, + "end_line": 68, "tags": { "PURPOSE": "Create and start a new task for a given plugin.", "PARAM": "task_manager (TaskManager) - The task manager instance.", @@ -17565,8 +18442,8 @@ "name": "list_tasks", "type": "Function", "tier": "STANDARD", - "start_line": 54, - "end_line": 75, + "start_line": 71, + "end_line": 92, "tags": { "PURPOSE": "Retrieve a list of tasks with pagination and optional status filter.", "PARAM": "task_manager (TaskManager) - The task manager instance.", @@ -17586,8 +18463,8 @@ "name": "get_task", "type": "Function", "tier": "STANDARD", - "start_line": 78, - "end_line": 98, + "start_line": 95, + "end_line": 115, "tags": { "PURPOSE": "Retrieve the details of a specific task.", "PARAM": "task_manager (TaskManager) - The task manager instance.", @@ -17607,8 +18484,8 @@ "name": "get_task_logs", "type": "Function", "tier": "STANDARD", - "start_line": 101, - "end_line": 121, + "start_line": 118, + "end_line": 138, "tags": { "PURPOSE": "Retrieve logs for a specific task.", "PARAM": "task_manager (TaskManager) - The task manager instance.", @@ -17628,8 +18505,8 @@ "name": "resolve_task", "type": "Function", "tier": "STANDARD", - "start_line": 124, - "end_line": 147, + "start_line": 141, + "end_line": 164, "tags": { "PURPOSE": "Resolve a task that is awaiting mapping.", "PARAM": "task_manager (TaskManager) - The task manager instance.", @@ -17649,8 +18526,8 @@ "name": "resume_task", "type": "Function", "tier": "STANDARD", - "start_line": 150, - "end_line": 173, + "start_line": 167, + "end_line": 190, "tags": { "PURPOSE": "Resume a task that is awaiting input (e.g., passwords).", "PARAM": "task_manager (TaskManager) - The task manager instance.", @@ -17670,8 +18547,8 @@ "name": "clear_tasks", "type": "Function", "tier": "STANDARD", - "start_line": 176, - "end_line": 193, + "start_line": 193, + "end_line": 210, "tags": { "PURPOSE": "Clear tasks matching the status filter.", "PARAM": "task_manager (TaskManager) - The task manager instance.", @@ -17699,6 +18576,106 @@ "score": 0.85 } }, + { + "name": "backend.src.models.llm", + "type": "Module", + "tier": "STANDARD", + "start_line": 1, + "end_line": 46, + "tags": { + "TIER": "STANDARD", + "SEMANTICS": "llm, models, sqlalchemy, persistence", + "PURPOSE": "SQLAlchemy models for LLM provider configuration and validation results.", + "LAYER": "Domain" + }, + "relations": [ + { + "type": "INHERITS_FROM", + "target": "backend.src.models.mapping.Base" + } + ], + "children": [ + { + "name": "LLMProvider", + "type": "Class", + "tier": "STANDARD", + "start_line": 16, + "end_line": 29, + "tags": { + "PURPOSE": "SQLAlchemy model for LLM provider configuration." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 16 + }, + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 16 + } + ], + "score": 0.7000000000000001 + } + }, + { + "name": "ValidationRecord", + "type": "Class", + "tier": "STANDARD", + "start_line": 31, + "end_line": 44, + "tags": { + "PURPOSE": "SQLAlchemy model for dashboard validation history." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 31 + }, + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 31 + } + ], + "score": 0.7000000000000001 + } + }, + { + "name": "generate_uuid", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 13, + "end_line": 13, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + } + ], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, { "name": "GitModels", "type": "Module", @@ -18322,6 +19299,454 @@ "score": 1.0 } }, + { + "name": "backend.src.services.llm_provider", + "type": "Module", + "tier": "STANDARD", + "start_line": 1, + "end_line": 117, + "tags": { + "TIER": "STANDARD", + "SEMANTICS": "service, llm, provider, encryption", + "PURPOSE": "Service for managing LLM provider configurations with encrypted API keys.", + "LAYER": "Domain" + }, + "relations": [ + { + "type": "DEPENDS_ON", + "target": "backend.src.core.database" + }, + { + "type": "DEPENDS_ON", + "target": "backend.src.models.llm" + } + ], + "children": [ + { + "name": "EncryptionManager", + "type": "Class", + "tier": "STANDARD", + "start_line": 17, + "end_line": 30, + "tags": { + "PURPOSE": "Handles encryption and decryption of sensitive data like API keys.", + "INVARIANT": "Uses a secret key from environment or a default one (fallback only for dev)." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 17 + }, + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 17 + } + ], + "score": 0.7000000000000001 + } + }, + { + "name": "LLMProviderService", + "type": "Class", + "tier": "STANDARD", + "start_line": 32, + "end_line": 115, + "tags": { + "PURPOSE": "Service to manage LLM provider lifecycle." + }, + "relations": [], + "children": [ + { + "name": "get_all_providers", + "type": "Function", + "tier": "STANDARD", + "start_line": 39, + "end_line": 44, + "tags": { + "PURPOSE": "Returns all configured LLM providers." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 39 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 39 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 39 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 39 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 39 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 39 + } + ], + "score": 0.26666666666666655 + } + }, + { + "name": "get_provider", + "type": "Function", + "tier": "STANDARD", + "start_line": 46, + "end_line": 51, + "tags": { + "PURPOSE": "Returns a single LLM provider by ID." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 46 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 46 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 46 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 46 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 46 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 46 + } + ], + "score": 0.26666666666666655 + } + }, + { + "name": "create_provider", + "type": "Function", + "tier": "STANDARD", + "start_line": 53, + "end_line": 70, + "tags": { + "PURPOSE": "Creates a new LLM provider with encrypted API key." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 53 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 53 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 53 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 53 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 53 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 53 + } + ], + "score": 0.26666666666666655 + } + }, + { + "name": "update_provider", + "type": "Function", + "tier": "STANDARD", + "start_line": 72, + "end_line": 91, + "tags": { + "PURPOSE": "Updates an existing LLM provider." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 72 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 72 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 72 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 72 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 72 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 72 + } + ], + "score": 0.26666666666666655 + } + }, + { + "name": "delete_provider", + "type": "Function", + "tier": "STANDARD", + "start_line": 93, + "end_line": 103, + "tags": { + "PURPOSE": "Deletes an LLM provider." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 93 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 93 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 93 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 93 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 93 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 93 + } + ], + "score": 0.26666666666666655 + } + }, + { + "name": "get_decrypted_api_key", + "type": "Function", + "tier": "STANDARD", + "start_line": 105, + "end_line": 113, + "tags": { + "PURPOSE": "Returns the decrypted API key for a provider." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 105 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 105 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 105 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 105 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 105 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 105 + } + ], + "score": 0.26666666666666655 + } + } + ], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 32 + }, + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 32 + } + ], + "score": 0.7000000000000001 + } + }, + { + "name": "__init__", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 21, + "end_line": 21, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "encrypt", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 25, + "end_line": 25, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "decrypt", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 28, + "end_line": 28, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "__init__", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 35, + "end_line": 35, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + } + ], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, { "name": "backend.src.services.auth_service", "type": "Module", @@ -20719,6 +22144,919 @@ "score": 0.85 } }, + { + "name": "schedule_dashboard_validation", + "type": "Function", + "tier": "STANDARD", + "start_line": 12, + "end_line": 54, + "tags": { + "PURPOSE": "Schedules a recurring dashboard validation task.", + "PARAM": "params (Dict[str, Any]) - Task parameters (environment_id, provider_id)." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 12 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 12 + } + ], + "score": 0.6666666666666667 + } + }, + { + "name": "scheduler", + "type": "Module", + "tier": "TRIVIAL", + "start_line": 1, + "end_line": 56, + "tags": { + "PURPOSE": "Auto-generated module for backend/src/plugins/llm_analysis/scheduler.py", + "TIER": "TRIVIAL", + "LAYER": "Unknown" + }, + "relations": [], + "children": [ + { + "name": "job_func", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 24, + "end_line": 24, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "_parse_cron", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 42, + "end_line": 42, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + } + ], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "LLMProviderType", + "type": "Class", + "tier": "STANDARD", + "start_line": 12, + "end_line": 18, + "tags": { + "PURPOSE": "Enum for supported LLM providers." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 12 + } + ], + "score": 0.8 + } + }, + { + "name": "LLMProviderConfig", + "type": "Class", + "tier": "STANDARD", + "start_line": 20, + "end_line": 30, + "tags": { + "PURPOSE": "Configuration for an LLM provider." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 20 + } + ], + "score": 0.8 + } + }, + { + "name": "ValidationStatus", + "type": "Class", + "tier": "STANDARD", + "start_line": 32, + "end_line": 38, + "tags": { + "PURPOSE": "Enum for dashboard validation status." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 32 + } + ], + "score": 0.8 + } + }, + { + "name": "DetectedIssue", + "type": "Class", + "tier": "STANDARD", + "start_line": 40, + "end_line": 46, + "tags": { + "PURPOSE": "Model for a single issue detected during validation." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 40 + } + ], + "score": 0.8 + } + }, + { + "name": "ValidationResult", + "type": "Class", + "tier": "STANDARD", + "start_line": 48, + "end_line": 59, + "tags": { + "PURPOSE": "Model for dashboard validation result." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 48 + } + ], + "score": 0.8 + } + }, + { + "name": "backend.src.plugins.llm_analysis.plugin", + "type": "Module", + "tier": "STANDARD", + "start_line": 1, + "end_line": 266, + "tags": { + "TIER": "STANDARD", + "SEMANTICS": "plugin, llm, analysis, documentation", + "PURPOSE": "Implements DashboardValidationPlugin and DocumentationPlugin.", + "LAYER": "Domain" + }, + "relations": [ + { + "type": "INHERITS_FROM", + "target": "backend.src.core.plugin_base.PluginBase" + } + ], + "children": [ + { + "name": "DashboardValidationPlugin", + "type": "Class", + "tier": "STANDARD", + "start_line": 20, + "end_line": 137, + "tags": { + "PURPOSE": "Plugin for automated dashboard health analysis using LLMs." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 20 + }, + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 20 + } + ], + "score": 0.7000000000000001 + } + }, + { + "name": "DocumentationPlugin", + "type": "Class", + "tier": "STANDARD", + "start_line": 139, + "end_line": 264, + "tags": { + "PURPOSE": "Plugin for automated dataset documentation using LLMs." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 139 + }, + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 139 + } + ], + "score": 0.7000000000000001 + } + }, + { + "name": "id", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 24, + "end_line": 24, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "name", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 28, + "end_line": 28, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "description", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 32, + "end_line": 32, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "version", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 36, + "end_line": 36, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "get_schema", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 39, + "end_line": 39, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "execute", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 50, + "end_line": 50, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "id", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 143, + "end_line": 143, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "name", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 147, + "end_line": 147, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "description", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 151, + "end_line": 151, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "version", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 155, + "end_line": 155, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "get_schema", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 158, + "end_line": 158, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "execute", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 169, + "end_line": 169, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + } + ], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "backend.src.plugins.llm_analysis.service", + "type": "Module", + "tier": "STANDARD", + "start_line": 1, + "end_line": null, + "tags": { + "TIER": "STANDARD", + "SEMANTICS": "service, llm, screenshot, playwright, openai", + "PURPOSE": "Services for LLM interaction and dashboard screenshots.", + "LAYER": "Domain" + }, + "relations": [ + { + "type": "DEPENDS_ON", + "target": "playwright" + }, + { + "type": "DEPENDS_ON", + "target": "openai" + }, + { + "type": "DEPENDS_ON", + "target": "tenacity" + } + ], + "children": [ + { + "name": "ScreenshotService", + "type": "Class", + "tier": "STANDARD", + "start_line": 19, + "end_line": null, + "tags": { + "PURPOSE": "Handles capturing screenshots of Superset dashboards.", + "PRE": "env is a valid Environment object." + }, + "relations": [], + "children": [ + { + "name": "capture_dashboard", + "type": "Function", + "tier": "STANDARD", + "start_line": 26, + "end_line": null, + "tags": { + "PURPOSE": "Captures a screenshot of a dashboard using Playwright.", + "PARAM": "output_path (str) - Path to save the screenshot.", + "RETURN": "bool - True if successful." + }, + "relations": [], + "children": [ + { + "name": "LLMClient", + "type": "Class", + "tier": "STANDARD", + "start_line": 136, + "end_line": null, + "tags": { + "PURPOSE": "Wrapper for LLM provider APIs." + }, + "relations": [], + "children": [ + { + "name": "get_json_completion", + "type": "Function", + "tier": "STANDARD", + "start_line": 146, + "end_line": null, + "tags": { + "PURPOSE": "Helper to handle LLM calls with JSON mode and fallback parsing.", + "PARAM": "messages (List[Dict]) - List of messages for the completion.", + "RETURN": "Dict[str, Any] - Parsed JSON response." + }, + "relations": [], + "children": [ + { + "name": "analyze_dashboard", + "type": "Function", + "tier": "STANDARD", + "start_line": 205, + "end_line": 258, + "tags": { + "PURPOSE": "Sends dashboard data to LLM for analysis." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 205 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 205 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 205 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 205 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 205 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 205 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 205 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 205 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 205 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 205 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 205 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 205 + } + ], + "score": 0.0 + } + } + ], + "compliance": { + "valid": false, + "issues": [ + { + "message": "Unclosed Anchor: [DEF:get_json_completion:Function] started at line 146", + "severity": "ERROR", + "line_number": 146 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 146 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 146 + }, + { + "message": "Unclosed Anchor: [DEF:get_json_completion:Function] started at line 146", + "severity": "ERROR", + "line_number": 146 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 146 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 146 + }, + { + "message": "Unclosed Anchor: [DEF:get_json_completion:Function] started at line 146", + "severity": "ERROR", + "line_number": 146 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 146 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 146 + }, + { + "message": "Unclosed Anchor: [DEF:get_json_completion:Function] started at line 146", + "severity": "ERROR", + "line_number": 146 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 146 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 146 + }, + { + "message": "Unclosed Anchor: [DEF:get_json_completion:Function] started at line 146", + "severity": "ERROR", + "line_number": 146 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 146 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 146 + } + ], + "score": 0.0 + } + } + ], + "compliance": { + "valid": false, + "issues": [ + { + "message": "Unclosed Anchor: [DEF:LLMClient:Class] started at line 136", + "severity": "ERROR", + "line_number": 136 + }, + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 136 + }, + { + "message": "Unclosed Anchor: [DEF:LLMClient:Class] started at line 136", + "severity": "ERROR", + "line_number": 136 + }, + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 136 + }, + { + "message": "Unclosed Anchor: [DEF:LLMClient:Class] started at line 136", + "severity": "ERROR", + "line_number": 136 + }, + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 136 + }, + { + "message": "Unclosed Anchor: [DEF:LLMClient:Class] started at line 136", + "severity": "ERROR", + "line_number": 136 + }, + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 136 + } + ], + "score": 0.0 + } + } + ], + "compliance": { + "valid": false, + "issues": [ + { + "message": "Unclosed Anchor: [DEF:capture_dashboard:Function] started at line 26", + "severity": "ERROR", + "line_number": 26 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 26 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 26 + }, + { + "message": "Unclosed Anchor: [DEF:capture_dashboard:Function] started at line 26", + "severity": "ERROR", + "line_number": 26 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 26 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 26 + }, + { + "message": "Unclosed Anchor: [DEF:capture_dashboard:Function] started at line 26", + "severity": "ERROR", + "line_number": 26 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 26 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 26 + } + ], + "score": 0.0 + } + } + ], + "compliance": { + "valid": false, + "issues": [ + { + "message": "Unclosed Anchor: [DEF:ScreenshotService:Class] started at line 19", + "severity": "ERROR", + "line_number": 19 + }, + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 19 + }, + { + "message": "Unclosed Anchor: [DEF:ScreenshotService:Class] started at line 19", + "severity": "ERROR", + "line_number": 19 + }, + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 19 + } + ], + "score": 0.0 + } + }, + { + "name": "__init__", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 23, + "end_line": 23, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, + { + "name": "__init__", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 139, + "end_line": 139, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + } + ], + "compliance": { + "valid": false, + "issues": [ + { + "message": "Unclosed Anchor: [DEF:backend.src.plugins.llm_analysis.service:Module] started at line 1", + "severity": "ERROR", + "line_number": 1 + } + ], + "score": 0.0 + } + }, { "name": "StoragePlugin", "type": "Module", @@ -21135,6 +23473,123 @@ "score": 0.85 } }, + { + "name": "GitLLMExtension", + "type": "Class", + "tier": "STANDARD", + "start_line": 14, + "end_line": null, + "tags": { + "PURPOSE": "Provides LLM capabilities to the Git plugin." + }, + "relations": [], + "children": [ + { + "name": "suggest_commit_message", + "type": "Function", + "tier": "STANDARD", + "start_line": 20, + "end_line": null, + "tags": { + "PURPOSE": "Generates a suggested commit message based on a diff and history.", + "PARAM": "history (List[str]) - Recent commit messages for context.", + "RETURN": "str - The suggested commit message." + }, + "relations": [], + "children": [], + "compliance": { + "valid": false, + "issues": [ + { + "message": "Unclosed Anchor: [DEF:suggest_commit_message:Function] started at line 20", + "severity": "ERROR", + "line_number": 20 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 20 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 20 + }, + { + "message": "Unclosed Anchor: [DEF:suggest_commit_message:Function] started at line 20", + "severity": "ERROR", + "line_number": 20 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 20 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 20 + } + ], + "score": 0.0 + } + } + ], + "compliance": { + "valid": false, + "issues": [ + { + "message": "Unclosed Anchor: [DEF:GitLLMExtension:Class] started at line 14", + "severity": "ERROR", + "line_number": 14 + }, + { + "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 14 + } + ], + "score": 0.0 + } + }, + { + "name": "llm_extension", + "type": "Module", + "tier": "TRIVIAL", + "start_line": 1, + "end_line": 66, + "tags": { + "PURPOSE": "Auto-generated module for backend/src/plugins/git/llm_extension.py", + "TIER": "TRIVIAL", + "LAYER": "Unknown" + }, + "relations": [], + "children": [ + { + "name": "__init__", + "type": "Function", + "tier": "TRIVIAL", + "start_line": 17, + "end_line": 17, + "tags": { + "PURPOSE": "Auto-detected function (orphan)", + "TIER": "TRIVIAL" + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + } + ], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, { "name": "test_environment_model", "type": "Function", diff --git a/specs/017-llm-analysis-plugin/data-model.md b/specs/017-llm-analysis-plugin/data-model.md index 97229f13..9c4116e5 100644 --- a/specs/017-llm-analysis-plugin/data-model.md +++ b/specs/017-llm-analysis-plugin/data-model.md @@ -27,6 +27,7 @@ Stores the outcome of a dashboard validation task. | timestamp | DateTime | Yes | When the validation ran | | status | Enum | Yes | `PASS`, `WARN`, `FAIL` | | screenshot_path | String | No | Path to the captured screenshot (if stored) | +| screenshot_metadata | JSON | No | `{width: 1920, height: dynamic, tabs_processed: []}` | | issues | JSON | Yes | List of detected issues `[{severity, message, location}]` | | raw_response | Text | No | Full LLM response for debugging | diff --git a/specs/017-llm-analysis-plugin/plan.md b/specs/017-llm-analysis-plugin/plan.md index bbdbb651..c900719b 100644 --- a/specs/017-llm-analysis-plugin/plan.md +++ b/specs/017-llm-analysis-plugin/plan.md @@ -23,6 +23,25 @@ This feature implements two new plugins for the `ss-tools` platform: `DashboardV **Constraints**: Must integrate with existing `PluginBase` and `TaskManager`. Secure storage for API keys. **Scale/Scope**: Support for configurable LLM providers, scheduled tasks, and notification integration. +## Implementation Notes + +### Screenshot Capture Implementation +The screenshot service uses Playwright with Chrome DevTools Protocol (CDP) to avoid font loading timeouts in headless mode. Key implementation details: +- **Full Page Capture**: Uses CDP `Page.captureScreenshot` with `captureBeyondViewport: true` and `fromSurface: true` +- **Tab Switching**: Implements recursive tab switching to trigger lazy-loaded chart rendering on multi-tab dashboards +- **Authentication**: Uses direct UI login flow (navigating to `/login/` and filling credentials) instead of API cookie injection for better reliability +- **Resolution**: 1920px width with dynamic full page height calculation + +### Authentication Flow +The service authenticates via Playwright UI login rather than API authentication: +1. Navigate to `/login/` page +2. Fill username/password fields (supports multiple Superset versions) +3. Click login button +4. Verify successful redirect to `/superset/welcome/` +5. Navigate to dashboard with valid session + +This approach is more reliable than API-to-UI cookie injection which was causing 403 Forbidden errors. + ## Constitution Check *GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* @@ -60,7 +79,8 @@ backend/ │ │ ├── llm_analysis/ # New Plugin Directory │ │ │ ├── __init__.py │ │ │ ├── plugin.py # Implements DashboardValidationPlugin & DocumentationPlugin -│ │ │ ├── service.py # LLM interaction logic +│ │ │ ├── service.py # LLM interaction logic + ScreenshotService (CDP, tab switching) +│ │ │ ├── scheduler.py # Scheduled validation task handler │ │ │ └── models.py # Pydantic models for LLM config/results │ │ └── git/ # Existing Git Plugin │ │ └── ... # Update for commit message generation diff --git a/specs/017-llm-analysis-plugin/tasks.md b/specs/017-llm-analysis-plugin/tasks.md index c1f06cd6..4338f001 100644 --- a/specs/017-llm-analysis-plugin/tasks.md +++ b/specs/017-llm-analysis-plugin/tasks.md @@ -30,7 +30,7 @@ **Goal**: Implement core services and shared infrastructure. - [x] T008 Implement `LLMProviderService` in `backend/src/services/llm_provider.py` (CRUD for providers, AES-256 encryption) -- [x] T009 Implement `ScreenshotService` in `backend/src/plugins/llm_analysis/service.py` (Playwright + API strategies, fallback logic, min 1280x720px resolution) +- [x] T009 Implement `ScreenshotService` in `backend/src/plugins/llm_analysis/service.py` (Playwright + API strategies, fallback logic, 1920px width, full page height, CDP screenshot, recursive tab switching) - [x] T010 Implement `LLMClient` in `backend/src/plugins/llm_analysis/service.py` (OpenAI SDK wrapper, retry logic with tenacity, rate limit handling) - [x] T011 Create `backend/src/plugins/llm_analysis/plugin.py` with `PluginBase` implementation stubs - [x] T012 Define database schema updates for `LLMProviderConfig` in `backend/src/models/llm.py` (or appropriate location) @@ -43,7 +43,7 @@ - [x] T014 [US1] Implement `validate_dashboard` task logic in `backend/src/plugins/llm_analysis/plugin.py` - [x] T015 [US1] Implement log retrieval logic (fetch recent logs, limit 100 lines/24h) in `backend/src/plugins/llm_analysis/plugin.py` - [x] T016 [US1] Construct multimodal prompt (image + text) in `backend/src/plugins/llm_analysis/service.py` (implement PII/credential filtering) -- [x] T017 [US1] Implement result parsing and persistence (ValidationResult) in `backend/src/plugins/llm_analysis/plugin.py` (ensure JSON structure: status, issues, summary) +- [x] T017 [US1] Implement result parsing and persistence (ValidationResult) in `backend/src/plugins/llm_analysis/plugin.py` (ensure JSON structure: status, issues, summary, log results to task output with `[ANALYSIS_SUMMARY]` and `[ANALYSIS_ISSUE]` markers) - [x] T018 [US1] Add `validate` endpoint trigger in `backend/src/api/routes/tasks.py` (or reuse existing dispatch) - [x] T019 [US1] Implement notification dispatch (Email/Pulse) on failure in `backend/src/plugins/llm_analysis/plugin.py` (Summary + Link format) - [x] T020 [US1] Create `frontend/src/components/llm/ValidationReport.svelte` for viewing results diff --git a/specs/project_map.md b/specs/project_map.md index 124c8bab..9bf51fdb 100644 --- a/specs/project_map.md +++ b/specs/project_map.md @@ -247,6 +247,14 @@ - 📝 Fetches AD mappings and roles from the backend to populate the UI. - ƒ **handleCreateMapping** (`Function`) - 📝 Submits a new AD Group to Role mapping to the backend. +- 🧩 **LLMSettingsPage** (`Component`) + - 📝 Admin settings page for LLM provider configuration. + - 🏗️ Layer: UI +- 📦 **+page** (`Module`) `[TRIVIAL]` + - 📝 Auto-generated module for frontend/src/routes/admin/settings/llm/+page.svelte + - 🏗️ Layer: Unknown + - ƒ **fetchProviders** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) - 🧩 **MigrationDashboard** (`Component`) - 📝 Main dashboard for configuring and starting migrations. - 🏗️ Layer: Page @@ -492,10 +500,12 @@ - 📝 Displays a grid of dashboards with selection and pagination. - 🏗️ Layer: Component - 🔒 Invariant: Selected IDs must be a subset of available dashboards. - - 📥 Props: dashboards: DashboardMetadata[] , selectedIds: number[] + - 📥 Props: dashboards: DashboardMetadata[] , selectedIds: number[] , environmentId: string - ⚡ Events: selectionChanged - ➡️ WRITES_TO `t` - ⬅️ READS_FROM `t` + - ƒ **handleValidate** (`Function`) + - 📝 Triggers dashboard validation task. - ƒ **handleSort** (`Function`) - 📝 Toggles sort direction or changes sort column. - ƒ **handleSelectionChange** (`Function`) @@ -647,6 +657,13 @@ - 📝 Fetches environments and saved connections. - ƒ **handleRunMapper** (`Function`) - 📝 Triggers the MapperPlugin task. + - ƒ **handleGenerateDocs** (`Function`) + - 📝 Triggers the LLM Documentation task. +- 📦 **MapperTool** (`Module`) `[TRIVIAL]` + - 📝 Auto-generated module for frontend/src/components/tools/MapperTool.svelte + - 🏗️ Layer: Unknown + - ƒ **handleApplyDoc** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) - 🧩 **DebugTool** (`Component`) - 📝 UI component for system diagnostics and debugging API responses. - 🏗️ Layer: UI @@ -692,6 +709,8 @@ - 🏗️ Layer: Component - 📥 Props: dashboardId: any, show: any - ⚡ Events: commit + - ƒ **handleGenerateMessage** (`Function`) + - 📝 Generates a commit message using LLM. - ƒ **loadStatus** (`Function`) - 📝 Загружает текущий статус репозитория и diff. - ƒ **handleCommit** (`Function`) @@ -728,6 +747,41 @@ - 📝 Pushes local commits to the remote repository. - ƒ **handlePull** (`Function`) - 📝 Pulls changes from the remote repository. +- 🧩 **DocPreview** (`Component`) + - 📝 UI component for previewing generated dataset documentation before saving. + - 🏗️ Layer: UI + - 📥 Props: documentation: any, onSave: any, onCancel: any + - ➡️ WRITES_TO `t` + - ⬅️ READS_FROM `t` +- 📦 **DocPreview** (`Module`) `[TRIVIAL]` + - 📝 Auto-generated module for frontend/src/components/llm/DocPreview.svelte + - 🏗️ Layer: Unknown + - ƒ **handleSave** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) +- 🧩 **ProviderConfig** (`Component`) + - 📝 UI form for managing LLM provider configurations. + - 🏗️ Layer: UI + - 📥 Props: providers: any, onSave: any + - ➡️ WRITES_TO `t` + - ⬅️ READS_FROM `t` +- 📦 **ProviderConfig** (`Module`) `[TRIVIAL]` + - 📝 Auto-generated module for frontend/src/components/llm/ProviderConfig.svelte + - 🏗️ Layer: Unknown + - ƒ **resetForm** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **handleEdit** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **testConnection** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **handleSubmit** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **toggleActive** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) +- 📦 **ValidationReport** (`Module`) `[TRIVIAL]` + - 📝 Auto-generated module for frontend/src/components/llm/ValidationReport.svelte + - 🏗️ Layer: Unknown + - ƒ **getStatusColor** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) - 📦 **backend.delete_running_tasks** (`Module`) - 📝 Script to delete tasks with RUNNING status from the database. - 🏗️ Layer: Utility @@ -752,6 +806,8 @@ - 📝 Serves frontend static files or index.html for SPA routing. - ƒ **read_root** (`Function`) - 📝 A simple root endpoint to confirm that the API is running when frontend is missing. + - ƒ **network_error_handler** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) - 📦 **Dependencies** (`Module`) - 📝 Manages the creation and provision of shared application dependencies, such as the PluginLoader and TaskManager, to avoid circular imports. - 🏗️ Layer: Core @@ -941,6 +997,8 @@ - 🏗️ Layer: Core - 🔒 Invariant: A single engine instance is used for the entire application. - 🔗 DEPENDS_ON -> `sqlalchemy` + - 📦 **BASE_DIR** (`Variable`) + - 📝 Base directory for the backend (where .db files should reside). - 📦 **DATABASE_URL** (`Constant`) - 📝 URL for the main mappings database. - 📦 **TASKS_DATABASE_URL** (`Constant`) @@ -1328,6 +1386,20 @@ - 📝 Initiates the ADFS OIDC login flow. - ƒ **auth_callback_adfs** (`Function`) - 📝 Handles the callback from ADFS after successful authentication. +- 📦 **router** (`Global`) + - 📝 APIRouter instance for LLM routes. +- ƒ **get_providers** (`Function`) + - 📝 Retrieve all LLM provider configurations. +- ƒ **create_provider** (`Function`) + - 📝 Create a new LLM provider configuration. +- ƒ **update_provider** (`Function`) + - 📝 Update an existing LLM provider configuration. +- ƒ **delete_provider** (`Function`) + - 📝 Delete an LLM provider configuration. +- ƒ **test_connection** (`Function`) + - 📝 Test connection to an LLM provider. +- ƒ **test_provider_config** (`Function`) + - 📝 Test connection with a provided configuration (not yet saved). - 📦 **backend.src.api.routes.git** (`Module`) - 📝 Provides FastAPI endpoints for Git integration operations. - 🏗️ Layer: API @@ -1366,6 +1438,8 @@ - 📝 Get current Git status for a dashboard repository. - ƒ **get_repository_diff** (`Function`) - 📝 Get Git diff for a dashboard repository. + - ƒ **generate_commit_message** (`Function`) + - 📝 Generate a suggested commit message using LLM. - 📦 **ConnectionsRouter** (`Module`) - 📝 Defines the FastAPI router for managing external database connections. - 🏗️ Layer: UI (API) @@ -1546,6 +1620,15 @@ - 📝 Resume a task that is awaiting input (e.g., passwords). - ƒ **clear_tasks** (`Function`) - 📝 Clear tasks matching the status filter. +- 📦 **backend.src.models.llm** (`Module`) + - 📝 SQLAlchemy models for LLM provider configuration and validation results. + - 🏗️ Layer: Domain + - ℂ **LLMProvider** (`Class`) + - 📝 SQLAlchemy model for LLM provider configuration. + - ℂ **ValidationRecord** (`Class`) + - 📝 SQLAlchemy model for dashboard validation history. + - ƒ **generate_uuid** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) - 📦 **GitModels** (`Module`) - 📝 Git-specific SQLAlchemy models for configuration and repository tracking. - 🏗️ Layer: Model @@ -1608,6 +1691,36 @@ - ℂ **ADGroupMapping** (`Class`) - 📝 Maps an Active Directory group to a local System Role. - 🔗 DEPENDS_ON -> `Role` +- 📦 **backend.src.services.llm_provider** (`Module`) + - 📝 Service for managing LLM provider configurations with encrypted API keys. + - 🏗️ Layer: Domain + - 🔗 DEPENDS_ON -> `backend.src.core.database` + - 🔗 DEPENDS_ON -> `backend.src.models.llm` + - ℂ **EncryptionManager** (`Class`) + - 📝 Handles encryption and decryption of sensitive data like API keys. + - 🔒 Invariant: Uses a secret key from environment or a default one (fallback only for dev). + - ℂ **LLMProviderService** (`Class`) + - 📝 Service to manage LLM provider lifecycle. + - ƒ **get_all_providers** (`Function`) + - 📝 Returns all configured LLM providers. + - ƒ **get_provider** (`Function`) + - 📝 Returns a single LLM provider by ID. + - ƒ **create_provider** (`Function`) + - 📝 Creates a new LLM provider with encrypted API key. + - ƒ **update_provider** (`Function`) + - 📝 Updates an existing LLM provider. + - ƒ **delete_provider** (`Function`) + - 📝 Deletes an LLM provider. + - ƒ **get_decrypted_api_key** (`Function`) + - 📝 Returns the decrypted API key for a provider. + - ƒ **__init__** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **encrypt** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **decrypt** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **__init__** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) - 📦 **backend.src.services.auth_service** (`Module`) - 📝 Orchestrates authentication business logic. - 🏗️ Layer: Service @@ -1814,6 +1927,72 @@ - 📝 Executes the dashboard migration logic. - 📦 **MigrationPlugin.execute** (`Action`) - 📝 Execute the migration logic with proper task logging. +- ƒ **schedule_dashboard_validation** (`Function`) + - 📝 Schedules a recurring dashboard validation task. +- 📦 **scheduler** (`Module`) `[TRIVIAL]` + - 📝 Auto-generated module for backend/src/plugins/llm_analysis/scheduler.py + - 🏗️ Layer: Unknown + - ƒ **job_func** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **_parse_cron** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) +- ℂ **LLMProviderType** (`Class`) + - 📝 Enum for supported LLM providers. +- ℂ **LLMProviderConfig** (`Class`) + - 📝 Configuration for an LLM provider. +- ℂ **ValidationStatus** (`Class`) + - 📝 Enum for dashboard validation status. +- ℂ **DetectedIssue** (`Class`) + - 📝 Model for a single issue detected during validation. +- ℂ **ValidationResult** (`Class`) + - 📝 Model for dashboard validation result. +- 📦 **backend.src.plugins.llm_analysis.plugin** (`Module`) + - 📝 Implements DashboardValidationPlugin and DocumentationPlugin. + - 🏗️ Layer: Domain + - ℂ **DashboardValidationPlugin** (`Class`) + - 📝 Plugin for automated dashboard health analysis using LLMs. + - ℂ **DocumentationPlugin** (`Class`) + - 📝 Plugin for automated dataset documentation using LLMs. + - ƒ **id** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **name** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **description** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **version** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **get_schema** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **execute** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **id** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **name** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **description** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **version** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **get_schema** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **execute** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) +- 📦 **backend.src.plugins.llm_analysis.service** (`Module`) + - 📝 Services for LLM interaction and dashboard screenshots. + - 🏗️ Layer: Domain + - 🔗 DEPENDS_ON -> `playwright` + - 🔗 DEPENDS_ON -> `openai` + - 🔗 DEPENDS_ON -> `tenacity` + - ℂ **ScreenshotService** (`Class`) + - 📝 Handles capturing screenshots of Superset dashboards. + - ƒ **capture_dashboard** (`Function`) + - 📝 Captures a screenshot of a dashboard using Playwright. + - ℂ **LLMClient** (`Class`) + - 📝 Wrapper for LLM provider APIs. + - ƒ **__init__** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) + - ƒ **__init__** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) - 📦 **StoragePlugin** (`Module`) - 📝 Provides core filesystem operations for managing backups and repositories. - 🏗️ Layer: App @@ -1854,6 +2033,15 @@ - 📝 Deletes a file or directory from the specified category and path. - ƒ **get_file_path** (`Function`) - 📝 Returns the absolute path of a file for download. +- ℂ **GitLLMExtension** (`Class`) + - 📝 Provides LLM capabilities to the Git plugin. + - ƒ **suggest_commit_message** (`Function`) + - 📝 Generates a suggested commit message based on a diff and history. +- 📦 **llm_extension** (`Module`) `[TRIVIAL]` + - 📝 Auto-generated module for backend/src/plugins/git/llm_extension.py + - 🏗️ Layer: Unknown + - ƒ **__init__** (`Function`) `[TRIVIAL]` + - 📝 Auto-detected function (orphan) - ƒ **test_environment_model** (`Function`) - 📝 Tests that Environment model correctly stores values. - ƒ **test_belief_scope_logs_entry_action_exit** (`Function`)