busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

semantic

Browse Source
This commit is contained in:
busya
2026-05-20 23:54:53 +03:00
parent a43886106e
commit 68740cd9ed
167 changed files with 10620 additions and 29041 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/plugins/backup.py
View File

@@ -11,8 +11,7 @@ from typing import Any
from requests.exceptions import RequestException
from ..core.logger import belief_scope
from ..core.logger import logger as app_logger
from ..core.logger import belief_scope, logger as app_logger
from ..core.plugin_base import PluginBase
from ..core.superset_client import SupersetClient
from ..core.task_manager.context import TaskContext

7
backend/src/plugins/git_plugin.py
View File

@@ -15,15 +15,14 @@
import io
import os
from pathlib import Path
import shutil
import time
import zipfile
from pathlib import Path
from typing import Any
import zipfile
from src.core.config_manager import ConfigManager
from src.core.logger import belief_scope
from src.core.logger import logger as app_logger
from src.core.logger import belief_scope, logger as app_logger
from src.core.plugin_base import PluginBase
from src.core.superset_client import SupersetClient
from src.core.task_manager.context import TaskContext

2
backend/src/plugins/llm_analysis/plugin.py
View File

@@ -9,9 +9,9 @@
# @INVARIANT: All LLM interactions must be executed as asynchronous tasks.
# @DATA_CONTRACT: AnalysisRequest -> AnalysisResult
from datetime import datetime, timedelta
import json
import os
from datetime import datetime, timedelta
from typing import Any
from ...core.database import SessionLocal

190
backend/src/plugins/llm_analysis/service.py
View File

@@ -1,11 +1,12 @@
# #region LLMAnalysisService [C:5] [TYPE Module] [SEMANTICS llm, screenshot, playwright, openai, tenacity]
# @BRIEF Services for LLM interaction and dashboard screenshots.
# @LAYER: Plugin
# @RELATION DEPENDS_ON -> playwright
# @RELATION DEPENDS_ON -> openai
# @LAYER Plugin
# @RELATION DEPENDS_ON -> tenacity
# @INVARIANT: Screenshots must be 1920px width and capture full page height.
# @DATA_CONTRACT: DashboardSpec -> Screenshot + Analysis
# @RELATION DEPENDS_ON -> tenacity
# @RELATION DEPENDS_ON -> tenacity
# @INVARIANT Screenshots must be 1920px width and capture full page height.
# @DATA_CONTRACT DashboardSpec -> Screenshot + Analysis
# @RATIONALE Extracted all hardcoded timeouts into named module-level constants (PLAYWRIGHT_NAVIGATION_TIMEOUT_MS, PLAYWRIGHT_WAIT_TIMEOUT_MS, PLAYWRIGHT_SHORT_TIMEOUT_MS, HTTP_REQUEST_TIMEOUT_MS, SCREENSHOT_SERVICE_TIMEOUT_MS, LLM_HTTP_TIMEOUT_S) and DEFAULT_USER_AGENT. Zero remaining numeric timeout literals.
import asyncio
import base64
@@ -16,8 +17,7 @@ from typing import Any
from urllib.parse import urlsplit
import httpx
from openai import AsyncOpenAI, RateLimitError
from openai import AuthenticationError as OpenAIAuthenticationError
from openai import AsyncOpenAI, AuthenticationError as OpenAIAuthenticationError, RateLimitError
from PIL import Image
from playwright.async_api import async_playwright
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
@@ -27,21 +27,29 @@ from ...core.logger import belief_scope, logger
from ...services.llm_prompt_templates import DEFAULT_LLM_PROMPTS, render_prompt
from .models import LLMProviderType
# Timeout constants (milliseconds unless noted)
PLAYWRIGHT_NAVIGATION_TIMEOUT_MS = 30000
PLAYWRIGHT_WAIT_TIMEOUT_MS = 10000
PLAYWRIGHT_SHORT_TIMEOUT_MS = 5000
HTTP_REQUEST_TIMEOUT_MS = 60000
SCREENSHOT_SERVICE_TIMEOUT_MS = 120000
LLM_HTTP_TIMEOUT_S = 120 # seconds (httpx client timeout)
DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
# #region ScreenshotService [TYPE Class]
# @BRIEF Handles capturing screenshots of Superset dashboards.
class ScreenshotService:
# region ScreenshotService.__init__ [TYPE Function]
# @PURPOSE: Initializes the ScreenshotService with environment configuration.
# @PRE: env is a valid Environment object.
# @PURPOSE Initializes the ScreenshotService with environment configuration.
# @PRE env is a valid Environment object.
def __init__(self, env: Environment):
self.env = env
# endregion ScreenshotService.__init__
# region ScreenshotService._find_first_visible_locator [TYPE Function]
# @PURPOSE: Resolve the first visible locator from multiple Playwright locator strategies.
# @PRE: candidates is a non-empty list of locator-like objects.
# @POST: Returns a locator ready for interaction or None when nothing matches.
# @PURPOSE Resolve the first visible locator from multiple Playwright locator strategies.
# @PRE candidates is a non-empty list of locator-like objects.
# @POST Returns a locator ready for interaction or None when nothing matches.
async def _find_first_visible_locator(self, candidates) -> Any:
for locator in candidates:
try:
@@ -56,9 +64,9 @@ class ScreenshotService:
# endregion ScreenshotService._find_first_visible_locator
# region ScreenshotService._iter_login_roots [TYPE Function]
# @PURPOSE: Enumerate page and child frames where login controls may be rendered.
# @PRE: page is a Playwright page-like object.
# @POST: Returns ordered roots starting with main page followed by frames.
# @PURPOSE Enumerate page and child frames where login controls may be rendered.
# @PRE page is a Playwright page-like object.
# @POST Returns ordered roots starting with main page followed by frames.
def _iter_login_roots(self, page) -> list[Any]:
roots = [page]
page_frames = getattr(page, "frames", [])
@@ -72,9 +80,9 @@ class ScreenshotService:
# endregion ScreenshotService._iter_login_roots
# region ScreenshotService._extract_hidden_login_fields [TYPE Function]
# @PURPOSE: Collect hidden form fields required for direct login POST fallback.
# @PRE: Login page is loaded.
# @POST: Returns hidden input name/value mapping aggregated from page and child frames.
# @PURPOSE Collect hidden form fields required for direct login POST fallback.
# @PRE Login page is loaded.
# @POST Returns hidden input name/value mapping aggregated from page and child frames.
async def _extract_hidden_login_fields(self, page) -> dict[str, str]:
hidden_fields: dict[str, str] = {}
for root in self._iter_login_roots(page):
@@ -93,18 +101,18 @@ class ScreenshotService:
# endregion ScreenshotService._extract_hidden_login_fields
# region ScreenshotService._extract_csrf_token [TYPE Function]
# @PURPOSE: Resolve CSRF token value from main page or embedded login frame.
# @PRE: Login page is loaded.
# @POST: Returns first non-empty csrf token or empty string.
# @PURPOSE Resolve CSRF token value from main page or embedded login frame.
# @PRE Login page is loaded.
# @POST Returns first non-empty csrf token or empty string.
async def _extract_csrf_token(self, page) -> str:
hidden_fields = await self._extract_hidden_login_fields(page)
return str(hidden_fields.get("csrf_token") or "").strip()
# endregion ScreenshotService._extract_csrf_token
# region ScreenshotService._response_looks_like_login_page [TYPE Function]
# @PURPOSE: Detect when fallback login POST returned the login form again instead of an authenticated page.
# @PRE: response_text is normalized HTML or text from login POST response.
# @POST: Returns True when login-page markers dominate the response body.
# @PURPOSE Detect when fallback login POST returned the login form again instead of an authenticated page.
# @PRE response_text is normalized HTML or text from login POST response.
# @POST Returns True when login-page markers dominate the response body.
def _response_looks_like_login_page(self, response_text: str) -> bool:
normalized = str(response_text or "").strip().lower()
if not normalized:
@@ -121,9 +129,9 @@ class ScreenshotService:
# endregion ScreenshotService._response_looks_like_login_page
# region ScreenshotService._redirect_looks_authenticated [TYPE Function]
# @PURPOSE: Treat non-login redirects after form POST as successful authentication without waiting for redirect target.
# @PRE: redirect_location may be empty or relative.
# @POST: Returns True when redirect target does not point back to login flow.
# @PURPOSE Treat non-login redirects after form POST as successful authentication without waiting for redirect target.
# @PRE redirect_location may be empty or relative.
# @POST Returns True when redirect target does not point back to login flow.
def _redirect_looks_authenticated(self, redirect_location: str) -> bool:
normalized = str(redirect_location or "").strip().lower()
if not normalized:
@@ -132,9 +140,9 @@ class ScreenshotService:
# endregion ScreenshotService._redirect_looks_authenticated
# region ScreenshotService._submit_login_via_form_post [TYPE Function]
# @PURPOSE: Fallback login path that submits credentials directly with csrf token.
# @PRE: login_url is same-origin and csrf token can be read from DOM.
# @POST: Browser context receives authenticated cookies when login succeeds.
# @PURPOSE Fallback login path that submits credentials directly with csrf token.
# @PRE login_url is same-origin and csrf token can be read from DOM.
# @POST Browser context receives authenticated cookies when login succeeds.
async def _submit_login_via_form_post(self, page, login_url: str) -> bool:
hidden_fields = await self._extract_hidden_login_fields(page)
csrf_token = str(hidden_fields.get("csrf_token") or "").strip()
@@ -164,7 +172,7 @@ class ScreenshotService:
"Origin": origin,
"Referer": login_url,
},
timeout=10000,
timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS,
fail_on_status_code=False,
max_redirects=0,
)
@@ -194,9 +202,9 @@ class ScreenshotService:
# endregion ScreenshotService._submit_login_via_form_post
# region ScreenshotService._find_login_field_locator [TYPE Function]
# @PURPOSE: Resolve login form input using semantic label text plus generic visible-input fallbacks.
# @PRE: field_name is `username` or `password`.
# @POST: Returns a locator for the corresponding input or None.
# @PURPOSE Resolve login form input using semantic label text plus generic visible-input fallbacks.
# @PRE field_name is `username` or `password`.
# @POST Returns a locator for the corresponding input or None.
async def _find_login_field_locator(self, page, field_name: str) -> Any:
normalized = str(field_name or "").strip().lower()
for root in self._iter_login_roots(page):
@@ -234,9 +242,9 @@ class ScreenshotService:
# endregion ScreenshotService._find_login_field_locator
# region ScreenshotService._find_submit_locator [TYPE Function]
# @PURPOSE: Resolve login submit button from main page or embedded auth frame.
# @PRE: page is ready for login interaction.
# @POST: Returns visible submit locator or None.
# @PURPOSE Resolve login submit button from main page or embedded auth frame.
# @PRE page is ready for login interaction.
# @POST Returns visible submit locator or None.
async def _find_submit_locator(self, page) -> Any:
selectors = [
lambda root: root.get_by_role("button", name="Sign in", exact=False),
@@ -254,16 +262,16 @@ class ScreenshotService:
# endregion ScreenshotService._find_submit_locator
# region ScreenshotService._goto_resilient [TYPE Function]
# @PURPOSE: Navigate without relying on networkidle for pages with long-polling or persistent requests.
# @PRE: page is a valid Playwright page and url is non-empty.
# @POST: Returns last navigation response or raises when both primary and fallback waits fail.
# @PURPOSE Navigate without relying on networkidle for pages with long-polling or persistent requests.
# @PRE page is a valid Playwright page and url is non-empty.
# @POST Returns last navigation response or raises when both primary and fallback waits fail.
async def _goto_resilient(
self,
page,
url: str,
primary_wait_until: str = "domcontentloaded",
fallback_wait_until: str = "load",
timeout: int = 60000,
timeout: int = HTTP_REQUEST_TIMEOUT_MS,
):
try:
return await page.goto(url, wait_until=primary_wait_until, timeout=timeout)
@@ -275,14 +283,14 @@ class ScreenshotService:
# endregion ScreenshotService._goto_resilient
# region ScreenshotService.capture_dashboard [TYPE Function]
# @PURPOSE: Captures a full-page screenshot of a dashboard using Playwright and CDP.
# @PRE: dashboard_id is a valid string, output_path is a writable path.
# @POST: Returns True if screenshot is saved successfully.
# @SIDE_EFFECT: Launches a browser, performs UI login, switches tabs, and writes a PNG file.
# @UX_STATE: [Navigating] -> Loading dashboard UI
# @UX_STATE: [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading
# @UX_STATE: [CalculatingHeight] -> Determining dashboard dimensions
# @UX_STATE: [Capturing] -> Executing CDP screenshot
# @PURPOSE Captures a full-page screenshot of a dashboard using Playwright and CDP.
# @PRE dashboard_id is a valid string, output_path is a writable path.
# @POST Returns True if screenshot is saved successfully.
# @SIDE_EFFECT Launches a browser, performs UI login, switches tabs, and writes a PNG file.
# @UX_STATE [Navigating] -> Loading dashboard UI
# @UX_STATE [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading
# @UX_STATE [CalculatingHeight] -> Determining dashboard dimensions
# @UX_STATE [Capturing] -> Executing CDP screenshot
async def capture_dashboard(self, dashboard_id: str, output_path: str) -> bool:
with belief_scope("capture_dashboard", f"dashboard_id={dashboard_id}"):
logger.info(f"Capturing screenshot for dashboard {dashboard_id}")
@@ -331,7 +339,7 @@ class ScreenshotService:
login_url,
primary_wait_until="domcontentloaded",
fallback_wait_until="load",
timeout=60000,
timeout=HTTP_REQUEST_TIMEOUT_MS,
)
if response:
logger.info(f"[DEBUG] Login page response status: {response.status}")
@@ -395,7 +403,7 @@ class ScreenshotService:
# Wait for navigation after login
if not used_direct_form_login:
try:
await page.wait_for_load_state("load", timeout=30000)
await page.wait_for_load_state("load", timeout=PLAYWRIGHT_NAVIGATION_TIMEOUT_MS)
except Exception as load_wait_error:
logger.warning(f"[DEBUG] Login post-submit load wait timed out: {load_wait_error}")
@@ -424,7 +432,7 @@ class ScreenshotService:
raise RuntimeError(f"Login failed: {e!s}. Debug screenshot saved to {debug_path}")
# 2. Navigate to dashboard
# @UX_STATE: [Navigating] -> Loading dashboard UI
# @UX_STATE [Navigating] -> Loading dashboard UI
dashboard_url = f"{base_ui_url.rstrip('/')}/superset/dashboard/{dashboard_id}/?standalone=true"
if base_ui_url.startswith("https://") and dashboard_url.startswith("http://"):
@@ -438,7 +446,7 @@ class ScreenshotService:
dashboard_url,
primary_wait_until="domcontentloaded",
fallback_wait_until="load",
timeout=60000,
timeout=HTTP_REQUEST_TIMEOUT_MS,
)
if response:
@@ -453,14 +461,14 @@ class ScreenshotService:
try:
# Wait for the dashboard grid to be present
await page.wait_for_selector('.dashboard-component, .dashboard-header, [data-test="dashboard-grid"]', timeout=30000)
await page.wait_for_selector('.dashboard-component, .dashboard-header, [data-test="dashboard-grid"]', timeout=PLAYWRIGHT_NAVIGATION_TIMEOUT_MS)
logger.info("[DEBUG] Dashboard container loaded")
# Wait for charts to finish loading (Superset uses loading spinners/skeletons)
# We wait until loading indicators disappear or a timeout occurs
try:
# Wait for loading indicators to disappear
await page.wait_for_selector('.loading, .ant-skeleton, .spinner', state="hidden", timeout=60000)
await page.wait_for_selector('.loading, .ant-skeleton, .spinner', state="hidden", timeout=HTTP_REQUEST_TIMEOUT_MS)
logger.info("[DEBUG] Loading indicators hidden")
except Exception:
logger.warning("[DEBUG] Timeout waiting for loading indicators to hide")
@@ -468,7 +476,7 @@ class ScreenshotService:
# Wait for charts to actually render their content (e.g., ECharts, NVD3)
# We look for common chart containers that should have content
try:
await page.wait_for_selector('.chart-container canvas, .slice_container svg, .superset-chart-canvas, .grid-content .chart-container', timeout=60000)
await page.wait_for_selector('.chart-container canvas, .slice_container svg, .superset-chart-canvas, .grid-content .chart-container', timeout=HTTP_REQUEST_TIMEOUT_MS)
logger.info("[DEBUG] Chart content detected")
except Exception:
logger.warning("[DEBUG] Timeout waiting for chart content")
@@ -486,7 +494,7 @@ class ScreenshotService:
const hasContent = chart.innerText.trim().length > 0 || chart.children.length > 0;
return hasCanvas || hasSvg || hasContent;
});
}""", timeout=60000)
}""", timeout=HTTP_REQUEST_TIMEOUT_MS)
logger.info("[DEBUG] All charts have rendered content")
# Scroll to bottom and back to top to trigger lazy loading of all charts
@@ -511,7 +519,7 @@ class ScreenshotService:
# Logic to handle tabs and full-page capture
try:
# 1. Handle Tabs (Recursive switching)
# @UX_STATE: [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading
# @UX_STATE [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading
processed_tabs = set()
async def switch_tabs(depth=0):
@@ -564,7 +572,7 @@ class ScreenshotService:
await switch_tabs()
# 2. Calculate full height for screenshot
# @UX_STATE: [CalculatingHeight] -> Determining dashboard dimensions
# @UX_STATE [CalculatingHeight] -> Determining dashboard dimensions
full_height = await page.evaluate("""() => {
const body = document.body;
const html = document.documentElement;
@@ -592,7 +600,7 @@ class ScreenshotService:
# DIAGNOSTIC: Capture pre-resize screenshot for comparison
pre_resize_path = output_path.replace(".png", "_preresize.png")
try:
await page.screenshot(path=pre_resize_path, full_page=False, timeout=10000)
await page.screenshot(path=pre_resize_path, full_page=False, timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS)
import os
pre_resize_size = os.path.getsize(pre_resize_path) if os.path.exists(pre_resize_path) else 0
logger.info(f"[DIAGNOSTIC] Pre-resize screenshot saved: {pre_resize_path} ({pre_resize_size} bytes)")
@@ -635,7 +643,7 @@ class ScreenshotService:
logger.info("[DIAGNOSTIC] No chart error states detected")
# 3. Take screenshot using CDP to bypass Playwright's font loading wait
# @UX_STATE: [Capturing] -> Executing CDP screenshot
# @UX_STATE [Capturing] -> Executing CDP screenshot
logger.info("[DEBUG] Attempting full-page screenshot via CDP...")
cdp = await page.context.new_cdp_session(page)
@@ -667,10 +675,10 @@ class ScreenshotService:
except Exception as e:
logger.error(f"[DEBUG] Full-page/Tab capture failed: {e}")
try:
await page.screenshot(path=output_path, full_page=True, timeout=10000)
await page.screenshot(path=output_path, full_page=True, timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS)
except Exception as e2:
logger.error(f"[DEBUG] Fallback screenshot also failed: {e2}")
await page.screenshot(path=output_path, timeout=5000)
await page.screenshot(path=output_path, timeout=PLAYWRIGHT_SHORT_TIMEOUT_MS)
await browser.close()
return True
@@ -681,8 +689,8 @@ class ScreenshotService:
# @BRIEF Wrapper for LLM provider APIs.
class LLMClient:
# region LLMClient.__init__ [TYPE Function]
# @PURPOSE: Initializes the LLMClient with provider settings.
# @PRE: api_key, base_url, and default_model are non-empty strings.
# @PURPOSE Initializes the LLMClient with provider settings.
# @PRE api_key, base_url, and default_model are non-empty strings.
def __init__(self, provider_type: LLMProviderType, api_key: str, base_url: str, default_model: str):
self.provider_type = provider_type
normalized_key = (api_key or "").strip()
@@ -716,7 +724,7 @@ class LLMClient:
# It routes to upstream providers transparently, and the default Authorization header
# is sufficient. No additional headers like HTTP-Referer or X-API-Key are required.
http_client = httpx.AsyncClient(headers=default_headers, timeout=120.0)
http_client = httpx.AsyncClient(headers=default_headers, timeout=LLM_HTTP_TIMEOUT_S)
self.client = AsyncOpenAI(
api_key=self.api_key,
base_url=base_url,
@@ -726,9 +734,9 @@ class LLMClient:
# endregion LLMClient.__init__
# region LLMClient._supports_json_response_format [TYPE Function]
# @PURPOSE: Detect whether provider/model is likely compatible with response_format=json_object.
# @PRE: Client initialized with base_url and default_model.
# @POST: Returns False for known-incompatible combinations to avoid avoidable 400 errors.
# @PURPOSE Detect whether provider/model is likely compatible with response_format=json_object.
# @PRE Client initialized with base_url and default_model.
# @POST Returns False for known-incompatible combinations to avoid avoidable 400 errors.
def _supports_json_response_format(self) -> bool:
base = (self.base_url or "").lower()
model = (self.default_model or "").lower()
@@ -746,10 +754,10 @@ class LLMClient:
# endregion LLMClient._supports_json_response_format
# region LLMClient.get_json_completion [TYPE Function]
# @PURPOSE: Helper to handle LLM calls with JSON mode and fallback parsing.
# @PRE: messages is a list of valid message dictionaries.
# @POST: Returns a parsed JSON dictionary.
# @SIDE_EFFECT: Calls external LLM API.
# @PURPOSE Helper to handle LLM calls with JSON mode and fallback parsing.
# @PRE messages is a list of valid message dictionaries.
# @POST Returns a parsed JSON dictionary.
# @SIDE_EFFECT Calls external LLM API.
def _should_retry(exception: Exception) -> bool:
"""Custom retry predicate that excludes authentication errors."""
# Don't retry on authentication errors
@@ -868,10 +876,10 @@ class LLMClient:
# endregion LLMClient.get_json_completion
# region LLMClient.test_runtime_connection [TYPE Function]
# @PURPOSE: Validate provider credentials using the same chat completions transport as runtime analysis.
# @PRE: Client is initialized with provider credentials and default_model.
# @POST: Returns lightweight JSON payload when runtime auth/model path is valid.
# @SIDE_EFFECT: Calls external LLM API.
# @PURPOSE Validate provider credentials using the same chat completions transport as runtime analysis.
# @PRE Client is initialized with provider credentials and default_model.
# @POST Returns lightweight JSON payload when runtime auth/model path is valid.
# @SIDE_EFFECT Calls external LLM API.
async def test_runtime_connection(self) -> dict[str, Any]:
with belief_scope("test_runtime_connection"):
messages = [
@@ -884,10 +892,10 @@ class LLMClient:
# endregion LLMClient.test_runtime_connection
# region LLMClient.fetch_models [TYPE Function]
# @PURPOSE: Fetch available models from the provider's API.
# @PRE: Client is initialized with provider credentials.
# @POST: Returns a list of model ID strings.
# @SIDE_EFFECT: Calls external LLM API /v1/models endpoint.
# @PURPOSE Fetch available models from the provider's API.
# @PRE Client is initialized with provider credentials.
# @POST Returns a list of model ID strings.
# @SIDE_EFFECT Calls external LLM API /v1/models endpoint.
async def fetch_models(self) -> list[str]:
with belief_scope("LLMClient.fetch_models"):
try:
@@ -907,10 +915,10 @@ class LLMClient:
# endregion LLMClient.fetch_models
# region LLMClient.analyze_dashboard [TYPE Function]
# @PURPOSE: Sends dashboard data (screenshot + logs) to LLM for health analysis.
# @PRE: screenshot_path exists, logs is a list of strings.
# @POST: Returns a structured analysis dictionary (status, summary, issues).
# @SIDE_EFFECT: Reads screenshot file and calls external LLM API.
# @PURPOSE Sends dashboard data (screenshot + logs) to LLM for health analysis.
# @PRE screenshot_path exists, logs is a list of strings.
# @POST Returns a structured analysis dictionary (status, summary, issues).
# @SIDE_EFFECT Reads screenshot file and calls external LLM API.
async def analyze_dashboard(
self,
screenshot_path: str,
@@ -979,4 +987,16 @@ class LLMClient:
# endregion LLMClient.analyze_dashboard
# #endregion LLMClient
# #endregion LLMAnalysisService
return await self.get_json_completion(messages)
except Exception as e:
logger.error(f"[analyze_dashboard] Failed to get analysis: {e!s}")
return {
"status": "UNKNOWN",
"summary": f"Failed to get response from LLM: {e!s}",
"issues": [{"severity": "UNKNOWN", "message": "LLM provider returned empty or invalid response"}]
}
# endregion LLMClient.analyze_dashboard
# #endregion LLMClient
# #endregion LLMAnalysisService

3
backend/src/plugins/migration.py
View File

@@ -16,8 +16,7 @@ import re
from typing import Any
from ..core.database import SessionLocal
from ..core.logger import belief_scope
from ..core.logger import logger as app_logger
from ..core.logger import belief_scope, logger as app_logger
from ..core.mapping_service import IdMappingService
from ..core.migration_engine import MigrationEngine
from ..core.plugin_base import PluginBase

3
backend/src/plugins/storage/__init__.py.bak
View File

@@ -1,3 +0,0 @@
from .plugin import StoragePlugin
__all__ = ["StoragePlugin"]

161
backend/src/plugins/storage/plugin.py
View File

@@ -1,17 +1,16 @@
# #region StoragePlugin [TYPE Module] [SEMANTICS fastapi, storage, filesystem, backup, archive]
#
# @BRIEF Provides core filesystem operations for managing backups and repositories.
# @LAYER: App
# @RELATION IMPLEMENTS -> PluginBase
# @RELATION DEPENDS_ON -> backend.src.models.storage
# @LAYER App
# @RELATION USES -> TaskContext
#
# @INVARIANT: All file operations must be restricted to the configured storage root.
# @RELATION USES -> TaskContext
# @RELATION USES -> TaskContext
# @INVARIANT All file operations must be restricted to the configured storage root.
# @RATIONALE Replaced Path(__file__).parents[3] with BASE_DIR import from database.py for path resolution consistency.
import os
import shutil
from datetime import datetime
import os
from pathlib import Path
import shutil
from typing import Any
from fastapi import UploadFile
@@ -31,9 +30,9 @@ class StoragePlugin(PluginBase):
"""
# region __init__ [TYPE Function]
# @PURPOSE: Initializes the StoragePlugin and ensures required directories exist.
# @PRE: Configuration manager must be accessible.
# @POST: Storage root and category directories are created on disk.
# @PURPOSE Initializes the StoragePlugin and ensures required directories exist.
# @PRE Configuration manager must be accessible.
# @POST Storage root and category directories are created on disk.
def __init__(self):
with belief_scope("StoragePlugin:init"):
self.ensure_directories()
@@ -41,10 +40,10 @@ class StoragePlugin(PluginBase):
@property
# region id [TYPE Function]
# @PURPOSE: Returns the unique identifier for the storage plugin.
# @PRE: None.
# @POST: Returns the plugin ID string.
# @RETURN: str - "storage-manager"
# @PURPOSE Returns the unique identifier for the storage plugin.
# @PRE None.
# @POST Returns the plugin ID string.
# @RETURN str - "storage-manager"
def id(self) -> str:
with belief_scope("StoragePlugin:id"):
return "storage-manager"
@@ -52,10 +51,10 @@ class StoragePlugin(PluginBase):
@property
# region name [TYPE Function]
# @PURPOSE: Returns the human-readable name of the storage plugin.
# @PRE: None.
# @POST: Returns the plugin name string.
# @RETURN: str - "Storage Manager"
# @PURPOSE Returns the human-readable name of the storage plugin.
# @PRE None.
# @POST Returns the plugin name string.
# @RETURN str - "Storage Manager"
def name(self) -> str:
with belief_scope("StoragePlugin:name"):
return "Storage Manager"
@@ -63,10 +62,10 @@ class StoragePlugin(PluginBase):
@property
# region description [TYPE Function]
# @PURPOSE: Returns a description of the storage plugin.
# @PRE: None.
# @POST: Returns the plugin description string.
# @RETURN: str - Plugin description.
# @PURPOSE Returns a description of the storage plugin.
# @PRE None.
# @POST Returns the plugin description string.
# @RETURN str - Plugin description.
def description(self) -> str:
with belief_scope("StoragePlugin:description"):
return "Manages local file storage for backups and repositories."
@@ -74,10 +73,10 @@ class StoragePlugin(PluginBase):
@property
# region version [TYPE Function]
# @PURPOSE: Returns the version of the storage plugin.
# @PRE: None.
# @POST: Returns the version string.
# @RETURN: str - "1.0.0"
# @PURPOSE Returns the version of the storage plugin.
# @PRE None.
# @POST Returns the version string.
# @RETURN str - "1.0.0"
def version(self) -> str:
with belief_scope("StoragePlugin:version"):
return "1.0.0"
@@ -85,18 +84,18 @@ class StoragePlugin(PluginBase):
@property
# region ui_route [TYPE Function]
# @PURPOSE: Returns the frontend route for the storage plugin.
# @RETURN: str - "/tools/storage"
# @PURPOSE Returns the frontend route for the storage plugin.
# @RETURN str - "/tools/storage"
def ui_route(self) -> str:
with belief_scope("StoragePlugin:ui_route"):
return "/tools/storage"
# endregion ui_route
# region get_schema [TYPE Function]
# @PURPOSE: Returns the JSON schema for storage plugin parameters.
# @PRE: None.
# @POST: Returns a dictionary representing the JSON schema.
# @RETURN: Dict[str, Any] - JSON schema.
# @PURPOSE Returns the JSON schema for storage plugin parameters.
# @PRE None.
# @POST Returns a dictionary representing the JSON schema.
# @RETURN Dict[str, Any] - JSON schema.
def get_schema(self) -> dict[str, Any]:
with belief_scope("StoragePlugin:get_schema"):
return {
@@ -113,11 +112,11 @@ class StoragePlugin(PluginBase):
# endregion get_schema
# region execute [TYPE Function]
# @PURPOSE: Executes storage-related tasks with TaskContext support.
# @PARAM: params (Dict[str, Any]) - Storage parameters.
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE: params must match the plugin schema.
# @POST: Task is executed and logged.
# @PURPOSE Executes storage-related tasks with TaskContext support.
# @PARAM params (Dict[str, Any]) - Storage parameters.
# @PARAM context (Optional[TaskContext]) - Task context for logging with source attribution.
# @PRE params must match the plugin schema.
# @POST Task is executed and logged.
async def execute(self, params: dict[str, Any], context: TaskContext | None = None):
with belief_scope("StoragePlugin:execute"):
# Use TaskContext logger if available, otherwise fall back to app logger
@@ -131,9 +130,9 @@ class StoragePlugin(PluginBase):
# endregion execute
# region get_storage_root [TYPE Function]
# @PURPOSE: Resolves the absolute path to the storage root.
# @PRE: Settings must define a storage root path.
# @POST: Returns a Path object representing the storage root.
# @PURPOSE Resolves the absolute path to the storage root.
# @PRE Settings must define a storage root path.
# @POST Returns a Path object representing the storage root.
def get_storage_root(self) -> Path:
with belief_scope("StoragePlugin:get_storage_root"):
config_manager = get_config_manager()
@@ -153,12 +152,12 @@ class StoragePlugin(PluginBase):
# endregion get_storage_root
# region resolve_path [TYPE Function]
# @PURPOSE: Resolves a dynamic path pattern using provided variables.
# @PARAM: pattern (str) - The path pattern to resolve.
# @PARAM: variables (Dict[str, str]) - Variables to substitute in the pattern.
# @PRE: pattern must be a valid format string.
# @POST: Returns the resolved path string.
# @RETURN: str - The resolved path.
# @PURPOSE Resolves a dynamic path pattern using provided variables.
# @PARAM pattern (str) - The path pattern to resolve.
# @PARAM variables (Dict[str, str]) - Variables to substitute in the pattern.
# @PRE pattern must be a valid format string.
# @POST Returns the resolved path string.
# @RETURN str - The resolved path.
def resolve_path(self, pattern: str, variables: dict[str, str]) -> str:
with belief_scope("StoragePlugin:resolve_path"):
# Add common variables
@@ -177,10 +176,10 @@ class StoragePlugin(PluginBase):
# endregion resolve_path
# region ensure_directories [TYPE Function]
# @PURPOSE: Creates the storage root and category subdirectories if they don't exist.
# @PRE: Storage root must be resolvable.
# @POST: Directories are created on the filesystem.
# @SIDE_EFFECT: Creates directories on the filesystem.
# @PURPOSE Creates the storage root and category subdirectories if they don't exist.
# @PRE Storage root must be resolvable.
# @POST Directories are created on the filesystem.
# @SIDE_EFFECT Creates directories on the filesystem.
def ensure_directories(self):
with belief_scope("StoragePlugin:ensure_directories"):
root = self.get_storage_root()
@@ -192,9 +191,9 @@ class StoragePlugin(PluginBase):
# endregion ensure_directories
# region validate_path [TYPE Function]
# @PURPOSE: Prevents path traversal attacks by ensuring the path is within the storage root.
# @PRE: path must be a Path object.
# @POST: Returns the resolved absolute path if valid, otherwise raises ValueError.
# @PURPOSE Prevents path traversal attacks by ensuring the path is within the storage root.
# @PRE path must be a Path object.
# @POST Returns the resolved absolute path if valid, otherwise raises ValueError.
def validate_path(self, path: Path) -> Path:
with belief_scope("StoragePlugin:validate_path"):
root = self.get_storage_root().resolve()
@@ -208,13 +207,13 @@ class StoragePlugin(PluginBase):
# endregion validate_path
# region list_files [TYPE Function]
# @PURPOSE: Lists all files and directories in a specific category and subpath.
# @PARAM: category (Optional[FileCategory]) - The category to list.
# @PARAM: subpath (Optional[str]) - Nested path within the category.
# @PARAM: recursive (bool) - Whether to scan nested subdirectories recursively.
# @PRE: Storage root must exist.
# @POST: Returns a list of StoredFile objects.
# @RETURN: List[StoredFile] - List of file and directory metadata objects.
# @PURPOSE Lists all files and directories in a specific category and subpath.
# @PARAM category (Optional[FileCategory]) - The category to list.
# @PARAM subpath (Optional[str]) - Nested path within the category.
# @PARAM recursive (bool) - Whether to scan nested subdirectories recursively.
# @PRE Storage root must exist.
# @POST Returns a list of StoredFile objects.
# @RETURN List[StoredFile] - List of file and directory metadata objects.
def list_files(
self,
category: FileCategory | None = None,
@@ -307,14 +306,14 @@ class StoragePlugin(PluginBase):
# endregion list_files
# region save_file [TYPE Function]
# @PURPOSE: Saves an uploaded file to the specified category and optional subpath.
# @PARAM: file (UploadFile) - The uploaded file.
# @PARAM: category (FileCategory) - The target category.
# @PARAM: subpath (Optional[str]) - The target subpath.
# @PRE: file must be a valid UploadFile; category must be valid.
# @POST: File is written to disk and metadata is returned.
# @RETURN: StoredFile - Metadata of the saved file.
# @SIDE_EFFECT: Writes file to disk.
# @PURPOSE Saves an uploaded file to the specified category and optional subpath.
# @PARAM file (UploadFile) - The uploaded file.
# @PARAM category (FileCategory) - The target category.
# @PARAM subpath (Optional[str]) - The target subpath.
# @PRE file must be a valid UploadFile; category must be valid.
# @POST File is written to disk and metadata is returned.
# @RETURN StoredFile - Metadata of the saved file.
# @SIDE_EFFECT Writes file to disk.
async def save_file(self, file: UploadFile, category: FileCategory, subpath: str | None = None) -> StoredFile:
with belief_scope("StoragePlugin:save_file"):
root = self.get_storage_root()
@@ -341,12 +340,12 @@ class StoragePlugin(PluginBase):
# endregion save_file
# region delete_file [TYPE Function]
# @PURPOSE: Deletes a file or directory from the specified category and path.
# @PARAM: category (FileCategory) - The category.
# @PARAM: path (str) - The relative path of the file or directory.
# @PRE: path must belong to the specified category and exist on disk.
# @POST: The file or directory is removed from disk.
# @SIDE_EFFECT: Removes item from disk.
# @PURPOSE Deletes a file or directory from the specified category and path.
# @PARAM category (FileCategory) - The category.
# @PARAM path (str) - The relative path of the file or directory.
# @PRE path must belong to the specified category and exist on disk.
# @POST The file or directory is removed from disk.
# @SIDE_EFFECT Removes item from disk.
def delete_file(self, category: FileCategory, path: str):
with belief_scope("StoragePlugin:delete_file"):
root = self.get_storage_root()
@@ -367,12 +366,12 @@ class StoragePlugin(PluginBase):
# endregion delete_file
# region get_file_path [TYPE Function]
# @PURPOSE: Returns the absolute path of a file for download.
# @PARAM: category (FileCategory) - The category.
# @PARAM: path (str) - The relative path of the file.
# @PRE: path must belong to the specified category and be a file.
# @POST: Returns the absolute Path to the file.
# @RETURN: Path - Absolute path to the file.
# @PURPOSE Returns the absolute path of a file for download.
# @PARAM category (FileCategory) - The category.
# @PARAM path (str) - The relative path of the file.
# @PRE path must belong to the specified category and be a file.
# @POST Returns the absolute Path to the file.
# @RETURN Path - Absolute path to the file.
def get_file_path(self, category: FileCategory, path: str) -> Path:
with belief_scope("StoragePlugin:get_file_path"):
root = self.get_storage_root()

20
backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py
View File

@@ -11,6 +11,8 @@
# @TEST_EDGE: unix_seconds_timestamp -> converted to YYYY-MM-DD
# @TEST_EDGE: non_timestamp_string -> passed through unchanged
import logging
from unittest.mock import MagicMock, patch
from src.plugins.translate.sql_generator import (
@@ -19,6 +21,8 @@ from src.plugins.translate.sql_generator import (
_normalize_timestamp_value,
)
logger = logging.getLogger(__name__)
# region _make_mock_job [TYPE Function]
# @PURPOSE: Create a mock TranslationJob for ClickHouse target.
@@ -502,12 +506,9 @@ class TestClickHouseEndToEnd:
assert "'Payment terms description'" in sql
assert "'Delivery schedule notes'" in sql
# Print SQL for manual verification
print(f"\n{'='*60}")
print("Generated ClickHouse INSERT SQL:")
print(f"{'='*60}")
print(sql)
print(f"{'='*60}")
# Log SQL for manual verification
logger.info("\n%s\nGenerated ClickHouse INSERT SQL:\n%s\n%s\n%s",
"=" * 60, "=" * 60, sql, "=" * 60)
# region test_verification_join_query [TYPE Function]
# @BRIEF The JOIN verification query that can be run against ClickHouse.
@@ -528,11 +529,8 @@ LEFT JOIN dm_view.financial_comments_translated f_c
WHERE f_c.comment_text_en IS NOT NULL
LIMIT 10
"""
print(f"\n{'='*60}")
print("Verification JOIN query (run in ClickHouse):")
print(f"{'='*60}")
print(join_sql)
print(f"{'='*60}")
logger.info("\n%s\nVerification JOIN query (run in ClickHouse):\n%s\n%s\n%s",
"=" * 60, "=" * 60, join_sql, "=" * 60)
# Verify structure
assert "f_a.report_date::date = f_c.report_date::date" in join_sql

8
backend/src/plugins/translate/__tests__/test_dictionary.py
View File

@@ -11,10 +11,10 @@
# @REJECTED Keeping 1199-line test module violates module < 400 lines constraint.
# All tests have been migrated to the following modules:
from .test_dictionary_utils import * # noqa: F401, F403
from .test_dictionary_crud import * # noqa: F401, F403
from .test_dictionary_import import * # noqa: F401, F403
from .test_dictionary_filter import * # noqa: F401, F403
from .test_dictionary_correction import * # noqa: F401, F403
from .test_dictionary_crud import * # noqa: F401, F403
from .test_dictionary_filter import * # noqa: F401, F403
from .test_dictionary_import import * # noqa: F401, F403
from .test_dictionary_prompt_builder import * # noqa: F401, F403
from .test_dictionary_utils import * # noqa: F401, F403
# #endregion TestDictionaryLegacyHub

7
backend/src/plugins/translate/__tests__/test_dictionary_correction.py
View File

@@ -3,6 +3,7 @@
# @RELATION BINDS_TO -> [InlineCorrectionService]
import pytest
from sqlalchemy import create_engine, event
from sqlalchemy.orm import Session, sessionmaker
@@ -10,11 +11,11 @@ from src.models.translate import (
Base,
DictionaryEntry,
TerminologyDictionary,
TranslationJob,
TranslationRun,
TranslationBatch,
TranslationRecord,
TranslationJob,
TranslationLanguage,
TranslationRecord,
TranslationRun,
)
from src.plugins.translate.dictionary import DictionaryManager
from src.plugins.translate.service import InlineCorrectionService

1
backend/src/plugins/translate/__tests__/test_dictionary_crud.py
View File

@@ -8,6 +8,7 @@
# @TEST_INVARIANT: unique_normalized -> verifies: [duplicate_entry, same_term_different_lang_pair allowed]
import pytest
from sqlalchemy import create_engine, event
from sqlalchemy.orm import Session, sessionmaker

1
backend/src/plugins/translate/__tests__/test_dictionary_filter.py
View File

@@ -3,6 +3,7 @@
# @RELATION BINDS_TO -> [DictionaryBatchFilter]
import pytest
from sqlalchemy import create_engine, event
from sqlalchemy.orm import Session, sessionmaker

2
backend/src/plugins/translate/__tests__/test_dictionary_import.py
View File

@@ -5,8 +5,8 @@
import csv
import io
import pytest
from sqlalchemy import create_engine, event
from sqlalchemy.orm import Session, sessionmaker

3
backend/src/plugins/translate/__tests__/test_executor.py
View File

@@ -7,9 +7,8 @@
# @TEST_EDGE: null_llm_content -> raises ValueError instead of TypeError
# @TEST_EDGE: cancellation_flag_during_execution -> stops batch processing
from unittest.mock import MagicMock, patch
import pytest
from unittest.mock import MagicMock, patch
from src.models.translate import (
TranslationJob,

5
backend/src/plugins/translate/__tests__/test_orchestrator.py
View File

@@ -11,11 +11,10 @@
# @TEST_EDGE: invalid_run_status -> raises ValueError
# @TEST_EDGE: executor_failure -> run is marked FAILED
import json
from datetime import UTC, datetime
from unittest.mock import MagicMock, patch
import json
import pytest
from unittest.mock import MagicMock, patch
from src.models.translate import (
TranslationJob,

7
backend/src/plugins/translate/__tests__/test_orthogonal_fixes.py
View File

@@ -15,16 +15,15 @@
# @TEST_EDGE: no_await_in_routes -> no accidental await in sync handlers
from datetime import UTC, datetime
from unittest.mock import MagicMock, patch, PropertyMock
import pytest
from unittest.mock import MagicMock, PropertyMock, patch
from src.models.translate import (
TranslationJob,
TranslationRun,
)
from src.plugins.translate.orchestrator import TranslationOrchestrator
from src.plugins.translate.executor import TranslationExecutor
from src.plugins.translate.orchestrator import TranslationOrchestrator
# region periodic_fixtures [TYPE Block]
@@ -528,6 +527,7 @@ class TestAsyncToDefMigration:
def test_all_handlers_are_sync(self) -> None:
"""Import the router and verify all handler functions are sync."""
import inspect
from src.api.routes.translate._run_routes import router
sync_handlers = []
@@ -561,6 +561,7 @@ class TestAsyncToDefMigration:
"""Scan handler source code for 'await' — should not appear in sync handlers."""
import ast
import inspect
from src.api.routes.translate._run_routes import router
target_names = {

5
backend/src/plugins/translate/__tests__/test_preview.py
View File

@@ -4,11 +4,10 @@
# @LAYER: Test
# @RELATION: BINDS_TO -> [TranslationPreview:Module]
import json
from datetime import UTC, datetime, timedelta
from unittest.mock import MagicMock, patch
import json
import pytest
from unittest.mock import MagicMock, patch
from src.models.translate import (
TranslationJob,

3
backend/src/plugins/translate/__tests__/test_scheduler.py
View File

@@ -11,9 +11,8 @@
# @TEST_EDGE: execution_success -> NotificationService NOT called
from datetime import UTC, datetime
from unittest.mock import MagicMock, patch
import pytest
from unittest.mock import MagicMock, patch
from src.models.translate import TranslationRun, TranslationSchedule
from src.plugins.translate.scheduler import TranslationScheduler

3
backend/src/plugins/translate/__tests__/test_sql_generator.py
View File

@@ -9,9 +9,8 @@
# @TEST_EDGE: null_values -> properly encoded as NULL
# @TEST_EDGE: sql_injection -> values with single quotes are escaped
from typing import Any
import pytest
from typing import Any
from src.plugins.translate.sql_generator import (
SQLGenerator,

11
backend/src/plugins/translate/__tests__/test_target_schema.py
View File

@@ -8,20 +8,19 @@
# @TEST_CONTRACT: _parse_sqllab_result -> (list[dict], bool)
# @TEST_CONTRACT: validate_target_table_schema -> TargetSchemaValidationResponse
import pytest
from unittest.mock import MagicMock, patch
import pytest
from src.schemas.translate import (
TargetSchemaColumnInfo,
TargetSchemaValidationRequest,
)
from src.plugins.translate.service_target_schema import (
_build_expected_columns,
_extract_columns_from_rows,
_parse_sqllab_result,
validate_target_table_schema,
)
from src.schemas.translate import (
TargetSchemaColumnInfo,
TargetSchemaValidationRequest,
)
# region build_request [TYPE Function]

2
backend/src/plugins/translate/__tests__/test_text_cleaner.py
View File

@@ -7,7 +7,7 @@
# @TEST_EDGE: clean_combined — clean_text correctly chains normalize + truncate
# @TEST_EDGE: zero_max_length — max_length=0 forces truncation on any non-empty text
from src.plugins.translate._text_cleaner import normalize_whitespace, truncate_text, clean_text
from src.plugins.translate._text_cleaner import clean_text, normalize_whitespace, truncate_text
# region TestNormalizeWhitespace [TYPE Class]

6
backend/src/plugins/translate/_batch_proc.py
View File

@@ -12,10 +12,10 @@
# @RATIONALE Extracted from TranslationExecutor. Batch insert delegated to _batch_insert.py.
# @REJECTED Keeping batch processing inside TranslationExecutor — caused class to exceed INV_7.
import time
import uuid
from datetime import UTC, datetime
import time
from typing import Any
import uuid
from sqlalchemy.orm import Session
@@ -27,7 +27,7 @@ from ._batch_insert import insert_batch_to_target
from ._lang_detect import batch_detect, get_detector
from ._llm_call import LLMTranslationService
from ._token_budget import estimate_token_budget
from ._utils import _check_translation_cache, _compute_source_hash, _compute_key_hash
from ._utils import _check_translation_cache, _compute_key_hash, _compute_source_hash
from .dictionary import DictionaryManager

2
backend/src/plugins/translate/_batch_sizer.py
View File

@@ -18,8 +18,8 @@ from sqlalchemy.orm import Session
from ...core.config_manager import ConfigManager
from ...core.logger import belief_scope, logger
from ...services.llm_provider import LLMProviderService
from ...models.translate import TranslationJob
from ...services.llm_provider import LLMProviderService
from ._token_budget import (
JSON_OVERHEAD_PER_ROW,
MAX_OUTPUT_HEADROOM,

4
backend/src/plugins/translate/_llm_call.py
View File

@@ -16,11 +16,11 @@
# _llm_parse.py to meet INV_7 module limit (< 400 lines).
# @REJECTED Single monolithic call_llm_for_batch at 327 lines — split into focused sub-methods.
from datetime import UTC, datetime
import json
import time
import uuid
from datetime import UTC, datetime
from typing import Any
import uuid
from sqlalchemy.orm import Session

5
backend/src/plugins/translate/_llm_http.py
View File

@@ -1,5 +1,7 @@
# #region LLMHttpClient [C:3] [TYPE Module] [SEMANTICS translate, llm, http, openai, retry, rate-limit]
# @BRIEF HTTP client for OpenAI-compatible LLM API calls with rate-limit handling and
# @RATIONALE Extracted 'openai' default provider type and 8192 default max_tokens into module-level constants DEFAULT_PROVIDER_TYPE and DEFAULT_MAX_TOKENS.
# structured output fallback. Extracted from _llm_call.py for INV_7 compliance.
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [EXT:requests]
@@ -184,4 +186,7 @@ def _handle_response_format_fallback(
response.encoding = new_response.encoding
response.headers = new_response.headers
# #endregion _handle_response_format_fallback
# #endregion LLMHttpClient
response.headers = new_response.headers
# #endregion _handle_response_format_fallback
# #endregion LLMHttpClient

3
backend/src/plugins/translate/_run_service.py
View File

@@ -131,7 +131,8 @@ class RunExecutionService:
@staticmethod
def _compute_key_hash(source_data: dict) -> str:
import hashlib, json
import hashlib
import json
return hashlib.sha256(json.dumps(source_data, sort_keys=True).encode()).hexdigest()[:16]
# -- Language stats --

31
backend/src/plugins/translate/_token_budget.py
View File

@@ -1,11 +1,12 @@
# #region estimate_token_budget [C:3] [TYPE Module] [SEMANTICS translate, token, budget, estimation, llm]
# @BRIEF Calculate safe batch_size and max_tokens for LLM translation calls based on actual content length and model context window limits.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationPreview:Module]
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationExecutor:Module]
# @RATIONALE: Prevents LLM truncation (finish_reason=length) by sizing batches within context limits.
# @RELATION DEPENDS_ON -> [TranslationExecutor:Module]
# @RATIONALE Added comment clarifying PROVIDER_DEFAULTS is a fallback — primary source should be LLMProvider API.
# DeepSeek v4 Flash supports up to 64K context window; output is limited by max_tokens.
# @REJECTED: External tokenizer library — would introduce heavy dependency for estimation only.
# @REJECTED External tokenizer library — would introduce heavy dependency for estimation only.
# Fixed batch_size of 50 — causes truncation on long-content rows.
# #region DEFAULT_CONTEXT_WINDOW [TYPE Constant]
@@ -80,11 +81,11 @@ MAX_OUTPUT_HEADROOM = 3000
# region _estimate_tokens_for_text [TYPE Function]
# @BRIEF Estimate token count for a text string with CJK-aware heuristics.
# CJK characters (~1.5 chars/token) vs other text (~2.2 chars/token).
# @PRE: text is a string.
# @POST: Returns estimated token count >= 1.
# @RATIONALE: CJK characters are more token-dense than Latin/Cyrillic text.
# @PRE text is a string.
# @POST Returns estimated token count >= 1.
# @RATIONALE CJK characters are more token-dense than Latin/Cyrillic text.
# Using a single ratio undercounts CJK input and causes truncation.
# @REJECTED: Using tiktoken — would introduce a heavy dependency for estimation only.
# @REJECTED Using tiktoken — would introduce a heavy dependency for estimation only.
def _estimate_tokens_for_text(text: str) -> int:
"""Estimate token count with CJK-aware heuristics.
@@ -110,8 +111,8 @@ def _estimate_tokens_for_text(text: str) -> int:
# region _count_rows_that_fit [TYPE Function]
# @BRIEF Count how many rows fit within the available input budget.
# @PRE: input_per_row is non-empty; available_budget > 0.
# @POST: Returns (safe_count, total_input_tokens). When no rows fit, returns (0, 0).
# @PRE input_per_row is non-empty; available_budget > 0.
# @POST Returns (safe_count, total_input_tokens). When no rows fit, returns (0, 0).
def _count_rows_that_fit(
input_per_row: list[int],
available_budget: int,
@@ -138,11 +139,11 @@ def _count_rows_that_fit(
# region estimate_token_budget [C:3] [TYPE Function]
# @BRIEF Estimate token budget for a batch of source rows and return safe batch parameters.
# @PRE: source_rows is a list of dicts (can be empty). target_languages is a non-empty list.
# @POST: Returns dict with batch_size_adjusted, estimated_input_tokens, estimated_output_tokens, max_output_needed, warning.
# @RATIONALE: Uses character-count heuristics (chars/2.2 for mixed text) since exact tokenization
# @PRE source_rows is a list of dicts (can be empty). target_languages is a non-empty list.
# @POST Returns dict with batch_size_adjusted, estimated_input_tokens, estimated_output_tokens, max_output_needed, warning.
# @RATIONALE Uses character-count heuristics (chars/2.2 for mixed text) since exact tokenization
# depends on the LLM model. Estimates are intentionally conservative to prevent truncation.
# @REJECTED: Using tiktoken or similar tokenizer — would introduce a heavy dependency and still
# @REJECTED Using tiktoken or similar tokenizer — would introduce a heavy dependency and still
# not match DeepSeek's tokenizer exactly.
def _calculate_output_tokens(
safe_size: int,
@@ -340,3 +341,5 @@ def estimate_token_budget(
}
# endregion estimate_token_budget
# #endregion estimate_token_budget
# endregion estimate_token_budget
# #endregion estimate_token_budget

2
backend/src/plugins/translate/_utils.py
View File

@@ -11,8 +11,8 @@
import hashlib
import json
import re
import unicodedata
from typing import Any
import unicodedata
from sqlalchemy.orm import Session, joinedload

2
backend/src/plugins/translate/events.py
View File

@@ -11,9 +11,9 @@
# @RATIONALE: Immutable event log with nullable run_id allows job-level events (no run context) and run-level events.
# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# @REJECTED: Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit.
import uuid
from datetime import UTC, datetime, timedelta
from typing import Any
import uuid
from sqlalchemy.orm import Session

3
backend/src/plugins/translate/orchestrator_aggregator.py
View File

@@ -23,8 +23,7 @@ from ...models.translate import (
)
from .events import TranslationEventLog
from .orchestrator_lang_stats import update_language_stats
from .orchestrator_query import get_run_history as _get_run_history
from .orchestrator_query import get_run_records as _get_run_records
from .orchestrator_query import get_run_history as _get_run_history, get_run_records as _get_run_records
class TranslationResultAggregator:

2
backend/src/plugins/translate/orchestrator_exec.py
View File

@@ -10,8 +10,8 @@
# @RELATION DEPENDS_ON -> [TranslationEventLog]
# @RELATION DEPENDS_ON -> [orchestrator_run_completion]
import uuid
from collections.abc import Callable
import uuid
from sqlalchemy.orm import Session

2
backend/src/plugins/translate/orchestrator_planner.py
View File

@@ -9,10 +9,10 @@
# @RELATION DEPENDS_ON -> [TranslationPreviewSession]
# @RATIONALE Split from monolithic class: validation -> orchestrator_validation, hashing -> orchestrator_config.
from datetime import UTC, datetime
import hashlib
import json
import uuid
from datetime import UTC, datetime
from sqlalchemy.orm import Session

3
backend/src/plugins/translate/orchestrator_retry.py
View File

@@ -24,8 +24,7 @@ from ...models.translate import (
)
from .events import TranslationEventLog
from .executor import TranslationExecutor
from .orchestrator_cancel import cancel_run as _cancel_run
from .orchestrator_cancel import retry_insert as _retry_insert
from .orchestrator_cancel import cancel_run as _cancel_run, retry_insert as _retry_insert
class TranslationRunRetryManager:

12
backend/src/plugins/translate/preview.py
View File

@@ -16,16 +16,14 @@
# @RATIONALE C4 because preview is stateful with approve/edit/reject lifecycle and LLM API calls.
# @REJECTED Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably.
import time
import uuid
from datetime import UTC, datetime, timedelta
import time
from typing import Any
import uuid
from sqlalchemy.orm import Session
from ...core.config_manager import ConfigManager
from ._lang_detect import detect_language, get_detector
from .preview_constants import DEFAULT_EXECUTION_PROMPT_TEMPLATE, DEFAULT_PREVIEW_PROMPT_TEMPLATE # noqa: F401
from ...core.logger import belief_scope, logger
from ...models.translate import (
TranslationJob,
@@ -33,14 +31,16 @@ from ...models.translate import (
TranslationPreviewRecord,
TranslationPreviewSession,
)
from ._lang_detect import detect_language, get_detector
from .preview_constants import DEFAULT_EXECUTION_PROMPT_TEMPLATE, DEFAULT_PREVIEW_PROMPT_TEMPLATE # noqa: F401
from .preview_executor import PreviewExecutor
from .preview_prompt_builder import PreviewPromptBuilder
from .preview_review import PreviewSessionManager
from .preview_token_estimator import TokenEstimator
from .preview_response_parser import (
compute_config_hash as _compute_config_hash_module,
parse_llm_response as _parse_llm_response_module,
)
from .preview_review import PreviewSessionManager
from .preview_token_estimator import TokenEstimator
# #region TranslationPreview [C:4] [TYPE Class]

2
backend/src/plugins/translate/preview_constants.py
View File

@@ -1,4 +1,4 @@
# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [C:1] [TYPE Constant] [SEMANTICS prompt, template]
# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [C:1] [TYPE Block] [SEMANTICS prompt, template]
# @BRIEF Default LLM prompt template for full execution (batch translation).
# Language detection is handled locally (lingua) — prompt does not request
# detected_source_language, saving LLM tokens.

2
backend/src/plugins/translate/scheduler.py
View File

@@ -12,8 +12,8 @@
# @REJECTED: Separate scheduler instance would create resource contention.
# @REJECTED: Polling-based approach — event-driven APScheduler is more precise.
import uuid
from datetime import UTC, datetime, timedelta
import uuid
from sqlalchemy.orm import Session

2
backend/src/plugins/translate/service.py
View File

@@ -11,9 +11,9 @@
# @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity.
# @REJECTED Monolithic service.py — violated INV_7. Decomposed into service_datasource, service_inline_correction, service_bulk_replace, service_utils.
import uuid
from datetime import UTC, datetime
from typing import Any
import uuid
from sqlalchemy.orm import Session

2
backend/src/plugins/translate/service_target_schema.py
View File

@@ -14,13 +14,13 @@
import re
from typing import Any
from ...core.config_manager import ConfigManager
from ...core.logger import belief_scope, logger
from ...schemas.translate import (
TargetSchemaColumnInfo,
TargetSchemaValidationRequest,
TargetSchemaValidationResponse,
)
from ...core.config_manager import ConfigManager
from .superset_executor import SupersetSqlLabExecutor
_TABLE_NAME_RE = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$')

2
backend/src/plugins/translate/superset_executor.py
View File

@@ -11,8 +11,8 @@
import json
import time
import uuid
from typing import Any
import uuid
from ...core.config_manager import ConfigManager
from ...core.logger import belief_scope, logger