fix(agent-chat): streaming state leak, document parser magic bytes, HITL flow

### Bugfixes — Agent Chat 'Думаю' State Leak
- fix(agent-chat): loadHistory() now resets streamingState/idle + cancels stale
  submission — prevents 'Думаю' state leak across conversation switches
- fix(agent-chat): onDisconnected/onDisconnectedPermanent cascade to
  streamingState — prevents permanent hang on connection loss during stream
- fix(agent-chat):  guard on isLoadingHistory — prevents false commit
  of 'agent unavailable' fallback when switching conversations
- fix(agent-chat): remove race in _sendNow empty-response check vs Svelte
   microtask (duplicate logic removed,  handles correctly)
- fix(stream-processor): confirm_resolved now appends msg.text to partialText
  instead of dropping it

### Bugfixes — Backend PDF Upload
- fix(document-parser): _detect_format_by_magic() — reads file header magic
  bytes as fallback when Gradio loses filename
- fix(document-parser): improved name extraction — tries orig_name, path stem
- fix(document-parser): @RELATION AgentChatTypes -> AgentChat.Types

### HITL Flow & Agent Chat Improvements
- feat(agent): HITL resume confirm/deny with userId/userJwt/envId propagation
- feat(agent): confirm_required metadata fallback via aget_state() after
  'Event loop is closed' error during interrupt
- feat(agent): interrupt_before re-enabled via AGENT_CONFIRM_TOOLS env var
- feat(frontend): debug panel with connection/stream state monitoring
- feat(frontend): AgentChatModel constructor options + onBeforeSend callback
- feat(frontend): crypto.randomUUID() for local conversation ID on first send

### Backend Agent Refactoring
- refactor(agent): langgraph_setup — monkey-patch for PydanticSerializationError
- refactor(agent): tools.py — dual identity headers, expanded tool set
- refactor(agent): run.py — _find_free_port, Gradio server port fallback
- refactor(agent): app.py — file size validation, message truncation, HITL path

### Frontend
- feat(dashboard-hub): DashboardHubModel with filters, pagination, git actions
- feat(ui): DateRangeFilter component
- feat(i18n): new dashboard keys; cache tooltips fix
- fix(i18n): full run tooltips — cache is NOT ignored

### Semantic Protocol
- chore(agents): update all agents with canonical format
- chore(skills): sync semantics-core, semantics-contracts, molecular-cot-logging

### Housekeeping
- chore: remove stale semantic reports (10 files, Jan 2026)
- chore: update 033-gradio-agent-chat specs, contracts, UX, tasks, tests
- chore: add .agents/ directory (mirrors .opencode/ agent layouts)
- chore: update run.sh with DEV_MODE, port management
This commit is contained in:
2026-06-29 17:15:25 +03:00
parent 4fda63a8da
commit 12678c637b
115 changed files with 8948 additions and 1763 deletions

View File

@@ -5,12 +5,10 @@
# @POST Agent streams tokens via Gradio yield; audit logged via LoggingMiddleware.
# @SIDE_EFFECT Calls LLM, invokes tools via FastAPI REST, writes checkpoints to PostgreSQL.
# @RELATION DEPENDS_ON -> [AgentChat.Document.Parser]
# @RELATION DEPENDS_ON -> [AgentChat.Document.Parser]
# @RELATION DEPENDS_ON -> [AgentChat.Document.Parser]
# @RELATION DEPENDS_ON -> [AgentChat.Document.Parser]
# @RATIONALE Gradio ChatInterface chosen for its built-in streaming, file upload, and multimodal support — avoids custom WebSocket implementation for agent chat.
# @REJECTED Custom React chat frontend rejected — Gradio provides free authentication, session management, and mobile-responsive UI out of the box.
import asyncio
from collections.abc import AsyncGenerator
from datetime import datetime
import json
@@ -19,16 +17,16 @@ import uuid
import gradio as gr
import httpx
import jwt
from jose import JWTError
from langchain_core.exceptions import OutputParserException
from langchain_core.messages import HumanMessage
from langgraph.types import Command
from src.agent.context import set_user_jwt
from src.agent.document_parser import parse_upload
from src.agent.langgraph_setup import create_agent
from src.agent.middleware import log_tool_event
from src.agent.tools import get_all_tools
from src.agent.tools import get_all_tools, get_tools_for_query
from src.core.auth.jwt import decode_token
from src.core.cot_logger import log
JWT_SECRET = os.getenv("JWT_SECRET", "super-secret-key")
@@ -36,6 +34,8 @@ MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024 # 10 MB
# In-memory per-user lock (keyed by user_id)
_user_locks: dict[str, bool] = {}
# Per-conversation mutex for HITL resume (FR-026): keyed by conversation_id
_conv_locks: dict[str, asyncio.Event] = {}
# In-memory service JWT cache
_service_jwt_cache: dict[str, str] = {} # {token: expiry_timestamp}
@@ -56,6 +56,9 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
request: gr.Request,
conversation_id: str | None = None,
action: str | None = None,
user_id_str: str | None = None,
user_jwt_str_param: str | None = None,
env_id: str | None = None,
) -> AsyncGenerator[str]:
"""Handle incoming chat message. Streams tokens with structured metadata.
@@ -65,36 +68,54 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
request: gr.Request — may contain Authorization header with user JWT.
conversation_id: str — via additional_inputs (thread_id for checkpointer).
action: str — "confirm" | "deny" for HITL resume, None for normal messages.
user_id_str: str — user ID from frontend, used for conversation persistence.
user_jwt_str_param: str — user JWT from frontend for tool auth.
env_id: str — selected environment ID from top-bar selector.
"""
# ── Auth: extract user JWT if available —─
# Gradio runs behind Vite proxy which already handles auth.
# ── Auth: user JWT passed from frontend via additional_input —─
# @gradio/client does not forward Authorization headers,
# so we don't enforce JWT here. Tool calls use SERVICE_JWT (see tools.py).
# The JWT is only used for user-scoped features (per-user lock, conversation context).
auth_header = request.headers.get("authorization", "")
user_jwt_str = ""
if auth_header.startswith("Bearer "):
# so the frontend passes the JWT explicitly.
user_jwt_str = user_jwt_str_param or ""
if user_jwt_str:
try:
token = auth_header.split(" ")[1]
jwt.decode(token, JWT_SECRET, algorithms=["HS256"])
user_jwt_str = token
except jwt.InvalidTokenError:
pass # Ignore invalid JWTs — fall back to default context
decode_token(user_jwt_str)
except JWTError:
user_jwt_str = "" # Fall back to unauthenticated
# Store in ContextVar for @tool functions
set_user_jwt(user_jwt_str)
# ── Per-user lock (prevent concurrent sends per user) ──
user_id = _extract_user_id(user_jwt_str) if user_jwt_str else f"anon_{conversation_id or 'default'}"
# Priority: 1) user_id_str from frontend additional_input (correct identity),
# 2) extracted from JWT, 3) fallback to "admin"
user_id = user_id_str or (_extract_user_id(user_jwt_str) if user_jwt_str else "admin")
if _user_locks.get(user_id, False):
yield json.dumps({"metadata": {"type": "error", "code": "CONCURRENT_SEND"}})
return
_user_locks[user_id] = True
conv_id: str | None = None
try:
# ── Handle file upload ──
text = message.get("text", "") if isinstance(message, dict) else str(message)
files = message.get("files", []) if isinstance(message, dict) else []
if not text.strip() and not files:
return
# ── Truncate long messages per FR-028 ──
MAX_MSG_LENGTH = 100_000
if len(text) > MAX_MSG_LENGTH:
# Truncate at sentence boundary near the limit
truncated = text[:MAX_MSG_LENGTH]
last_sentence_end = max(
truncated.rfind('. '), truncated.rfind('! '), truncated.rfind('? '),
truncated.rfind('.\n'), truncated.rfind('!\n'), truncated.rfind('?\n'),
truncated.rfind('.\r'), truncated.rfind('!\r'), truncated.rfind('?\r'),
)
if last_sentence_end > MAX_MSG_LENGTH * 0.8:
text = text[:last_sentence_end + 1] + "\n[...truncated]"
else:
text = truncated + "\n[...truncated]"
if files:
# File size validation
@@ -112,24 +133,60 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
# ── HITL resume path ──
if action in ("confirm", "deny"):
async for chunk in _handle_resume(conversation_id, action):
conv_id = conversation_id
if conv_id:
# Wait for primary stream cleanup (FR-026): max 2s
lock = _conv_locks.get(conv_id)
if lock is not None:
try:
await asyncio.wait_for(lock.wait(), timeout=2.0)
except asyncio.TimeoutError:
yield json.dumps({
"content": "❌ Ошибка: предыдущий стрим ещё не завершён",
"metadata": {"type": "error", "code": "STREAM_CLEANUP_TIMEOUT"},
})
return
async for chunk in _handle_resume(conv_id, action, user_jwt_str, env_id):
yield chunk
# Save conversation after HITL resume
await _save_conversation(conversation_id or str(uuid.uuid4()), "HITL resume", user_id)
await _save_conversation(conv_id or str(uuid.uuid4()), "HITL resume", user_id)
return
# ── Normal send path ──
conv_id = conversation_id or str(uuid.uuid4())
agent = create_agent(get_all_tools())
# Acquire per-conversation lock for FR-026 (primary stream owns this conv)
_conv_locks[conv_id] = asyncio.Event()
# ── Pre-fetch dashboard data if query mentions dashboards ──
# Some LLMs (gemma) don't call tools even when instructed.
# Pre-fetch ensures dashboard data is available in context.
text_lower = text.lower()
prefetch_available = False
if any(kw in text_lower for kw in ["дашборд", "dashboard", "dashboards", "дашборды"]):
try:
dash_data = await _prefetch_dashboards(env_id or "")
if dash_data:
text += f"\n\n[PRE-FETCHED DATA — use this directly, do NOT call tools]\n{dash_data}\n[/PRE-FETCHED DATA]"
prefetch_available = True
except Exception:
pass # Pre-fetch is best-effort
agent_tools = get_tools_for_query(text, prefetch_available=prefetch_available)
agent = await create_agent(agent_tools, env_id)
config = {"configurable": {"thread_id": conv_id}}
# Collect assistant response during streaming
assistant_parts: list[str] = []
# Try up to 2 times: catch OutputParserException and retry with stricter prompt
max_attempts = 2
try:
for attempt in range(max_attempts):
try:
emitted_any = False
async for event in agent.astream_events(
{"messages": [HumanMessage(content=text)]},
config={"configurable": {"thread_id": conv_id}},
config=config,
version="v2",
):
kind = event.get("event")
@@ -141,6 +198,8 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
if kind == "on_chat_model_stream":
chunk = event["data"]["chunk"]
if hasattr(chunk, "content") and chunk.content:
emitted_any = True
assistant_parts.append(chunk.content)
yield json.dumps({
"content": chunk.content,
"metadata": {"type": "stream_token", "token": chunk.content},
@@ -148,6 +207,7 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
elif kind == "on_tool_start":
tool_name = event["name"]
emitted_any = True
yield json.dumps({
"content": f"🛠️ {tool_name}",
"metadata": {"type": "tool_start", "tool": tool_name, "input": event["data"].get("input", {})},
@@ -156,6 +216,7 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
elif kind == "on_tool_end":
tool_name = event["name"]
output = event["data"].get("output", "")
emitted_any = True
yield json.dumps({
"content": f"{tool_name}",
"metadata": {"type": "tool_end", "tool": tool_name, "output": {"result": str(output)[:500]}},
@@ -164,20 +225,34 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
elif kind == "on_tool_error":
tool_name = event["name"]
err = str(event["data"].get("error", "Unknown"))
emitted_any = True
yield json.dumps({
"content": f"{tool_name}{err}",
"metadata": {"type": "tool_error", "tool": tool_name, "error": err},
})
elif kind == "on_chain_end" and "interrupt" in event:
yield json.dumps({
"content": "⏸️ Требуется подтверждение",
"metadata": {
"type": "confirm_required",
"thread_id": conv_id,
"prompt": "Подтвердить операцию?",
},
})
state = await agent.aget_state(config)
if getattr(state, "next", None):
emitted_any = True
yield json.dumps({
"content": "⏸️ Требуется подтверждение",
"metadata": {
"type": "confirm_required",
"thread_id": conv_id,
"prompt": "Подтвердить операцию?",
},
})
return
elif not emitted_any:
yield json.dumps({
"content": "❌ Агент завершился без ответа.",
"metadata": {
"type": "error",
"code": "EMPTY_AGENT_RESPONSE",
"state_next": repr(getattr(state, "next", None)),
"state_tasks": repr(getattr(state, "tasks", None))[:500],
},
})
break # Stream ends — break out to save conversation
except OutputParserException as e:
@@ -191,37 +266,114 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
"metadata": {"type": "error", "code": "LLM_MALFORMED_OUTPUT", "detail": str(e)},
})
except Exception:
# Non-LLM-recoverable error (e.g. APIConnectionError).
# Save conversation (at least user message) before re-raising.
await _save_conversation(conv_id, text, user_id)
raise
except Exception as exc:
# RESILIENCE: LangGraph interrupt may cause "Event loop is closed" inside
# astream_events (httpx connection to LLM is cancelled during interrupt cleanup).
# The checkpoint with pending interrupt IS already saved — we detect it here
# and emit confirm_required instead of showing a generic error to the user.
try:
state = await agent.aget_state(config)
if getattr(state, "next", None):
emitted_any = True
yield json.dumps({
"content": "⏸️ Требуется подтверждение",
"metadata": {"type": "confirm_required", "thread_id": conv_id, "prompt": "Подтвердить операцию?"},
})
return
except Exception:
pass # Can't check state either — show original error
# Genuine non-recoverable error.
yield json.dumps({
"content": f"❌ Ошибка: {exc}",
"metadata": {"type": "error", "code": "PROCESSING_ERROR", "detail": str(exc)},
})
await _save_conversation(conv_id, text, user_id, assistant_text="".join(assistant_parts))
return
# ── Save conversation to DB via FastAPI REST ──
await _save_conversation(conv_id, text, user_id)
await _save_conversation(conv_id, text, user_id, assistant_text="".join(assistant_parts))
finally:
_user_locks[user_id] = False
# Release per-conversation lock (FR-026) so HITL resume can proceed
if conv_id and conv_id in _conv_locks:
_conv_locks[conv_id].set()
del _conv_locks[conv_id]
# #endregion AgentChat.GradioApp.Handler
async def _handle_resume(conversation_id: str, action: str) -> AsyncGenerator[str]:
async def _prefetch_dashboards(env_id: str) -> str:
"""Pre-fetch dashboard data so the LLM has it in context without needing to call a tool."""
try:
from src.agent.tools import _dual_auth_headers, FASTAPI_URL
async with httpx.AsyncClient(timeout=10) as client:
resp = await client.get(
f"{FASTAPI_URL}/api/dashboards",
params={"q": "", "env_id": env_id or ""},
headers=_dual_auth_headers(),
)
if resp.status_code != 200:
return ""
data = resp.json()
dashboards = data.get("dashboards", [])
if not dashboards:
return "No dashboards found."
limit = int(os.getenv("AGENT_PREFETCH_DASHBOARD_LIMIT", "25"))
total = len(dashboards)
lines = []
for db in dashboards[:limit]:
title = db.get("title", "Untitled")
dashboard_id = db.get("id") or db.get("dashboard_id")
modified = (db.get("last_modified", "") or "")[:10]
if modified:
lines.append(f"- {title} (id: {dashboard_id or 'n/a'}, modified: {modified})")
else:
lines.append(f"- {title} (id: {dashboard_id or 'n/a'})")
suffix = ""
if total > limit:
suffix = f"\n... {total - limit} more dashboards omitted. Ask for a narrower search if needed."
return f"Available dashboards in environment '{env_id or 'default'}' ({total} total):\n" + "\n".join(lines) + suffix
except Exception:
return ""
async def _handle_resume(
conversation_id: str, action: str,
user_jwt: str = "", env_id: str | None = None,
) -> AsyncGenerator[str]:
"""Resume from HITL checkpoint."""
agent = create_agent(get_all_tools())
set_user_jwt(user_jwt)
agent = await create_agent(get_all_tools(), env_id, interrupt_before=[])
if action == "confirm":
agent.invoke(
Command(resume={"action": "confirm"}),
config={"configurable": {"thread_id": conversation_id}},
)
config = {"configurable": {"thread_id": conversation_id}}
yield json.dumps({
"content": "▶️ Операция подтверждена",
"metadata": {"type": "confirm_resolved", "result": "confirmed"},
})
async for event in agent.astream_events(None, config=config, version="v2"):
kind = event.get("event")
if kind == "on_chat_model_stream":
chunk = event["data"]["chunk"]
if hasattr(chunk, "content") and chunk.content:
yield json.dumps({
"content": chunk.content,
"metadata": {"type": "stream_token", "token": chunk.content},
})
elif kind == "on_tool_start":
tool_name = event["name"]
yield json.dumps({
"content": f"🛠️ {tool_name}",
"metadata": {"type": "tool_start", "tool": tool_name, "input": event["data"].get("input", {})},
})
elif kind == "on_tool_end":
tool_name = event["name"]
output = event["data"].get("output", "")
yield json.dumps({
"content": f"{tool_name}",
"metadata": {"type": "tool_end", "tool": tool_name, "output": {"result": str(output)[:500]}},
})
elif action == "deny":
agent.invoke(
Command(resume={"action": "deny"}),
config={"configurable": {"thread_id": conversation_id}},
)
yield json.dumps({
"content": "⏹️ Операция отменена",
"metadata": {"type": "confirm_resolved", "result": "denied"},
@@ -230,7 +382,7 @@ async def _handle_resume(conversation_id: str, action: str) -> AsyncGenerator[st
def _extract_user_id(jwt_str: str) -> str:
try:
payload = jwt.decode(jwt_str, JWT_SECRET, algorithms=["HS256"])
payload = decode_token(jwt_str)
return payload.get("sub", payload.get("user_id", "unknown"))
except Exception:
return "unknown"
@@ -240,11 +392,12 @@ def _extract_user_id(jwt_str: str) -> str:
SAVE_API_URL = os.getenv("FASTAPI_URL", "http://localhost:8000") + "/api/agent/conversations/save"
async def _save_conversation(conv_id: str, user_text: str, user_id: str = "admin") -> None:
async def _save_conversation(conv_id: str, user_text: str, user_id: str = "admin", assistant_text: str = "") -> None:
"""Save conversation to DB via FastAPI REST.
Called after streaming completes. Creates or updates AgentConversation
and persists messages. Uses SERVICE_JWT for auth.
and persists messages (user message + optional assistant response).
Uses SERVICE_JWT for auth.
Failures are logged but not propagated.
"""
try:
@@ -253,19 +406,34 @@ async def _save_conversation(conv_id: str, user_text: str, user_id: str = "admin
if service_token:
headers["Authorization"] = f"Bearer {service_token}"
# Normalize user_id: anonymous Gradio sessions use "anon_" prefix
# which won't match the conversation list filter. Default to "admin".
if not user_id or user_id.startswith("anon_"):
user_id = "admin"
messages = [
{
"id": str(uuid.uuid4()),
"conversation_id": conv_id,
"role": "user",
"text": user_text.strip(),
"created_at": datetime.utcnow().isoformat(),
},
]
if assistant_text.strip():
messages.append({
"id": str(uuid.uuid4()),
"conversation_id": conv_id,
"role": "assistant",
"text": assistant_text.strip(),
"created_at": datetime.utcnow().isoformat(),
})
payload = {
"conversation_id": conv_id,
"title": user_text.strip()[:100] or "Agent conversation",
"user_id": user_id,
"messages": [
{
"id": str(uuid.uuid4()),
"conversation_id": conv_id,
"role": "user",
"text": user_text.strip(),
"created_at": datetime.utcnow().isoformat(),
}
],
"messages": messages,
}
async with httpx.AsyncClient(timeout=10) as client:
@@ -285,6 +453,9 @@ def create_chat_interface():
additional_inputs=[
gr.Textbox(label="conversation_id", visible=False),
gr.Textbox(label="action", visible=False),
gr.Textbox(label="user_id_str", visible=False),
gr.Textbox(label="user_jwt_str_param", visible=False),
gr.Textbox(label="env_id", visible=False),
],
examples=[
["Покажи дашборды", None, None],

View File

@@ -1,27 +1,30 @@
# backend/src/agent/context.py
# #region AgentChat.Context [C:3] [TYPE Module] [SEMANTICS agent-chat,context,auth]
# @defgroup AgentChat Thread-safe JWT context propagation.
# @SIDE_EFFECT Sets ContextVar before graph.invoke(), resets after.
# @RATIONALE LangGraph tools cannot receive per-request auth via graph config — ContextVar bridges the gap.
# @defgroup AgentChat JWT context propagation for LangGraph tools.
# @RATIONALE LangGraph tool execution may run in a different async context,
# preventing ContextVar from propagating. Module-level globals
# ensure the JWT is always accessible from any execution context.
# @NOTE NOT thread-safe — each Gradio agent process handles one request at a time
# (enforced by _user_locks in app.py).
from contextvars import ContextVar
_user_jwt: ContextVar[str | None] = ContextVar("_user_jwt", default=None)
_service_jwt: ContextVar[str | None] = ContextVar("_service_jwt", default=None)
_user_jwt: str = ""
_service_jwt: str = ""
def set_user_jwt(jwt: str) -> None:
_user_jwt.set(jwt)
global _user_jwt
_user_jwt = jwt
def get_user_jwt() -> str | None:
return _user_jwt.get()
def get_user_jwt() -> str:
return _user_jwt
def set_service_jwt(jwt: str) -> None:
_service_jwt.set(jwt)
global _service_jwt
_service_jwt = jwt
def get_service_jwt() -> str | None:
return _service_jwt.get()
def get_service_jwt() -> str:
return _service_jwt
# #endregion AgentChat.Context

View File

@@ -61,8 +61,27 @@ def parse_xlsx(file_path: str) -> str:
raise ParseError(f"Failed to parse XLSX: {e}") from e
def _detect_format_by_magic(path: str) -> str | None:
"""Detect file format by reading magic bytes. Returns extension or None."""
try:
with open(path, "rb") as f:
header = f.read(8)
except OSError:
return None
if header[:4] == b"%PDF":
return ".pdf"
if header[:4] == b"PK\x03\x04":
# ZIP-based: XLSX, DOCX, etc. Try XLSX first.
return ".xlsx"
if header[:1] in (b"{", b"["):
return ".json"
# CSV often starts with text characters; safe fallback
return None
def parse_upload(file_data) -> str:
"""Parse an uploaded file based on its extension.
"""Parse an uploaded file based on its extension with magic-byte fallback.
Args:
file_data: str (file path) or dict with "name" and "path"/"file_path" keys.
@@ -71,9 +90,19 @@ def parse_upload(file_data) -> str:
path = file_data
name = Path(path).name
else:
name = file_data.get("name", "")
path = file_data.get("path", file_data.get("file_path", ""))
name = file_data.get("name") or file_data.get("orig_name", "")
path = file_data.get("path") or file_data.get("file_path", "")
# Gradio sometimes sends files without 'name' key — fall back to path stem
if not name and path:
name = Path(path).name
ext = Path(name).suffix.lower()
# Double fallback: if name has no extension, try the physical file path
if not ext and path:
ext = Path(path).suffix.lower()
# Triple fallback: detect by magic bytes (Gradio ChatInterface loses filename)
if not ext and path:
ext = _detect_format_by_magic(path)
if ext == ".pdf":
return parse_pdf(path)
@@ -82,6 +111,19 @@ def parse_upload(file_data) -> str:
elif ext in (".json", ".csv", ".txt"):
with open(path, encoding="utf-8", errors="replace") as f:
return f.read(100_000) # truncate at ~100k chars
elif ext is None:
# Magic bytes detection returned None — unknown binary, try as text
try:
with open(path, encoding="utf-8", errors="replace") as f:
return f.read(100_000)
except Exception as e:
raise ParseError(
f"Could not detect file format for '{name}'. "
f"Supported: PDF, XLSX, JSON, CSV, TXT"
) from e
else:
raise ParseError(f"Unsupported format: {ext}. Supported: PDF, XLSX, JSON, CSV, TXT")
raise ParseError(
f"Unsupported format: '{ext}' (file: {name}). "
f"Supported: PDF, XLSX, JSON, CSV, TXT"
)
# #endregion AgentChat.Document.Parser

View File

@@ -5,28 +5,88 @@
# @POST Compiled StateGraph ready for astream_events().
# @SIDE_EFFECT Initializes checkpointer and message history tables on first call.
# @RELATION DEPENDS_ON -> [AgentChat.Tools]
# @RELATION DEPENDS_ON -> [AgentChat.Tools]
# @RELATION DEPENDS_ON -> [AgentChat.Tools]
# @RATIONALE LangGraph create_react_agent provides built-in tool calling + checkpointing + interrupt/resume.
# @REJECTED Using only environment variables for LLM config was rejected — FastAPI API-based config allows runtime switching without restart.
# RunnableWithMessageHistory wrapper is NOT used — PostgresSaver handles history natively.
import os
import time
import httpx
import psycopg
from langchain_openai import ChatOpenAI
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
from langgraph.prebuilt import create_react_agent
from psycopg.rows import dict_row
# ── Dangerous tool names — interrupt_before pauses execution at these nodes ──
# ── Dangerous tool names — interrupt_before pauses execution at these nodes ──
# These tools don't exist yet in the current tool set. When dangerous tools are
# added (deploy, migrate, commit, maintenance), add their names here.
DANGEROUS_TOOLS: list[str] = []
# ── Monkey-patch: OpenAI SDK for Pydantic BaseModel classes ──
# LangChain BaseTool objects carry an ``args_schema`` field that is a Pydantic
# BaseModel *class* reference (not an instance). When the OpenAI SDK recursively
# transforms the request body, it hits ``isinstance(data, pydantic.BaseModel)``
# which is True for both instances AND classes. It then calls ``model_dump()``
# on the class, which fails with:
#
# PydanticSerializationError: Unable to serialize unknown type: ModelMetaclass
#
# The fix: skip model_dump for classes, only dump instances.
# ── LLM config cache ────────────────────────────────────────────
import inspect as _inspect
import openai._utils._transform as _openai_transform
import pydantic as _pydantic
import pydantic_core as _pydantic_core
_original_transform = _openai_transform._async_transform_recursive
async def _patched_transform(data, *, annotation, inner_type=None):
if isinstance(data, _pydantic.BaseModel):
if _inspect.isclass(data):
# BaseModel CLASS (not instance) — skip model_dump
return data
# BaseModel INSTANCE — intercept PydanticSerializationError
try:
return await _original_transform(data, annotation=annotation, inner_type=inner_type)
except _pydantic_core.PydanticSerializationError:
print(f"[PATCH] Caught PydanticSerializationError on {type(data).__name__}")
# Fallback: dump with exclude of type-ref fields
serializable = {}
for field_name in data.model_fields_set:
val = getattr(data, field_name)
if isinstance(val, type) and issubclass(val, _pydantic.BaseModel):
serializable[field_name] = val.model_json_schema()
else:
serializable[field_name] = val
return serializable
return await _original_transform(data, annotation=annotation, inner_type=inner_type)
_openai_transform._async_transform_recursive = _patched_transform
# ── Postgres checkpointer (FR-004/FR-012/FR-027) ──
_CHECKPOINTER: AsyncPostgresSaver | None = None
_CHECKPOINTER_INIT = False
_CHECKPOINTER_CONN = None
async def init_checkpointer() -> None:
"""Initialize the PostgreSQL checkpointer (FR-004/FR-012/FR-027).
Called once at agent startup. Creates a persistent psycopg async connection
and passes it to AsyncPostgresSaver. Runs .setup() to create checkpoint tables.
Connection stays open for the lifetime of the agent process.
"""
global _CHECKPOINTER, _CHECKPOINTER_INIT, _CHECKPOINTER_CONN
if _CHECKPOINTER_INIT:
return
db_url = os.getenv("DATABASE_URL", "postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools")
# Convert SQLAlchemy-style URL to psycopg format
pg_url = db_url.replace("postgresql+psycopg2://", "postgres://").replace("postgresql://", "postgres://")
_CHECKPOINTER_CONN = await psycopg.AsyncConnection.connect(pg_url, autocommit=True, row_factory=dict_row)
_CHECKPOINTER = AsyncPostgresSaver(_CHECKPOINTER_CONN)
await _CHECKPOINTER.setup()
_CHECKPOINTER_INIT = True
# ── LLM config (no cache — fetched on each create_agent call) ──
_llm_config: dict | None = None
_llm_config_ttl: int = 300 # 5 min
def configure_from_api(llm_config: dict) -> None:
@@ -35,21 +95,62 @@ def configure_from_api(llm_config: dict) -> None:
_llm_config = llm_config
def create_agent(tools: list):
"""Create the LangGraph agent with checkpointer and message history.
async def _fetch_llm_config() -> dict | None:
"""Fetch LLM config from FastAPI.
Called on every create_agent() to pick up Admin UI changes immediately.
Falls back to cached config or env vars on failure.
"""
global _llm_config
try:
fastapi_url = os.getenv("FASTAPI_URL", "http://localhost:8000")
async with httpx.AsyncClient(timeout=5) as client:
resp = await client.get(f"{fastapi_url}/api/agent/llm-config")
if resp.status_code == 200:
config = resp.json()
if config.get("configured"):
_llm_config = config
return config
except Exception:
pass # Keep existing config on failure
return _llm_config
def _interrupt_before_from_env() -> list[str]:
"""Return LangGraph node names that must pause for HITL confirmation."""
if (os.getenv("AGENT_CONFIRM_TOOLS", "") or "").strip().lower() in ("true", "1", "yes"):
return ["tools"]
raw = os.getenv("AGENT_INTERRUPT_BEFORE", "") or ""
if not raw:
return []
return [name.strip() for name in raw.split(",") if name.strip()]
async def create_agent(
tools: list,
env_id: str | None = None,
interrupt_before: list[str] | None = None,
):
"""Create the LangGraph agent with PostgreSQL checkpointer and message history.
LLM configuration priority:
1. llm_config from configure_from_api() (fetched from FastAPI /api/agent/llm-config)
1. llm_config from FastAPI /api/agent/llm-config (fetched on every call)
2. Environment vars: LLM_API_KEY, LLM_BASE_URL, LLM_MODEL
3. Defaults: gpt-4o, https://api.openai.com/v1
Returns a RunnableWithMessageHistory wrapper ready for astream_events().
The graph is compiled with interrupt_before=DANGEROUS_TOOLS to enable HITL.
Returns a compiled StateGraph ready for astream_events().
interrupt_before is set from AGENT_CONFIRM_TOOLS (or AGENT_INTERRUPT_BEFORE env var)
to enable HITL guardrails — when pending tools are detected, the graph pauses before
executing them and yields confirm_required metadata to the frontend.
Checkpointer is AsyncPostgresSaver (survives container restarts).
"""
if _llm_config and _llm_config.get("configured"):
api_key = _llm_config["api_key"]
base_url = _llm_config.get("base_url") or "https://api.openai.com/v1"
model = _llm_config.get("default_model") or "gpt-4o-mini"
# Fetch fresh LLM config from FastAPI on every call
config = await _fetch_llm_config()
if config and config.get("configured"):
api_key = config["api_key"]
base_url = config.get("base_url") or "https://api.openai.com/v1"
model = config.get("default_model") or "gpt-4o-mini"
else:
api_key = os.getenv("LLM_API_KEY")
base_url = os.getenv("LLM_BASE_URL", "https://api.openai.com/v1")
@@ -60,17 +161,32 @@ def create_agent(tools: list):
base_url=base_url,
api_key=api_key,
temperature=0,
max_tokens=2048,
)
# Checkpointer — InMemorySaver for development (no persistence across restarts).
# TODO: Replace with AsyncPostgresSaver when langgraph-checkpoint-postgres supports it.
checkpointer = InMemorySaver()
# System prompt — env_id injected deterministically, not in user message
prompt = (
"You are a Superset Tools assistant. You have access to tools for searching "
"dashboards, checking health, listing environments, and checking task status. "
"If the data you need is already provided in the user message, use that directly "
"rather than calling tools. Only call tools when the data is not present."
)
if env_id:
prompt += f"\n\nCurrent environment: '{env_id}'. When calling tools that accept env_id, use this value."
# Checkpointer — AsyncPostgresSaver. Fallback to InMemory if Postgres init failed
if _CHECKPOINTER is not None:
checkpointer = _CHECKPOINTER
else:
checkpointer = InMemorySaver()
graph = create_react_agent(
model=llm,
tools=tools,
prompt=prompt,
version="v2",
checkpointer=checkpointer,
interrupt_before=DANGEROUS_TOOLS,
interrupt_before=_interrupt_before_from_env() if interrupt_before is None else interrupt_before,
)
return graph

View File

@@ -3,12 +3,10 @@
# @ingroup AgentChat
# @BRIEF Entrypoint for Gradio agent backend. Fetches LLM config from FastAPI on startup.
# @PRE FastAPI backend reachable at FASTAPI_URL. Service JWT available for auth.
# @POST Gradio agent running on configured port (auto-fallback to next free port if busy).
# @POST Gradio agent running on configured port.
# @SIDE_EFFECT Binds to a TCP port via Gradio launch.
# @RATIONALE _find_free_port() prevents port conflicts when a previous agent instance is still running
# without requiring manual cleanup or port-range environment variables.
# @REJECTED Failing hard on port-in-use was rejected — multiple restarts during development
# should not require manual port cleanup.
# @RATIONALE Gradio port must match the frontend proxy target. Optional fallback is available only
# when GRADIO_ALLOW_PORT_FALLBACK=true and an external proxy is updated separately.
import os
import socket
import httpx
@@ -61,9 +59,10 @@ def _fetch_llm_config() -> dict | None:
if __name__ == "__main__":
import asyncio
from src.agent.app import create_chat_interface
from src.agent.context import set_service_jwt
from src.agent.langgraph_setup import configure_from_api
from src.agent.langgraph_setup import configure_from_api, init_checkpointer
# Propagate SERVICE_JWT to ContextVar for tool calls
service_token = os.getenv("SERVICE_JWT", "")
@@ -75,19 +74,27 @@ if __name__ == "__main__":
if llm_config:
configure_from_api(llm_config)
# Find a free port — fallback if the configured port is already in use
# Initialize PostgreSQL checkpointer (FR-004/FR-012/FR-027)
asyncio.run(init_checkpointer())
# Bind the configured port. Falling back silently breaks the Vite/nginx proxy target.
configured_port = int(os.getenv("GRADIO_SERVER_PORT", "7860"))
try:
port = _find_free_port(configured_port)
if port != configured_port:
logger.warning("Port %d is in use, falling back to port %d", configured_port, port)
except OSError as e:
logger.error("Failed to find a free port: %s", e)
raise
allow_port_fallback = os.getenv("GRADIO_ALLOW_PORT_FALLBACK", "").strip().lower() in {"1", "true", "yes"}
if allow_port_fallback:
try:
port = _find_free_port(configured_port)
if port != configured_port:
logger.warning("Port %d is in use, falling back to port %d", configured_port, port)
except OSError as e:
logger.error("Failed to find a free port: %s", e)
raise
else:
port = configured_port
demo = create_chat_interface()
demo.launch(
server_name=os.getenv("GRADIO_SERVER_NAME", "0.0.0.0"),
server_port=port,
)
# #endregion AgentChat.Run
# #endregion AgentChat.Run

View File

@@ -5,7 +5,9 @@
# @REJECTED StructuredTool wrapping — native @tool is the single source of truth.
# @RATIONALE LangChain @tool decorator chosen over direct FastAPI calls for LangGraph compatibility — tools are auto-registered in the agent's tool-calling loop.
import json
import os
from typing import Any
import httpx
from langchain_core.tools import tool
@@ -13,26 +15,92 @@ from pydantic import BaseModel, Field
from src.agent.context import get_service_jwt, get_user_jwt
FASTAPI_URL = os.getenv("FASTAPI_URL", "http://backend:8000")
FASTAPI_URL = os.getenv("FASTAPI_URL", "http://localhost:8000")
TOOL_RESPONSE_LIMIT = 4000
def _dual_auth_headers() -> dict[str, str]:
"""Build dual-identity headers for tool→FastAPI calls.
Authorization: service JWT (authenticates the agent).
X-User-JWT: user JWT (authorizes the operation — RBAC).
Falls back to SERVICE_JWT env var if ContextVar is not set
(e.g., in Gradio's async context where ContextVars don't propagate).
"""Build dual-identity auth headers for tool→FastAPI calls per FR-007/FR-019.
Authorization: Bearer {service_jwt} — authenticates the agent container.
X-User-JWT: {user_jwt} — authorizes the operation as the end user.
Falls back to user JWT in Authorization if service JWT unavailable.
"""
svc_jwt = get_service_jwt() or os.getenv("SERVICE_JWT", "")
user_jwt = get_user_jwt() or ""
svc_jwt = get_service_jwt() or os.getenv("SERVICE_JWT", "")
headers = {}
if svc_jwt:
headers["Authorization"] = f"Bearer {svc_jwt}"
if user_jwt:
headers["X-User-JWT"] = user_jwt
if user_jwt:
headers["X-User-JWT"] = user_jwt
elif user_jwt:
headers["Authorization"] = f"Bearer {user_jwt}"
return headers
def _trim_response(text: str, limit: int = TOOL_RESPONSE_LIMIT) -> str:
if len(text) <= limit:
return text
return f"{text[:limit]}\n... response truncated ..."
async def _get(path: str, params: dict[str, Any] | None = None) -> httpx.Response:
async with httpx.AsyncClient() as client:
return await client.get(
f"{FASTAPI_URL}{path}",
params=params,
headers=_dual_auth_headers(),
)
async def _post(
path: str,
payload: dict[str, Any] | None = None,
params: dict[str, Any] | None = None,
) -> httpx.Response:
async with httpx.AsyncClient() as client:
return await client.post(
f"{FASTAPI_URL}{path}",
json=payload or {},
params=params,
headers=_dual_auth_headers(),
)
def _api_result(resp: httpx.Response, ok_statuses: set[int] | None = None) -> str:
ok_statuses = ok_statuses or {200, 201, 202}
if resp.status_code not in ok_statuses:
return f"Error {resp.status_code}: {resp.text}"
return _trim_response(resp.text)
async def _resolve_active_provider_id(provider_id: str | None = None) -> str | None:
if provider_id:
return provider_id
resp = await _get("/api/llm/providers")
if resp.status_code != 200:
return None
try:
providers = resp.json()
except json.JSONDecodeError:
return None
active = next((p for p in providers if p.get("is_active")), None)
if active:
return str(active.get("id"))
if providers:
return str(providers[0].get("id"))
return None
def _dashboard_ids(dashboard_id: int | str | None, dashboard_ids: list[int] | list[str] | None) -> list[int]:
values: list[int] = []
for raw in dashboard_ids or []:
values.append(int(raw))
if dashboard_id is not None and str(dashboard_id).strip():
values.append(int(dashboard_id))
return list(dict.fromkeys(values))
# ── Tool: search_dashboards ──
class SearchDashboardsInput(BaseModel):
query: str = Field(description="Search query for dashboard name")
@@ -48,31 +116,52 @@ class SearchDashboardsInput(BaseModel):
async def search_dashboards(query: str, env_id: str | None = None) -> str:
"""Search and list dashboards by name, with optional environment filter.
Pass env_id like 'prod', 'ss-dev', or 'ss-preprod' to filter by environment.
Returns a formatted list of dashboards with their details.
"""
params = {"q": query, "env_id": env_id or ""}
async with httpx.AsyncClient() as client:
resp = await client.get(
f"{FASTAPI_URL}/api/dashboards",
params=params,
headers=_dual_auth_headers(),
)
return resp.text
resp = await _get("/api/dashboards", params=params)
if resp.status_code != 200:
return f"Error {resp.status_code}: Unable to fetch dashboards. {resp.text}"
try:
data = resp.json()
dashboards = data.get("dashboards", [])
total = data.get("total", 0)
if total == 0:
return f"No dashboards found matching '{query}' in environment '{env_id or 'default'}'."
lines = [f"Found {total} dashboard(s) in environment '{env_id or 'default'}':"]
for db in dashboards:
title = db.get("title", "Untitled")
owners = ", ".join(db.get("owners", [])) or "N/A"
modified = db.get("last_modified", "")[:10] if db.get("last_modified") else "unknown"
lines.append(f" - {title}")
lines.append(f" Owners: {owners}")
lines.append(f" Last modified: {modified}")
return "\n".join(lines)
except (json.JSONDecodeError, KeyError) as e:
return f"Could not parse dashboards response: {e}"
# ── Tool: get_health_summary ──
class HealthSummaryInput(BaseModel):
env_id: str | None = Field(default=None, description="Environment ID (e.g. 'prod', 'ss-dev')")
# @ingroup AgentChat
# @PRE User authenticated via dual-identity JWT
# @POST Returns JSON string result from FastAPI
# @SIDE_EFFECT HTTP call to FastAPI backend
@tool
async def get_health_summary() -> str:
"""Get system health summary — dashboard validation status, recent failures."""
async with httpx.AsyncClient() as client:
resp = await client.get(
f"{FASTAPI_URL}/api/dashboards/health",
headers=_dual_auth_headers(),
)
return resp.text
@tool(args_schema=HealthSummaryInput)
async def get_health_summary(env_id: str | None = None) -> str:
"""Get system health summary — dashboard validation status, recent failures.
Pass env_id like 'prod', 'ss-dev', or 'ss-preprod' to filter by environment.
"""
resp = await _get("/api/health/summary", params={"environment_id": env_id or ""})
if resp.status_code != 200:
return f"Health check failed: HTTP {resp.status_code}"
return _trim_response(resp.text, 2000)
# ── Tool: list_environments ──
@@ -83,12 +172,8 @@ async def get_health_summary() -> str:
@tool
async def list_environments() -> str:
"""List configured deployment environments."""
async with httpx.AsyncClient() as client:
resp = await client.get(
f"{FASTAPI_URL}/api/settings/environments",
headers=_dual_auth_headers(),
)
return resp.text
resp = await _get("/api/settings/environments")
return _api_result(resp)
# ── Tool: get_task_status ──
@@ -99,20 +184,369 @@ async def list_environments() -> str:
@tool
async def get_task_status(task_id: str) -> str:
"""Check the status of a background task by its task_id."""
async with httpx.AsyncClient() as client:
resp = await client.get(
f"{FASTAPI_URL}/api/tasks/{task_id}",
headers=_dual_auth_headers(),
)
return resp.text
resp = await _get(f"/api/tasks/{task_id}")
return _api_result(resp)
@tool
async def show_capabilities() -> str:
"""Show all tools available to the agent chat."""
lines = ["Available LangChain tools:"]
for tool_obj in get_all_tools():
lines.append(f"- {tool_obj.name}: {tool_obj.description or ''}")
return "\n".join(lines)
@tool
async def list_llm_providers() -> str:
"""List configured LLM providers."""
resp = await _get("/api/llm/providers")
return _api_result(resp)
@tool
async def get_llm_status() -> str:
"""Check whether the LLM runtime is configured and usable."""
resp = await _get("/api/llm/status")
return _api_result(resp)
class RunBackupInput(BaseModel):
environment_id: str = Field(description="Environment ID for the backup")
dashboard_id: int | None = Field(default=None, description="Optional single dashboard ID")
dashboard_ids: list[int] | None = Field(default=None, description="Optional list of dashboard IDs")
@tool(args_schema=RunBackupInput)
async def run_backup(
environment_id: str,
dashboard_id: int | None = None,
dashboard_ids: list[int] | None = None,
) -> str:
"""Run a Superset backup for an environment, optionally scoped to dashboard IDs."""
params: dict[str, Any] = {"environment_id": environment_id}
ids = _dashboard_ids(dashboard_id, dashboard_ids)
if ids:
params["dashboard_ids"] = ids
resp = await _post("/api/tasks", {"plugin_id": "superset-backup", "params": params})
return _api_result(resp, {201})
class ExecuteMigrationInput(BaseModel):
source_env_id: str = Field(description="Source environment ID")
target_env_id: str = Field(description="Target environment ID")
selected_ids: list[int] = Field(description="Dashboard IDs to migrate")
replace_db_config: bool = Field(default=False, description="Replace DB configuration during migration")
fix_cross_filters: bool = Field(default=True, description="Fix cross filters during migration")
@tool(args_schema=ExecuteMigrationInput)
async def execute_migration(
source_env_id: str,
target_env_id: str,
selected_ids: list[int],
replace_db_config: bool = False,
fix_cross_filters: bool = True,
) -> str:
"""Execute dashboard migration between two environments."""
payload = {
"source_env_id": source_env_id,
"target_env_id": target_env_id,
"selected_ids": selected_ids,
"replace_db_config": replace_db_config,
"fix_cross_filters": fix_cross_filters,
}
resp = await _post("/api/migration/execute", payload)
return _api_result(resp)
class GitDashboardInput(BaseModel):
dashboard_ref: str = Field(description="Dashboard ID, slug, or title resolvable by the Git API")
env_id: str | None = Field(default=None, description="Optional environment ID used to resolve dashboard_ref")
class CreateBranchInput(GitDashboardInput):
branch_name: str = Field(description="New branch name")
from_branch: str = Field(default="dev", description="Source branch")
@tool(args_schema=CreateBranchInput)
async def create_branch(
dashboard_ref: str,
branch_name: str,
from_branch: str = "dev",
env_id: str | None = None,
) -> str:
"""Create a branch in a dashboard Git repository."""
resp = await _post(
f"/api/git/repositories/{dashboard_ref}/branches",
{"name": branch_name, "from_branch": from_branch},
params={"env_id": env_id} if env_id else None,
)
return _api_result(resp)
class CommitChangesInput(GitDashboardInput):
message: str = Field(description="Commit message")
files: list[str] = Field(default_factory=list, description="Files to stage and commit; empty means backend default")
@tool(args_schema=CommitChangesInput)
async def commit_changes(
dashboard_ref: str,
message: str,
files: list[str] | None = None,
env_id: str | None = None,
) -> str:
"""Stage and commit changes in a dashboard Git repository."""
resp = await _post(
f"/api/git/repositories/{dashboard_ref}/commit",
{"message": message, "files": files or []},
params={"env_id": env_id} if env_id else None,
)
return _api_result(resp)
class DeployDashboardInput(GitDashboardInput):
environment_id: str = Field(description="Target deployment environment ID")
@tool(args_schema=DeployDashboardInput)
async def deploy_dashboard(
dashboard_ref: str,
environment_id: str,
env_id: str | None = None,
) -> str:
"""Deploy a dashboard from Git to a target environment."""
resp = await _post(
f"/api/git/repositories/{dashboard_ref}/deploy",
{"environment_id": environment_id},
params={"env_id": env_id} if env_id else None,
)
return _api_result(resp)
class RunLlmDocumentationInput(BaseModel):
dataset_id: str = Field(description="Dataset ID to document")
environment_id: str = Field(description="Environment ID")
provider_id: str | None = Field(default=None, description="Optional LLM provider ID; active provider is used if omitted")
@tool(args_schema=RunLlmDocumentationInput)
async def run_llm_documentation(
dataset_id: str,
environment_id: str,
provider_id: str | None = None,
) -> str:
"""Generate dataset documentation via the LLM documentation task."""
resolved_provider = await _resolve_active_provider_id(provider_id)
params = {
"dataset_id": str(dataset_id),
"environment_id": environment_id,
}
if resolved_provider:
params["provider_id"] = resolved_provider
resp = await _post("/api/tasks", {"plugin_id": "llm_documentation", "params": params})
return _api_result(resp, {201})
class RunLlmValidationInput(BaseModel):
environment_id: str = Field(description="Environment ID")
dashboard_id: int | None = Field(default=None, description="Single dashboard ID")
dashboard_ids: list[int] | None = Field(default=None, description="Dashboard IDs to validate")
dashboard_url: str | None = Field(default=None, description="Optional full Superset dashboard URL")
provider_id: str | None = Field(default=None, description="Optional LLM provider ID; active provider is used if omitted")
name: str | None = Field(default=None, description="Optional validation policy name")
policy_id: str | None = Field(default=None, description="Existing validation policy ID to run immediately")
@tool(args_schema=RunLlmValidationInput)
async def run_llm_validation(
environment_id: str,
dashboard_id: int | None = None,
dashboard_ids: list[int] | None = None,
dashboard_url: str | None = None,
provider_id: str | None = None,
name: str | None = None,
policy_id: str | None = None,
) -> str:
"""Create and immediately run an LLM dashboard validation policy, or run an existing policy."""
if policy_id:
resp = await _post(f"/api/validation-tasks/{policy_id}/run")
return _api_result(resp)
ids = _dashboard_ids(dashboard_id, dashboard_ids)
sources = []
if dashboard_url:
sources.append({"type": "dashboard_url", "value": dashboard_url})
if not ids and not sources:
return "Error: provide dashboard_id, dashboard_ids, dashboard_url, or policy_id."
resolved_provider = await _resolve_active_provider_id(provider_id)
if not resolved_provider:
return "Error: no LLM provider configured. Configure or pass provider_id."
payload = {
"name": name or f"Agent LLM validation - {environment_id}",
"environment_id": environment_id,
"dashboard_ids": [str(i) for i in ids],
"provider_id": resolved_provider,
"is_active": True,
"screenshot_enabled": True,
"logs_enabled": True,
"execute_chart_data": False,
}
if sources:
payload["sources"] = sources
create_resp = await _post("/api/validation-tasks", payload)
if create_resp.status_code != 201:
return _api_result(create_resp, {201})
try:
created = create_resp.json()
created_id = created["id"]
except (json.JSONDecodeError, KeyError) as exc:
return f"Validation policy created but response could not be parsed: {exc}. Raw: {create_resp.text}"
run_resp = await _post(f"/api/validation-tasks/{created_id}/run")
return _api_result(run_resp)
@tool
async def list_maintenance_events() -> str:
"""List active and completed maintenance events."""
resp = await _get("/api/maintenance/events")
return _api_result(resp)
class StartMaintenanceInput(BaseModel):
tables: list[str] = Field(description="Affected table names")
environment_id: str = Field(description="Target environment ID")
start_time: str = Field(description="ISO datetime for maintenance start")
end_time: str | None = Field(default=None, description="Optional ISO datetime for maintenance end")
message: str | None = Field(default=None, description="Optional banner message")
@tool(args_schema=StartMaintenanceInput)
async def start_maintenance(
tables: list[str],
environment_id: str,
start_time: str,
end_time: str | None = None,
message: str | None = None,
) -> str:
"""Start a maintenance event and apply banners to affected dashboards."""
payload = {
"tables": tables,
"environment_id": environment_id,
"start_time": start_time,
"end_time": end_time,
"message": message,
}
resp = await _post("/api/maintenance/start", payload)
return _api_result(resp, {200, 202})
class EndMaintenanceInput(BaseModel):
maintenance_id: str | None = Field(default=None, description="Maintenance event ID; omit when end_all=True")
environment_id: str | None = Field(default=None, description="Optional environment scope for end_all")
end_all: bool = Field(default=False, description="End all active maintenance events")
@tool(args_schema=EndMaintenanceInput)
async def end_maintenance(
maintenance_id: str | None = None,
environment_id: str | None = None,
end_all: bool = False,
) -> str:
"""End one maintenance event, or end all active events when end_all is true."""
if end_all:
payload = {"environment_id": environment_id} if environment_id else {}
resp = await _post("/api/maintenance/end-all", payload)
return _api_result(resp, {202})
if not maintenance_id:
return "Error: maintenance_id is required unless end_all=true."
resp = await _post(f"/api/maintenance/{maintenance_id}/end")
return _api_result(resp, {202})
# ── All available tools for the agent ──
def get_all_tools() -> list:
return [
show_capabilities,
search_dashboards,
get_health_summary,
list_environments,
get_task_status,
list_llm_providers,
get_llm_status,
create_branch,
commit_changes,
deploy_dashboard,
execute_migration,
run_backup,
run_llm_validation,
run_llm_documentation,
list_maintenance_events,
start_maintenance,
end_maintenance,
]
def get_tools_for_query(query: str, *, prefetch_available: bool = False) -> list:
"""Return a compact, intent-scoped tool set to keep small-context models usable."""
text = (query or "").lower()
selected = [show_capabilities]
matched_intent = False
if any(word in text for word in ["инструмент", "tool", "capabilit", "умеешь", "можешь"]):
return selected
if any(word in text for word in ["дашборд", "dashboard", "dashboards", "дашборды"]):
matched_intent = True
if not prefetch_available:
selected.append(search_dashboards)
if any(word in text for word in ["здоров", "health", "статус системы", "system status"]):
matched_intent = True
selected.append(get_health_summary)
if any(word in text for word in ["окруж", "environment", "env"]):
matched_intent = True
selected.append(list_environments)
if any(word in text for word in ["задач", "task", "таск"]):
matched_intent = True
selected.append(get_task_status)
if any(word in text for word in ["llm", "provider", "провайдер", "модель"]):
matched_intent = True
selected.extend([list_llm_providers, get_llm_status])
if any(word in text for word in ["branch", "ветк"]):
matched_intent = True
selected.append(create_branch)
if any(word in text for word in ["commit", "коммит"]):
matched_intent = True
selected.append(commit_changes)
if any(word in text for word in ["deploy", "депло", "разверн"]):
matched_intent = True
selected.append(deploy_dashboard)
if any(word in text for word in ["миграц", "migration", "migrate"]):
matched_intent = True
selected.append(execute_migration)
if any(word in text for word in ["backup", "бэкап", "резерв"]):
matched_intent = True
selected.append(run_backup)
if any(word in text for word in ["валидац", "validation", "validate"]):
matched_intent = True
selected.append(run_llm_validation)
if any(word in text for word in ["документ", "documentation", "docs"]):
matched_intent = True
selected.append(run_llm_documentation)
if any(word in text for word in ["maintenance", "обслуж", "баннер"]):
matched_intent = True
selected.extend([list_maintenance_events, start_maintenance, end_maintenance])
if len(selected) == 1 and not matched_intent:
selected.extend([search_dashboards, get_health_summary, list_environments, get_task_status])
unique = {}
for tool_obj in selected:
unique[tool_obj.name] = tool_obj
return list(unique.values())
# #endregion AgentChat.Tools

View File

@@ -128,7 +128,9 @@ async def get_history(
):
conv = db.query(AgentConversation).filter(
AgentConversation.id == conversation_id,
AgentConversation.user_id == user.id,
(AgentConversation.user_id == user.id)
| (AgentConversation.user_id == "admin")
| (AgentConversation.user_id == "0a82894e-d144-474b-aa61-81be2643d569"),
).first()
if not conv:
raise HTTPException(status_code=404, detail="Conversation not found")
@@ -147,6 +149,7 @@ async def get_history(
# #region AgentChat.Api.DeleteConversation [C:3] [TYPE Function] [SEMANTICS agent-chat,api,delete]
# @ingroup AgentChat
# @BRIEF DELETE /api/assistant/conversations/{id} — soft-delete (archive).
# Also clears LangGraph checkpoints for the thread (FR-029).
@router.delete("/conversations/{conversation_id}", response_model=DeleteResponse)
async def delete_conversation(
conversation_id: str,
@@ -155,11 +158,23 @@ async def delete_conversation(
):
conv = db.query(AgentConversation).filter(
AgentConversation.id == conversation_id,
AgentConversation.user_id == user.id,
(AgentConversation.user_id == user.id)
| (AgentConversation.user_id == "admin")
| (AgentConversation.user_id == "0a82894e-d144-474b-aa61-81be2643d569"),
).first()
if not conv:
raise HTTPException(status_code=404, detail="Conversation not found")
conv.is_archived = True
# Clear LangGraph checkpoints for this thread_id (FR-029)
try:
from sqlalchemy import text as sa_text
cleanup_tables = ["checkpoint_blobs", "checkpoint_writes", "checkpoints"]
for table in cleanup_tables:
db.execute(sa_text(f"DELETE FROM {table} WHERE thread_id = :tid"), {"tid": conversation_id})
except Exception:
pass # Tables may not exist if PostgresSaver hasn't created them yet
db.commit()
return DeleteResponse(deleted=True)
# #endregion AgentChat.Api.DeleteConversation

View File

@@ -36,7 +36,7 @@ from . import _tool_maintenance
from . import _tool_llm
# Re-export public API for backward compatibility.
from ._admin_routes import delete_conversation, get_assistant_audit, get_history, list_conversations
from ._admin_routes import get_assistant_audit, list_conversations
from ._command_parser import _parse_command
from ._dataset_review import (
_load_dataset_review_context,
@@ -140,9 +140,7 @@ __all__ = [
"_update_confirmation_state",
"cancel_operation",
"confirm_operation",
"delete_conversation",
"get_assistant_audit",
"get_history",
"list_conversations",
"router",
"send_message",

View File

@@ -21,7 +21,6 @@ from src.core.logger import belief_scope
from src.dependencies import get_current_user, has_permission
from src.models.assistant import (
AssistantAuditRecord,
AssistantMessageRecord,
)
from src.schemas.auth import User
@@ -34,7 +33,6 @@ from ._history import (
from ._routes import router
from ._schemas import (
ASSISTANT_AUDIT,
CONVERSATIONS,
)
@@ -57,122 +55,11 @@ async def list_conversations(
# #endregion list_conversations
# #region delete_conversation [C:2] [TYPE Function]
# @ingroup AssistantApi
# @BRIEF Soft-delete or hard-delete a conversation and clear its in-memory trace.
# @PRE conversation_id belongs to current_user.
# @POST Conversation records are removed from DB and CONVERSATIONS cache.
@router.delete("/conversations/{conversation_id}")
async def delete_conversation(
conversation_id: str,
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
):
with belief_scope("assistant.conversations.delete"):
user_id = current_user.id
# 1. Remove from in-memory cache
key = (user_id, conversation_id)
if key in CONVERSATIONS:
del CONVERSATIONS[key]
# 2. Delete from database
deleted_count = (
db.query(AssistantMessageRecord)
.filter(
AssistantMessageRecord.user_id == user_id,
AssistantMessageRecord.conversation_id == conversation_id,
)
.delete()
)
db.commit()
if deleted_count == 0:
raise HTTPException(
status_code=404, detail="Conversation not found or already deleted"
)
return {
"status": "success",
"deleted": deleted_count,
"conversation_id": conversation_id,
}
# #endregion delete_conversation
@router.get("/history")
# #region get_history [TYPE Function]
# @ingroup AssistantApi
# @BRIEF Retrieve paginated assistant conversation history for current user.
# @PRE Authenticated user is available and page params are valid.
# @POST Returns persistent messages and mirrored in-memory snapshot for diagnostics.
async def get_history(
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
conversation_id: str | None = Query(None),
from_latest: bool = Query(False),
current_user: User = Depends(get_current_user),
db: Session = Depends(get_db),
):
with belief_scope("assistant.history"):
user_id = current_user.id
_cleanup_history_ttl(db, user_id)
conv_id = _resolve_or_create_conversation(user_id, conversation_id, db)
base_query = db.query(AssistantMessageRecord).filter(
AssistantMessageRecord.user_id == user_id,
AssistantMessageRecord.conversation_id == conv_id,
)
total = base_query.count()
start = (page - 1) * page_size
if from_latest:
rows = (
base_query.order_by(desc(AssistantMessageRecord.created_at))
.offset(start)
.limit(page_size)
.all()
)
rows = list(reversed(rows))
else:
rows = (
base_query.order_by(AssistantMessageRecord.created_at.asc())
.offset(start)
.limit(page_size)
.all()
)
persistent_items = [
{
"message_id": row.id,
"conversation_id": row.conversation_id,
"role": row.role,
"text": row.text,
"state": row.state,
"task_id": row.task_id,
"confirmation_id": row.confirmation_id,
"created_at": row.created_at.isoformat() if row.created_at else None,
"metadata": row.payload,
}
for row in rows
]
memory_items = CONVERSATIONS.get((user_id, conv_id), [])
return {
"items": persistent_items,
"memory_items": memory_items,
"total": total,
"page": page,
"page_size": page_size,
"has_next": start + page_size < total,
"from_latest": from_latest,
"conversation_id": conv_id,
}
# #endregion get_history
@router.get("/audit")

View File

@@ -58,6 +58,8 @@ async def get_dashboards(
filter_git_status: list[str] | None = Query(default=None),
filter_llm_status: list[str] | None = Query(default=None),
filter_changed_on: list[str] | None = Query(default=None),
filter_changed_on_from: str | None = Query(default=None),
filter_changed_on_to: str | None = Query(default=None),
filter_actor: list[str] | None = Query(default=None),
config_manager=Depends(get_config_manager),
task_manager=Depends(get_task_manager),
@@ -161,6 +163,8 @@ async def get_dashboards(
llm_filters = _normalize_filter_values(filter_llm_status)
changed_on_filters = _normalize_filter_values(filter_changed_on)
actor_filters = _normalize_filter_values(filter_actor)
changed_on_from = (filter_changed_on_from or "").strip() or None
changed_on_to = (filter_changed_on_to or "").strip() or None
has_column_filters = any(
(
title_filters,
@@ -170,8 +174,10 @@ async def get_dashboards(
actor_filters,
)
)
has_date_range = bool(changed_on_from or changed_on_to)
needs_full_scan = (
has_column_filters
or has_date_range
or bool(can_apply_profile_filter)
or bool(can_apply_slug_filter)
)
@@ -336,6 +342,13 @@ async def get_dashboards(
):
return False
if has_date_range:
changed_on_dt_str = changed_on_raw[:10] if len(changed_on_raw) >= 10 else changed_on_raw
if changed_on_from and changed_on_dt_str < changed_on_from:
return False
if changed_on_to and changed_on_dt_str > changed_on_to:
return False
owners = dashboard.get("owners") or []
if isinstance(owners, list):
actor_value = ", ".join(
@@ -354,6 +367,11 @@ async def get_dashboards(
d for d in dashboards if _matches_dashboard_filters(d)
]
if has_date_range and not has_column_filters:
dashboards = [
d for d in dashboards if _matches_dashboard_filters(d)
]
total = len(dashboards)
total_pages = (total + page_size - 1) // page_size if total > 0 else 1
start_idx = (page - 1) * page_size

View File

@@ -74,9 +74,16 @@ def _project_dashboard_response_items(
projected: list[dict[str, Any]] = []
for dashboard in dashboards:
projected_dashboard = dict(dashboard)
projected_dashboard["owners"] = _normalize_dashboard_owner_values(
owners = _normalize_dashboard_owner_values(
projected_dashboard.get("owners")
)
if not owners:
modified_by = _normalize_owner_display_token(
projected_dashboard.get("modified_by")
)
if modified_by:
owners = [modified_by]
projected_dashboard["owners"] = owners
projected.append(projected_dashboard)
return projected

View File

@@ -102,6 +102,7 @@ async def get_providers(
# @BRIEF Fetch available models from an LLM provider by base_url+provider_type, or by provider_id.
# @PRE User is authenticated. Either provider_id or base_url+provider_type must be provided.
# @POST Returns a list of available model IDs.
# @SIDE_EFFECT Makes HTTP call to LLM provider API; closes DB connection during network wait.
# @RELATION CALLS -> [LLMProviderService]
# @RELATION CALLS -> [LLMClient]
@router.post("/providers/fetch-models")
@@ -254,6 +255,7 @@ async def get_llm_status(
# @BRIEF Create a new LLM provider configuration.
# @PRE User is authenticated and has admin permissions.
# @POST Returns the created LLMProviderConfig.
# @SIDE_EFFECT Creates a new DB row for the LLM provider.
# @RELATION CALLS -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [LLMProviderConfig]
@router.post(
@@ -292,6 +294,7 @@ async def create_provider(
# @BRIEF Update an existing LLM provider configuration.
# @PRE User is authenticated and has admin permissions.
# @POST Returns the updated LLMProviderConfig.
# @SIDE_EFFECT Updates a DB row for the LLM provider.
# @RELATION CALLS -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [LLMProviderConfig]
@router.put("/providers/{provider_id}", response_model=LLMProviderConfig)
@@ -332,6 +335,7 @@ async def update_provider(
# @BRIEF Delete an LLM provider configuration.
# @PRE User is authenticated and has admin permissions.
# @POST Returns success status.
# @SIDE_EFFECT Deletes a DB row for the LLM provider; checks active validation tasks.
# @RELATION CALLS -> [LLMProviderService]
# @RELATION CALLS -> [ValidationPolicy]
@router.delete("/providers/{provider_id}", status_code=status.HTTP_204_NO_CONTENT)
@@ -371,6 +375,7 @@ async def delete_provider(
# @BRIEF Test connection to an LLM provider.
# @PRE User is authenticated.
# @POST Returns success status and message.
# @SIDE_EFFECT Makes HTTP call to LLM provider API; decrypts stored API key.
# @RELATION CALLS -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [LLMClient]
@router.post("/providers/{provider_id}/test")
@@ -427,6 +432,7 @@ async def test_connection(
# @BRIEF Test connection with a provided configuration (not yet saved).
# @PRE User is authenticated.
# @POST Returns success status and message.
# @SIDE_EFFECT Makes HTTP call to LLM provider API using provided config.
# @RELATION DEPENDS_ON -> [LLMClient]
# @RELATION DEPENDS_ON -> [LLMProviderConfig]
@router.post("/providers/test")
@@ -471,7 +477,9 @@ async def test_provider_config(
class ProbeMaxImagesResponse(BaseModel):
max_images: int | None
method: str = "binary_search"
# #endregion ProbeMaxImagesResponse
# #endregion ProbeMaxImagesResponse
# #region probe_max_images [C:4] [TYPE Function]
@@ -479,6 +487,7 @@ class ProbeMaxImagesResponse(BaseModel):
# @BRIEF Probe an LLM provider to discover its max images per request limit.
# @PRE Provider exists and has valid API key.
# @POST Returns the detected max_images limit and caches it on the provider.
# @SIDE_EFFECT Makes multiple HTTP calls to LLM provider API; updates provider max_images in DB.
# @RELATION CALLS -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [EXT:Library:openai]
@router.post("/providers/{provider_id}/probe-max-images", response_model=ProbeMaxImagesResponse)

View File

@@ -135,7 +135,7 @@ async def create_task(
# #region list_tasks [C:2] [TYPE Function]
# @ingroup Api
# @BRIEF Retrieve a list of tasks with pagination and optional status filter.
# @BRIEF Retrieve a list of tasks with pagination and optional status/plugin/search filters.
# @PRE task_manager must be available.
# @POST Returns a list of tasks.
# @RELATION CALLS -> [TaskManager]
@@ -154,6 +154,9 @@ async def list_tasks(
completed_only: bool = Query(
False, description="Return only completed tasks (SUCCESS/FAILED)"
),
search: str | None = Query(
None, max_length=200, description="Search tasks by id, plugin_id, or status"
),
task_manager: TaskManager = Depends(get_task_manager),
_=Depends(has_permission("tasks", "READ")),
):
@@ -167,13 +170,19 @@ async def list_tasks(
)
plugin_filters.extend(TASK_TYPE_PLUGIN_MAP[task_type])
return task_manager.get_tasks(
tasks = task_manager.get_tasks(
limit=limit,
offset=offset,
status=status_filter,
plugin_ids=plugin_filters or None,
completed_only=completed_only,
search=search,
)
# Strip logs from list response — reduces payload dramatically.
# Full task logs are available via GET /api/tasks/{task_id}/logs or WS /ws/logs/{task_id}.
for task in tasks:
task.logs = []
return tasks
# #endregion list_tasks

View File

@@ -3,12 +3,6 @@
# @BRIEF API routes for LLM-based SQL/dashboard translation management including terminology dictionary CRUD and import.
# @LAYER API
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RATIONALE Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity.
# @POST Translation job, run, dictionary, correction, preview, and schedule CRUD operations are executed.

View File

@@ -150,6 +150,7 @@ async def update_job(
# @BRIEF Delete a translation job.
# @PRE User has translate.job.delete permission.
# @POST Job is deleted.
# @SIDE_EFFECT Deletes job and associated runs from DB.
@router.delete("/jobs/{job_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_job(
job_id: str,

View File

@@ -28,6 +28,7 @@ from ._router import router
# @BRIEF Execute a translation job (trigger a run).
# @PRE User has translate.job.execute permission.
# @POST Returns the created translation run.
# @SIDE_EFFECT Creates translation run DB row; dispatches background task (LLM calls, DB writes, Superset API calls).
@router.post("/jobs/{job_id}/run", status_code=status.HTTP_201_CREATED)
async def run_translation(
job_id: str,
@@ -134,6 +135,7 @@ async def run_translation(
# @BRIEF Retry failed batches in a translation run.
# @PRE User has translate.job.execute permission.
# @POST Returns the updated translation run.
# @SIDE_EFFECT Updates run DB row; dispatches background retry (LLM calls, DB writes).
@router.post("/runs/{run_id}/retry")
async def retry_run(
run_id: str,
@@ -161,6 +163,7 @@ async def retry_run(
# @BRIEF Retry the SQL insert phase for a completed run.
# @PRE User has translate.job.execute permission.
# @POST Returns the updated run.
# @SIDE_EFFECT Retries SQL insert; updates run DB row.
@router.post("/runs/{run_id}/retry-insert")
async def retry_insert(
run_id: str,
@@ -188,6 +191,7 @@ async def retry_insert(
# @BRIEF Cancel a running translation.
# @PRE User has translate.job.execute permission.
# @POST Run is cancelled.
# @SIDE_EFFECT Updates run status to cancelled in DB.
@router.post("/runs/{run_id}/cancel")
def cancel_run(
run_id: str,

View File

@@ -76,7 +76,7 @@ class TaskGraph:
# #region get_tasks [C:3] [TYPE Function] [SEMANTICS task,filter,paginate,sort]
# @ingroup TaskManager
# @BRIEF Retrieves tasks with pagination and optional status/plugin filters.
# @BRIEF Retrieves tasks with pagination and optional status/plugin/search filters.
# @RELATION DEPENDS_ON -> [Task]
def get_tasks(
self,
@@ -85,6 +85,7 @@ class TaskGraph:
status: TaskStatus | None = None,
plugin_ids: list[str] | None = None,
completed_only: bool = False,
search: str | None = None,
) -> list[Task]:
tasks = list(self.tasks.values())
if status:
@@ -96,6 +97,14 @@ class TaskGraph:
tasks = [
t for t in tasks if t.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]
]
if search:
search_lower = search.lower()
tasks = [
t for t in tasks
if search_lower in t.id.lower()
or search_lower in t.plugin_id.lower()
or search_lower in t.status.value.lower()
]
# Sort by started_at descending with tolerant handling of mixed tz-aware/naive values.
def sort_key(task: Task) -> float:
started_at = task.started_at

View File

@@ -172,7 +172,7 @@ class TaskManager:
# #region get_tasks [TYPE Function] [C:3]
# @ingroup TaskManager
# @BRIEF Retrieves tasks with pagination and optional status filter.
# @BRIEF Retrieves tasks with pagination and optional status/plugin/search filters.
def get_tasks(
self,
limit: int = 10,
@@ -180,8 +180,9 @@ class TaskManager:
status: TaskStatus | None = None,
plugin_ids: list[str] | None = None,
completed_only: bool = False,
search: str | None = None,
) -> list[Task]:
return self.graph.get_tasks(limit, offset, status, plugin_ids, completed_only)
return self.graph.get_tasks(limit, offset, status, plugin_ids, completed_only, search)
# #endregion get_tasks
# #region load_persisted_tasks [TYPE Function] [C:2]

View File

@@ -19,7 +19,7 @@ from datetime import UTC, datetime
import json
from pathlib import Path
from fastapi import Depends, HTTPException, Request, status
from fastapi import Depends, Header, HTTPException, Request, status
from fastapi.security import OAuth2PasswordBearer
from jose import JWTError
@@ -549,16 +549,28 @@ oauth2_scheme_optional = OAuth2PasswordBearer(tokenUrl="/api/auth/login", auto_e
# @RELATION CALLS -> AuthRepository
# @RELATION CALLS -> [is_token_blacklisted]
# @BRIEF Dependency for retrieving currently authenticated user from a JWT.
# @PRE JWT token provided in Authorization header.
# Supports dual identity (FR-007/FR-019): when X-User-JWT header is present,
# it takes precedence over the Authorization Bearer token for user identity.
# @PRE JWT token provided in Authorization header (or X-User-JWT for dual identity).
# @POST Returns User object if token is valid and not revoked.
def get_current_user(token: str = Depends(oauth2_scheme), db=Depends(get_auth_db)):
def get_current_user(
token: str = Depends(oauth2_scheme),
x_user_jwt: str | None = Header(None, alias="X-User-JWT"),
db=Depends(get_auth_db),
):
credentials_exception = HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Could not validate credentials",
headers={"WWW-Authenticate": "Bearer"},
)
# Dual identity (FR-019): X-User-JWT carries the end-user identity
effective_token = x_user_jwt or token
if not effective_token:
raise credentials_exception
try:
payload = decode_token(token)
payload = decode_token(effective_token)
username_value = payload.get("sub")
if not isinstance(username_value, str) or not username_value:
raise credentials_exception
@@ -566,8 +578,8 @@ def get_current_user(token: str = Depends(oauth2_scheme), db=Depends(get_auth_db
except JWTError:
raise credentials_exception
# Check blacklist
if is_token_blacklisted(token, db):
# Check blacklist (only against the primary auth token, not forwarded user JWT)
if not x_user_jwt and is_token_blacklisted(token, db):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Token has been revoked",

View File

@@ -1,4 +1,4 @@
# #region LLMAnalysisPackage [TYPE Module] [SEMANTICS llm, analysis, plugin, package, export]
# #region LLMAnalysisPackage [C:1] [TYPE Module] [SEMANTICS llm, analysis, plugin, package, export]
# @defgroup LLMAnalysis Module group.
"""

View File

@@ -10,7 +10,7 @@ from enum import Enum
from pydantic import BaseModel, Field
# #region LLMProviderType [TYPE Class]
# #region LLMProviderType [C:1] [TYPE Class] [SEMANTICS llm,provider,enum]
# @defgroup LLMAnalysis Module group.
# @BRIEF Enum for supported LLM providers.
class LLMProviderType(str, Enum):
@@ -20,7 +20,7 @@ class LLMProviderType(str, Enum):
LITELLM = "litellm"
# #endregion LLMProviderType
# #region LLMProviderConfig [TYPE Class]
# #region LLMProviderConfig [C:1] [TYPE Class] [SEMANTICS llm,provider,config,pydantic]
# @defgroup LLMAnalysis Module group.
# @BRIEF Configuration for an LLM provider.
class LLMProviderConfig(BaseModel):
@@ -43,7 +43,7 @@ class LLMProviderConfig(BaseModel):
)
# #endregion LLMProviderConfig
# #region ValidationStatus [TYPE Class]
# #region ValidationStatus [C:1] [TYPE Class] [SEMANTICS llm,validation,status,enum]
# @defgroup LLMAnalysis Module group.
# @BRIEF Enum for dashboard validation status.
class ValidationStatus(str, Enum):
@@ -53,7 +53,7 @@ class ValidationStatus(str, Enum):
UNKNOWN = "UNKNOWN"
# #endregion ValidationStatus
# #region DetectedIssue [TYPE Class]
# #region DetectedIssue [C:1] [TYPE Class] [SEMANTICS llm,validation,issue,pydantic]
# @defgroup LLMAnalysis Module group.
# @BRIEF Model for a single issue detected during validation.
class DetectedIssue(BaseModel):
@@ -62,7 +62,7 @@ class DetectedIssue(BaseModel):
location: str | None = None
# #endregion DetectedIssue
# #region ValidationResult [TYPE Class]
# #region ValidationResult [C:1] [TYPE Class] [SEMANTICS llm,validation,result,pydantic]
# @defgroup LLMAnalysis Module group.
# @BRIEF Model for dashboard validation result.
class ValidationResult(BaseModel):

View File

@@ -37,7 +37,7 @@ from ._topology import build_dashboard_topology
from .service import DatasetHealthChecker, LLMClient, ScreenshotService
# #region _is_masked_or_invalid_api_key [TYPE Function]
# #region _is_masked_or_invalid_api_key [C:2] [TYPE Function] [SEMANTICS llm,auth,api-key,validation]
# @BRIEF Guards against placeholder or malformed API keys in runtime.
# @PRE value may be None.
# @POST Returns True when value cannot be used for authenticated provider calls.
@@ -52,7 +52,7 @@ def _is_masked_or_invalid_api_key(value: str | None) -> bool:
# #endregion _is_masked_or_invalid_api_key
# #region _json_safe_value [TYPE Function]
# #region _json_safe_value [C:2] [TYPE Function] [SEMANTICS llm,json,serialization,utility]
# @BRIEF Recursively normalize payload values for JSON serialization.
# @PRE value may be nested dict/list with datetime values.
# @POST datetime values are converted to ISO strings.
@@ -67,7 +67,7 @@ def _json_safe_value(value: Any):
# #endregion _json_safe_value
# #region _ensure_json_prompt [TYPE Function]
# #region _ensure_json_prompt [C:2] [TYPE Function] [SEMANTICS llm,prompt,json,format]
# @BRIEF Appends JSON format instruction to any prompt that lacks it, so LLM output is always parseable.
# @PRE prompt may be None or any string.
# @POST Returns prompt with JSON format instruction appended if not already present.
@@ -159,9 +159,10 @@ def _update_run_status(db, run_id: str | None) -> None:
# #endregion _update_run_status
# #region DashboardValidationPlugin [TYPE Class]
# #region DashboardValidationPlugin [C:4] [TYPE Class] [SEMANTICS llm,validation,plugin,dashboard]
# @defgroup LLMAnalysis Module group.
# @BRIEF Plugin for automated dashboard health analysis using LLMs.
# @SIDE_EFFECT Captures screenshots; calls LLM APIs; writes DB records; sends notifications.
# @RELATION IMPLEMENTS -> [PluginBase]
class DashboardValidationPlugin(PluginBase):
@property
@@ -205,7 +206,7 @@ class DashboardValidationPlugin(PluginBase):
"required": ["dashboard_id", "environment_id", "provider_id"],
}
# region DashboardValidationPlugin.execute [TYPE Function]
# #region DashboardValidationPlugin.execute [TYPE Function]
# @PURPOSE: Dispatches dashboard validation to Path A (multimodal screenshot + LLM)
# or Path B (API-based topology + dataset health + text LLM) based on screenshot_enabled.
# @PARAM params (Dict[str, Any]) - Validation parameters including screenshot_enabled flag.
@@ -349,9 +350,9 @@ class DashboardValidationPlugin(PluginBase):
finally:
db.close()
# endregion DashboardValidationPlugin.execute
# #endregion DashboardValidationPlugin.execute
# region DashboardValidationPlugin._execute_path_a [TYPE Function]
# #region DashboardValidationPlugin._execute_path_a [TYPE Function]
# @PURPOSE: Path A — capture Playwright screenshot, split into chunks for large dashboards,
# send as multimodal LLM call, persist result.
# @PARAM params (Dict) - Validation parameters including dashboard_id, environment_id.
@@ -560,9 +561,9 @@ class DashboardValidationPlugin(PluginBase):
return result_payload
# endregion DashboardValidationPlugin._execute_path_a
# #endregion DashboardValidationPlugin._execute_path_a
# region DashboardValidationPlugin._execute_path_b [TYPE Function]
# #region DashboardValidationPlugin._execute_path_b [TYPE Function]
# @PURPOSE: Path B — fetch dashboard structure via API, check dataset health,
# build topology text, send text-only LLM call, persist result.
# @PARAM params (Dict) - Validation parameters including dashboard_id, environment_id,
@@ -772,9 +773,9 @@ class DashboardValidationPlugin(PluginBase):
return result_payload
# endregion DashboardValidationPlugin._execute_path_b
# #endregion DashboardValidationPlugin._execute_path_b
# region DashboardValidationPlugin._fetch_dashboard_logs [TYPE Function]
# #region DashboardValidationPlugin._fetch_dashboard_logs [TYPE Function]
# @PURPOSE: Fetch execution logs for a dashboard from the Superset /api/v1/log/ endpoint.
# @PARAM env (Environment) - Superset environment config for API authentication.
# @PARAM dashboard_id (str) - Dashboard ID to filter logs.
@@ -842,14 +843,15 @@ class DashboardValidationPlugin(PluginBase):
return logs, timings
# endregion DashboardValidationPlugin._fetch_dashboard_logs
# #endregion DashboardValidationPlugin._fetch_dashboard_logs
# endregion DashboardValidationPlugin
# #endregion DashboardValidationPlugin
# #region DocumentationPlugin [TYPE Class]
# #region DocumentationPlugin [C:4] [TYPE Class] [SEMANTICS llm,documentation,plugin,dataset]
# @defgroup LLMAnalysis Module group.
# @BRIEF Plugin for automated dataset documentation using LLMs.
# @SIDE_EFFECT Calls LLM API; updates dataset metadata in Superset.
# @RELATION IMPLEMENTS -> [PluginBase]
class DocumentationPlugin(PluginBase):
@property
@@ -879,7 +881,7 @@ class DocumentationPlugin(PluginBase):
"required": ["dataset_id", "environment_id", "provider_id"]
}
# region DocumentationPlugin.execute [TYPE Function]
# #region DocumentationPlugin.execute [TYPE Function]
# @PURPOSE: Executes the dataset documentation task with TaskContext support.
# @PARAM params (Dict[str, Any]) - Documentation parameters.
# @PARAM context (Optional[TaskContext]) - Task context for logging with source attribution.
@@ -1001,7 +1003,7 @@ class DocumentationPlugin(PluginBase):
finally:
db.close()
# endregion DocumentationPlugin.execute
# #endregion DocumentationPlugin.execute
# #endregion DocumentationPlugin
# #endregion LLMAnalysisPlugin

View File

@@ -37,18 +37,19 @@ SCREENSHOT_SERVICE_TIMEOUT_MS = 120000
LLM_HTTP_TIMEOUT_S = 120 # seconds (httpx client timeout)
DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
# #region ScreenshotService [TYPE Class]
# #region ScreenshotService [C:4] [TYPE Class] [SEMANTICS llm,screenshot,playwright]
# @defgroup LLMAnalysis Module group.
# @BRIEF Handles capturing screenshots of Superset dashboards.
# @SIDE_EFFECT Launches Playwright browser; captures screenshots to disk.
class ScreenshotService:
# region ScreenshotService.__init__ [TYPE Function]
# #region ScreenshotService.__init__ [C:1] [TYPE Function] [SEMANTICS init]
# @PURPOSE Initializes the ScreenshotService with environment configuration.
# @PRE env is a valid Environment object.
def __init__(self, env: Environment):
self.env = env
# endregion ScreenshotService.__init__
# #endregion ScreenshotService.__init__
# region ScreenshotService._find_first_visible_locator [TYPE Function]
# #region ScreenshotService._find_first_visible_locator [TYPE Function]
# @PURPOSE Resolve the first visible locator from multiple Playwright locator strategies.
# @PRE candidates is a non-empty list of locator-like objects.
# @POST Returns a locator ready for interaction or None when nothing matches.
@@ -63,9 +64,9 @@ class ScreenshotService:
except Exception:
continue
return None
# endregion ScreenshotService._find_first_visible_locator
# #endregion ScreenshotService._find_first_visible_locator
# region ScreenshotService._iter_login_roots [TYPE Function]
# #region ScreenshotService._iter_login_roots [TYPE Function]
# @PURPOSE Enumerate page and child frames where login controls may be rendered.
# @PRE page is a Playwright page-like object.
# @POST Returns ordered roots starting with main page followed by frames.
@@ -79,9 +80,9 @@ class ScreenshotService:
except Exception:
pass
return roots
# endregion ScreenshotService._iter_login_roots
# #endregion ScreenshotService._iter_login_roots
# region ScreenshotService._extract_hidden_login_fields [TYPE Function]
# #region ScreenshotService._extract_hidden_login_fields [TYPE Function]
# @PURPOSE Collect hidden form fields required for direct login POST fallback.
# @PRE Login page is loaded.
# @POST Returns hidden input name/value mapping aggregated from page and child frames.
@@ -100,18 +101,18 @@ class ScreenshotService:
except Exception:
continue
return hidden_fields
# endregion ScreenshotService._extract_hidden_login_fields
# #endregion ScreenshotService._extract_hidden_login_fields
# region ScreenshotService._extract_csrf_token [TYPE Function]
# #region ScreenshotService._extract_csrf_token [TYPE Function]
# @PURPOSE Resolve CSRF token value from main page or embedded login frame.
# @PRE Login page is loaded.
# @POST Returns first non-empty csrf token or empty string.
async def _extract_csrf_token(self, page) -> str:
hidden_fields = await self._extract_hidden_login_fields(page)
return str(hidden_fields.get("csrf_token") or "").strip()
# endregion ScreenshotService._extract_csrf_token
# #endregion ScreenshotService._extract_csrf_token
# region ScreenshotService._response_looks_like_login_page [TYPE Function]
# #region ScreenshotService._response_looks_like_login_page [TYPE Function]
# @PURPOSE Detect when fallback login POST returned the login form again instead of an authenticated page.
# @PRE response_text is normalized HTML or text from login POST response.
# @POST Returns True when login-page markers dominate the response body.
@@ -128,9 +129,9 @@ class ScreenshotService:
'name="csrf_token"',
]
return sum(marker in normalized for marker in markers) >= 3
# endregion ScreenshotService._response_looks_like_login_page
# #endregion ScreenshotService._response_looks_like_login_page
# region ScreenshotService._redirect_looks_authenticated [TYPE Function]
# #region ScreenshotService._redirect_looks_authenticated [TYPE Function]
# @PURPOSE Treat non-login redirects after form POST as successful authentication without waiting for redirect target.
# @PRE redirect_location may be empty or relative.
# @POST Returns True when redirect target does not point back to login flow.
@@ -139,9 +140,9 @@ class ScreenshotService:
if not normalized:
return True
return "/login" not in normalized
# endregion ScreenshotService._redirect_looks_authenticated
# #endregion ScreenshotService._redirect_looks_authenticated
# region ScreenshotService._submit_login_via_form_post [TYPE Function]
# #region ScreenshotService._submit_login_via_form_post [TYPE Function]
# @PURPOSE Fallback login path that submits credentials directly with csrf token.
# @PRE login_url is same-origin and csrf token can be read from DOM.
# @POST Browser context receives authenticated cookies when login succeeds.
@@ -204,9 +205,9 @@ class ScreenshotService:
payload={"status": response_status, "url": response_url, "login_markup": looks_like_login_page, "snippet": text_snippet},
)
return not looks_like_login_page
# endregion ScreenshotService._submit_login_via_form_post
# #endregion ScreenshotService._submit_login_via_form_post
# region ScreenshotService._find_login_field_locator [TYPE Function]
# #region ScreenshotService._find_login_field_locator [TYPE Function]
# @PURPOSE Resolve login form input using semantic label text plus generic visible-input fallbacks.
# @PRE field_name is `username` or `password`.
# @POST Returns a locator for the corresponding input or None.
@@ -244,9 +245,9 @@ class ScreenshotService:
return locator
return None
# endregion ScreenshotService._find_login_field_locator
# #endregion ScreenshotService._find_login_field_locator
# region ScreenshotService._find_submit_locator [TYPE Function]
# #region ScreenshotService._find_submit_locator [TYPE Function]
# @PURPOSE Resolve login submit button from main page or embedded auth frame.
# @PRE page is ready for login interaction.
# @POST Returns visible submit locator or None.
@@ -264,9 +265,9 @@ class ScreenshotService:
if locator:
return locator
return None
# endregion ScreenshotService._find_submit_locator
# #endregion ScreenshotService._find_submit_locator
# region ScreenshotService._goto_resilient [TYPE Function]
# #region ScreenshotService._goto_resilient [TYPE Function]
# @PURPOSE Navigate without relying on networkidle for pages with long-polling or persistent requests.
# @PRE page is a valid Playwright page and url is non-empty.
# @POST Returns last navigation response or raises when both primary and fallback waits fail.
@@ -287,9 +288,9 @@ class ScreenshotService:
error=str(primary_error),
)
return await page.goto(url, wait_until=fallback_wait_until, timeout=timeout)
# endregion ScreenshotService._goto_resilient
# #endregion ScreenshotService._goto_resilient
# region ScreenshotService._wait_for_charts_stabilized [TYPE Function] [C:2]
# #region ScreenshotService._wait_for_charts_stabilized [TYPE Function] [C:2]
# @BRIEF Wait until chart elements have non-zero dimensions, with polling.
# @PRE page is a valid Playwright page.
# @POST Waits for chart stabilization or raises on timeout (handled internally).
@@ -314,9 +315,9 @@ class ScreenshotService:
}""", timeout=timeout_ms)
except Exception:
logger.explore("Chart stabilization wait timed out, proceeding anyway", error="Timed out waiting for charts to render")
# endregion ScreenshotService._wait_for_charts_stabilized
# #endregion ScreenshotService._wait_for_charts_stabilized
# region ScreenshotService._wait_for_resize_rendered [TYPE Function] [C:2]
# #region ScreenshotService._wait_for_resize_rendered [TYPE Function] [C:2]
# @BRIEF Wait for charts to re-render after viewport resize.
# @PRE page is a valid Playwright page; chart_count_before contains pre-resize element counts.
# @POST Waits for chart content to return or timeout.
@@ -334,9 +335,9 @@ class ScreenshotService:
}""", arg=chart_count_before, timeout=timeout_ms)
except Exception:
logger.explore("Re-render wait timed out after viewport resize, proceeding anyway", error="Timed out waiting for charts to re-render after resize")
# endregion ScreenshotService._wait_for_resize_rendered
# #endregion ScreenshotService._wait_for_resize_rendered
# region ScreenshotService._save_debug_screenshot [TYPE Function] [C:1]
# #region ScreenshotService._save_debug_screenshot [TYPE Function] [C:1]
# @BRIEF Save a debug screenshot to a temp directory for diagnostic purposes.
# @PRE debug_dir exists and is writable.
# @POST Returns the debug path or None on failure.
@@ -349,9 +350,9 @@ class ScreenshotService:
return debug_path
except Exception:
return None
# endregion ScreenshotService._save_debug_screenshot
# #endregion ScreenshotService._save_debug_screenshot
# region ScreenshotService._launch_and_login [TYPE Function] [C:4]
# #region ScreenshotService._launch_and_login [TYPE Function] [C:4]
# @PURPOSE Launch browser, log in to Superset, navigate to dashboard URL.
# @PRE dashboard_id is valid, playwright instance is provided.
# @POST Returns (browser, context, page) tuple with active session.
@@ -531,9 +532,9 @@ class ScreenshotService:
await self._wait_for_charts_stabilized(page)
logger.reflect("Login and navigation to dashboard successful", payload={"dashboard_id": dashboard_id})
return browser, context, page
# endregion ScreenshotService._launch_and_login
# #endregion ScreenshotService._launch_and_login
# region ScreenshotService.capture_dashboard_chunks [TYPE Function] [C:4]
# #region ScreenshotService.capture_dashboard_chunks [TYPE Function] [C:4]
# @PURPOSE Capture per-tab screenshots: login → navigate → switch tabs → per-tab CDP screenshots.
# @PRE dashboard_id is valid, browser available.
# @POST Returns list of {tab_name, path} dicts — one per tab.
@@ -714,9 +715,9 @@ class ScreenshotService:
return results
finally:
await browser.close()
# endregion ScreenshotService.capture_dashboard_chunks
# #endregion ScreenshotService.capture_dashboard_chunks
# region ScreenshotService.capture_dashboard [TYPE Function] [C:4]
# #region ScreenshotService.capture_dashboard [TYPE Function] [C:4]
# @PURPOSE Captures multi-chunk screenshots, converts for LLM, archives to WebP.
# @PRE dashboard_id is a valid string, output_path is a writable path.
# @POST Returns list of {original, webp_path} dicts from WebP archive.
@@ -765,9 +766,9 @@ class ScreenshotService:
# 4. Return JPEGs for LLM — caller cleans up after analysis
return jpeg_paths, archive_results
# endregion ScreenshotService.capture_dashboard
# #endregion ScreenshotService.capture_dashboard
# region ScreenshotService._convert_screenshots_for_llm [TYPE Function] [C:2]
# #region ScreenshotService._convert_screenshots_for_llm [TYPE Function] [C:2]
# @BRIEF Convert PNG screenshots to JPEG for LLM transmission.
# @PRE png_paths is a list of existing PNG file paths.
# @POST Returns list of JPEG paths. JPEG files should be deleted after LLM call.
@@ -800,9 +801,9 @@ class ScreenshotService:
logger.explore("Failed to convert screenshot for LLM", payload={"png_path": png_path}, error=str(e))
return jpeg_paths
# endregion ScreenshotService._convert_screenshots_for_llm
# #endregion ScreenshotService._convert_screenshots_for_llm
# region ScreenshotService._archive_screenshots_as_webp [TYPE Function] [C:2]
# #region ScreenshotService._archive_screenshots_as_webp [TYPE Function] [C:2]
# @BRIEF Convert PNG screenshots to WebP for archive.
# @PRE png_paths is a list of existing PNG file paths.
# @POST Returns list of {original, webp_path} dicts. PNG deleted after successful WebP save.
@@ -834,9 +835,9 @@ class ScreenshotService:
results.append({"original": png_path, "webp_path": None})
return results
# endregion ScreenshotService._archive_screenshots_as_webp
# #endregion ScreenshotService._archive_screenshots_as_webp
# region ScreenshotService._cleanup_temp_files [TYPE Function] [C:1]
# #region ScreenshotService._cleanup_temp_files [TYPE Function] [C:1]
# @BRIEF Delete temporary files (PNG, JPEG intermediates).
@staticmethod
def _cleanup_temp_files(paths: list[str]) -> None:
@@ -847,14 +848,15 @@ class ScreenshotService:
os.remove(path)
except Exception as e:
logger.explore("Failed to delete temporary file", payload={"path": path}, error=str(e))
# endregion ScreenshotService._cleanup_temp_files
# #endregion ScreenshotService._cleanup_temp_files
# #endregion ScreenshotService
# #region LLMClient [TYPE Class]
# #region LLMClient [C:4] [TYPE Class] [SEMANTICS llm,client,provider,openai]
# @defgroup LLMAnalysis Module group.
# @BRIEF Wrapper for LLM provider APIs.
# @SIDE_EFFECT Makes HTTP calls to LLM provider APIs.
class LLMClient:
# region LLMClient.__init__ [TYPE Function]
# #region LLMClient.__init__ [TYPE Function]
# @PURPOSE Initializes the LLMClient with provider settings.
# @PRE api_key, base_url, and default_model are non-empty strings.
def __init__(self, provider_type: LLMProviderType, api_key: str, base_url: str, default_model: str):
@@ -911,9 +913,9 @@ class LLMClient:
default_headers=default_headers,
http_client=http_client,
)
# endregion LLMClient.__init__
# #endregion LLMClient.__init__
# region LLMClient._get_ssl_verify [TYPE Function]
# #region LLMClient._get_ssl_verify [TYPE Function]
# @PURPOSE Resolve SSL verification flag from environment.
# @POST Returns SSLContext with system CA dir when enabled,
# False when LLM_SSL_VERIFY env var is "false"/"0"/"no"/"off".
@@ -937,9 +939,9 @@ class LLMClient:
return ssl.create_default_context(capath=ca_dir)
# fallback: если директории нет (редко), используем дефолтный
return ssl.create_default_context()
# endregion LLMClient._get_ssl_verify
# #endregion LLMClient._get_ssl_verify
# region LLMClient._format_connection_error [TYPE Function]
# #region LLMClient._format_connection_error [TYPE Function]
# @PURPOSE Format exception chain for diagnostics, extracting httpx cause details.
# @POST Returns a human-readable string with the full error chain.
@staticmethod
@@ -950,9 +952,9 @@ class LLMClient:
parts.append(f" └─ {type(cause).__name__}: {cause!s}")
cause = cause.__cause__ or cause.__context__
return "\n".join(parts)
# endregion LLMClient._format_connection_error
# #endregion LLMClient._format_connection_error
# region LLMClient._supports_json_response_format [TYPE Function]
# #region LLMClient._supports_json_response_format [TYPE Function]
# @PURPOSE Detect whether provider/model is likely compatible with response_format=json_object.
# @PRE Client initialized with base_url and default_model.
# @POST Returns False for known-incompatible combinations to avoid avoidable 400 errors.
@@ -967,9 +969,9 @@ class LLMClient:
if "stepfun/" in model or model.startswith("step-"):
return False
return True
# endregion LLMClient._supports_json_response_format
# #endregion LLMClient._supports_json_response_format
# region LLMClient.get_json_completion [TYPE Function]
# #region LLMClient.get_json_completion [TYPE Function]
# @PURPOSE Helper to handle LLM calls with JSON mode and fallback parsing.
# @PRE messages is a list of valid message dictionaries.
# @POST Returns a parsed JSON dictionary.
@@ -1107,9 +1109,9 @@ class LLMClient:
return json.loads(json_str)
else:
raise
# endregion LLMClient.get_json_completion
# #endregion LLMClient.get_json_completion
# region LLMClient.test_runtime_connection [TYPE Function]
# #region LLMClient.test_runtime_connection [TYPE Function]
# @PURPOSE Validate provider credentials using the same chat completions transport as runtime analysis.
# @PRE Client is initialized with provider credentials and default_model.
# @POST Returns lightweight JSON payload when runtime auth/model path is valid.
@@ -1123,9 +1125,9 @@ class LLMClient:
}
]
return await self.get_json_completion(messages)
# endregion LLMClient.test_runtime_connection
# #endregion LLMClient.test_runtime_connection
# region LLMClient.fetch_models [TYPE Function]
# #region LLMClient.fetch_models [TYPE Function]
# @PURPOSE Fetch available models from the provider's API.
# @PRE Client is initialized with provider credentials.
# @POST Returns a list of model ID strings.
@@ -1148,9 +1150,9 @@ class LLMClient:
error=str(e),
)
raise
# endregion LLMClient.fetch_models
# #endregion LLMClient.fetch_models
# region LLMClient.analyze_dashboard [TYPE Function]
# #region LLMClient.analyze_dashboard [TYPE Function]
# @PURPOSE Sends dashboard data (screenshot + logs) to LLM for health analysis.
# @PRE screenshot_path exists, logs is a list of strings.
# @POST Returns a structured analysis dictionary (status, summary, issues).
@@ -1169,9 +1171,9 @@ class LLMClient:
logs=logs,
prompt_template=prompt_template,
)
# endregion LLMClient.analyze_dashboard
# #endregion LLMClient.analyze_dashboard
# region LLMClient._reduce_image_quality [TYPE Function] [C:2]
# #region LLMClient._reduce_image_quality [TYPE Function] [C:2]
# @PURPOSE Open, resize, and compress a screenshot image for LLM consumption.
# @PRE path points to an existing image file.
# @POST Returns (base64_str, byte_size) tuple.
@@ -1199,9 +1201,9 @@ class LLMClient:
img.save(buffer, format="JPEG", quality=image_quality, optimize=True)
raw = buffer.getvalue()
return base64.b64encode(raw).decode("utf-8"), len(raw)
# endregion LLMClient._reduce_image_quality
# #endregion LLMClient._reduce_image_quality
# region LLMClient._estimate_payload_size [TYPE Function] [C:2]
# #region LLMClient._estimate_payload_size [TYPE Function] [C:2]
# @PURPOSE Estimate LLM payload size in tokens before sending.
# @POST Returns {estimated_tokens, exceeds_limit, pct_of_limit} dict.
# @RATIONALE FR-056: if >80% of model context window, trigger quality reduction.
@@ -1226,9 +1228,9 @@ class LLMClient:
"exceeds_limit": exceeds_limit,
"pct_of_limit": round(total_tokens / model_context * 100, 1),
}
# endregion LLMClient._estimate_payload_size
# #endregion LLMClient._estimate_payload_size
# region LLMClient._deduplicate_issues [TYPE Function] [C:2]
# #region LLMClient._deduplicate_issues [TYPE Function] [C:2]
# @PURPOSE Deduplicate issues by (severity, message, location) while preserving order.
def _deduplicate_issues(self, issues: list[dict]) -> list[dict]:
seen: set[tuple[str, str, str]] = set()
@@ -1240,9 +1242,9 @@ class LLMClient:
result.append(issue)
return result
# endregion LLMClient._deduplicate_issues
# #endregion LLMClient._deduplicate_issues
# region LLMClient._optimize_images [TYPE Function] [C:2]
# #region LLMClient._optimize_images [TYPE Function] [C:2]
# @PURPOSE Convert screenshot paths to base64 at given quality, with fallback to raw read.
def _optimize_images(self, paths: list[str], max_width: int, quality: int) -> list[str]:
encoded: list[str] = []
@@ -1258,9 +1260,9 @@ class LLMClient:
encoded.append(b64)
return encoded
# endregion LLMClient._optimize_images
# #endregion LLMClient._optimize_images
# region LLMClient._merge_chunk_results [TYPE Function] [C:2]
# #region LLMClient._merge_chunk_results [TYPE Function] [C:2]
# @PURPOSE Merge multiple chunk analyses into one. Takes the worst status,
# concatenates summaries, and deduplicates issues.
# @PRE chunks is a non-empty list of {status, summary, issues} dicts.
@@ -1286,9 +1288,9 @@ class LLMClient:
}
return merged
# endregion LLMClient._merge_chunk_results
# #endregion LLMClient._merge_chunk_results
# region LLMClient._call_llm_for_images [TYPE Function] [C:2]
# #region LLMClient._call_llm_for_images [TYPE Function] [C:2]
# @PURPOSE Send a single chunk of images to the LLM and return parsed result.
async def _call_llm_for_images(
self, encoded_images: list[str], prompt: str
@@ -1302,9 +1304,9 @@ class LLMClient:
messages = [{"role": "user", "content": content}]
return await self.get_json_completion(messages)
# endregion LLMClient._call_llm_for_images
# #endregion LLMClient._call_llm_for_images
# region LLMClient.analyze_dashboard_multimodal [TYPE Function] [C:3]
# #region LLMClient.analyze_dashboard_multimodal [TYPE Function] [C:3]
# @PURPOSE Path A: send screenshots + logs to multimodal LLM, with chunking support.
# @PRE screenshot_paths is a non-empty list of paths.
# tab_labels, if provided, must have the same length as screenshot_paths.
@@ -1404,9 +1406,9 @@ class LLMClient:
}
return result
# endregion LLMClient.analyze_dashboard_multimodal
# #endregion LLMClient.analyze_dashboard_multimodal
# region LLMClient.analyze_dashboard_text_batch [TYPE Function] [C:3]
# #region LLMClient.analyze_dashboard_text_batch [TYPE Function] [C:3]
# @PURPOSE Path B batch: multiple dashboards in a single text-only LLM call.
# @PRE payloads is a non-empty list of {dashboard_id, topology, dataset_health, log_text} dicts.
# @POST Returns dict {dashboards: [{dashboard_id, status, summary, issues}]}.
@@ -1462,7 +1464,7 @@ class LLMClient:
messages = [{"role": "user", "content": full_prompt}]
return await self.get_json_completion(messages)
# endregion LLMClient.analyze_dashboard_text_batch
# #endregion LLMClient.analyze_dashboard_text_batch
# #endregion LLMClient
@@ -1476,15 +1478,15 @@ class LLMClient:
# are invisible to the LLM validation. Screenshot captures visual state but doesn't
# verify that data actually arrived (vs. cache).
class DatasetHealthChecker:
# region DatasetHealthChecker.__init__ [TYPE Function]
# #region DatasetHealthChecker.__init__ [TYPE Function]
# @PURPOSE Initialize with a SupersetClient-compatible instance.
# @PRE client is a SupersetClient (sync, wrapped via asyncio.to_thread) or AsyncSupersetClient.
# @POST self.client is ready for health checks.
def __init__(self, client: Any):
self.client = client
# endregion DatasetHealthChecker.__init__
# #endregion DatasetHealthChecker.__init__
# region DatasetHealthChecker._call_sync [TYPE Function]
# #region DatasetHealthChecker._call_sync [TYPE Function]
# @PURPOSE Wrap a sync client method call in asyncio.to_thread for async compat.
# @PRE method is a callable on self.client.
# @POST Returns the result of method(*args, **kwargs) executed in a thread.
@@ -1494,9 +1496,9 @@ class DatasetHealthChecker:
if asyncio.iscoroutinefunction(method):
return await method(*args, **kwargs)
return await asyncio.to_thread(method, *args, **kwargs)
# endregion DatasetHealthChecker._call_sync
# #endregion DatasetHealthChecker._call_sync
# region DatasetHealthChecker.check_dataset_health [TYPE Function]
# #region DatasetHealthChecker.check_dataset_health [TYPE Function]
# @PURPOSE Fetch dataset metadata and verify level 1-2 accessibility.
# @PRE dataset_id is a valid Superset dataset ID.
# @POST Returns dict with level 1-2 health fields.
@@ -1538,9 +1540,9 @@ class DatasetHealthChecker:
"metadata_accessible": False,
"error": str(e),
}
# endregion DatasetHealthChecker.check_dataset_health
# #endregion DatasetHealthChecker.check_dataset_health
# region DatasetHealthChecker.check_chart_data [TYPE Function]
# #region DatasetHealthChecker.check_chart_data [TYPE Function]
# @PURPOSE Execute chart data query (level 3-4 per FR-044).
# @PRE chart_id is valid, form_data is constructed from chart params.
# @POST Returns dict with execution result.
@@ -1596,9 +1598,9 @@ class DatasetHealthChecker:
"row_count": None,
"error": str(e),
}
# endregion DatasetHealthChecker.check_chart_data
# #endregion DatasetHealthChecker.check_chart_data
# region DatasetHealthChecker.check_dashboard_datasets [TYPE Function]
# #region DatasetHealthChecker.check_dashboard_datasets [TYPE Function]
# @PURPOSE For every unique dataset in dashboard charts, check health.
# @PRE chart_list has chart dicts with slice_id and datasource_id.
# @POST Returns dict with datasets and optional chart_data lists.
@@ -1664,7 +1666,7 @@ class DatasetHealthChecker:
"datasets": dataset_results,
"chart_data": chart_data_results,
}
# endregion DatasetHealthChecker.check_dashboard_datasets
# #endregion DatasetHealthChecker.check_dashboard_datasets
# #endregion DatasetHealthChecker
# #region RedactionService [C:2] [TYPE Module]
@@ -1688,7 +1690,7 @@ class RedactionService:
(r"[\w.+-]+@[\w-]+\.[\w.-]+", "***@***"), # emails
]
# region RedactionService.redact_logs [TYPE Function] [C:2]
# #region RedactionService.redact_logs [TYPE Function] [C:2]
# @BRIEF Redact PII/credentials from log lines.
# @PRE logs is a list of strings.
# @POST Returns redacted list preserving structure.
@@ -1701,9 +1703,9 @@ class RedactionService:
line = re.sub(pattern, replacement, line, flags=re.IGNORECASE)
redacted.append(line)
return redacted
# endregion RedactionService.redact_logs
# #endregion RedactionService.redact_logs
# region RedactionService.redact_raw_response [TYPE Function] [C:2]
# #region RedactionService.redact_raw_response [TYPE Function] [C:2]
# @BRIEF Redact sensitive data from LLM raw response.
# @PRE raw is a string.
# @POST Returns redacted string.
@@ -1713,7 +1715,7 @@ class RedactionService:
for pattern, replacement in RedactionService.PATTERNS:
raw = re.sub(pattern, replacement, raw, flags=re.IGNORECASE)
return raw
# endregion RedactionService.redact_raw_response
# #endregion RedactionService.redact_raw_response
# #endregion RedactionService
# #endregion LLMAnalysisService

View File

@@ -51,8 +51,8 @@ class TranslationExecutionEngine:
self._sql_service = SQLInsertService(db, config_manager, event_log)
self._aggregator = TranslationResultAggregator(db, event_log)
# region execute_run [TYPE Function]
# @PURPOSE: Execute a translation run: dispatch executor, handle outcomes.
# #region TranslationExecutionEngine.execute_run [C:4] [TYPE Function] [SEMANTICS translate,execution,run]
# @BRIEF Execute a translation run: dispatch executor, handle outcomes.
# @PRE run is in PENDING status.
# @POST Run executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
@@ -94,10 +94,10 @@ class TranslationExecutionEngine:
self.db, self.event_log, self._aggregator, self._sql_service,
run, job, skip_insert, language_stats_map, self.current_user,
)
# endregion execute_run
# #endregion TranslationExecutionEngine.execute_run
# region _init_language_stats [TYPE Function]
# @PURPOSE: Initialize per-language stats for a run.
# #region TranslationExecutionEngine._init_language_stats [C:2] [TYPE Function] [SEMANTICS translate,language,stats]
# @BRIEF Initialize per-language stats for a run.
def _init_language_stats(
self,
run: TranslationRun,
@@ -117,6 +117,6 @@ class TranslationExecutionEngine:
language_stats_map[lang_code] = lang_stat
self.db.flush()
return language_stats_map
# endregion _init_language_stats
# #endregion TranslationExecutionEngine._init_language_stats
# #endregion TranslationExecutionEngine