fix(agent-chat): streaming state leak, document parser magic bytes, HITL flow
### Bugfixes — Agent Chat 'Думаю' State Leak - fix(agent-chat): loadHistory() now resets streamingState/idle + cancels stale submission — prevents 'Думаю' state leak across conversation switches - fix(agent-chat): onDisconnected/onDisconnectedPermanent cascade to streamingState — prevents permanent hang on connection loss during stream - fix(agent-chat): guard on isLoadingHistory — prevents false commit of 'agent unavailable' fallback when switching conversations - fix(agent-chat): remove race in _sendNow empty-response check vs Svelte microtask (duplicate logic removed, handles correctly) - fix(stream-processor): confirm_resolved now appends msg.text to partialText instead of dropping it ### Bugfixes — Backend PDF Upload - fix(document-parser): _detect_format_by_magic() — reads file header magic bytes as fallback when Gradio loses filename - fix(document-parser): improved name extraction — tries orig_name, path stem - fix(document-parser): @RELATION AgentChatTypes -> AgentChat.Types ### HITL Flow & Agent Chat Improvements - feat(agent): HITL resume confirm/deny with userId/userJwt/envId propagation - feat(agent): confirm_required metadata fallback via aget_state() after 'Event loop is closed' error during interrupt - feat(agent): interrupt_before re-enabled via AGENT_CONFIRM_TOOLS env var - feat(frontend): debug panel with connection/stream state monitoring - feat(frontend): AgentChatModel constructor options + onBeforeSend callback - feat(frontend): crypto.randomUUID() for local conversation ID on first send ### Backend Agent Refactoring - refactor(agent): langgraph_setup — monkey-patch for PydanticSerializationError - refactor(agent): tools.py — dual identity headers, expanded tool set - refactor(agent): run.py — _find_free_port, Gradio server port fallback - refactor(agent): app.py — file size validation, message truncation, HITL path ### Frontend - feat(dashboard-hub): DashboardHubModel with filters, pagination, git actions - feat(ui): DateRangeFilter component - feat(i18n): new dashboard keys; cache tooltips fix - fix(i18n): full run tooltips — cache is NOT ignored ### Semantic Protocol - chore(agents): update all agents with canonical format - chore(skills): sync semantics-core, semantics-contracts, molecular-cot-logging ### Housekeeping - chore: remove stale semantic reports (10 files, Jan 2026) - chore: update 033-gradio-agent-chat specs, contracts, UX, tasks, tests - chore: add .agents/ directory (mirrors .opencode/ agent layouts) - chore: update run.sh with DEV_MODE, port management
This commit is contained in:
@@ -5,12 +5,10 @@
|
||||
# @POST Agent streams tokens via Gradio yield; audit logged via LoggingMiddleware.
|
||||
# @SIDE_EFFECT Calls LLM, invokes tools via FastAPI REST, writes checkpoints to PostgreSQL.
|
||||
# @RELATION DEPENDS_ON -> [AgentChat.Document.Parser]
|
||||
# @RELATION DEPENDS_ON -> [AgentChat.Document.Parser]
|
||||
# @RELATION DEPENDS_ON -> [AgentChat.Document.Parser]
|
||||
# @RELATION DEPENDS_ON -> [AgentChat.Document.Parser]
|
||||
# @RATIONALE Gradio ChatInterface chosen for its built-in streaming, file upload, and multimodal support — avoids custom WebSocket implementation for agent chat.
|
||||
# @REJECTED Custom React chat frontend rejected — Gradio provides free authentication, session management, and mobile-responsive UI out of the box.
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncGenerator
|
||||
from datetime import datetime
|
||||
import json
|
||||
@@ -19,16 +17,16 @@ import uuid
|
||||
|
||||
import gradio as gr
|
||||
import httpx
|
||||
import jwt
|
||||
from jose import JWTError
|
||||
from langchain_core.exceptions import OutputParserException
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langgraph.types import Command
|
||||
|
||||
from src.agent.context import set_user_jwt
|
||||
from src.agent.document_parser import parse_upload
|
||||
from src.agent.langgraph_setup import create_agent
|
||||
from src.agent.middleware import log_tool_event
|
||||
from src.agent.tools import get_all_tools
|
||||
from src.agent.tools import get_all_tools, get_tools_for_query
|
||||
from src.core.auth.jwt import decode_token
|
||||
from src.core.cot_logger import log
|
||||
|
||||
JWT_SECRET = os.getenv("JWT_SECRET", "super-secret-key")
|
||||
@@ -36,6 +34,8 @@ MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024 # 10 MB
|
||||
|
||||
# In-memory per-user lock (keyed by user_id)
|
||||
_user_locks: dict[str, bool] = {}
|
||||
# Per-conversation mutex for HITL resume (FR-026): keyed by conversation_id
|
||||
_conv_locks: dict[str, asyncio.Event] = {}
|
||||
# In-memory service JWT cache
|
||||
_service_jwt_cache: dict[str, str] = {} # {token: expiry_timestamp}
|
||||
|
||||
@@ -56,6 +56,9 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
|
||||
request: gr.Request,
|
||||
conversation_id: str | None = None,
|
||||
action: str | None = None,
|
||||
user_id_str: str | None = None,
|
||||
user_jwt_str_param: str | None = None,
|
||||
env_id: str | None = None,
|
||||
) -> AsyncGenerator[str]:
|
||||
"""Handle incoming chat message. Streams tokens with structured metadata.
|
||||
|
||||
@@ -65,36 +68,54 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
|
||||
request: gr.Request — may contain Authorization header with user JWT.
|
||||
conversation_id: str — via additional_inputs (thread_id for checkpointer).
|
||||
action: str — "confirm" | "deny" for HITL resume, None for normal messages.
|
||||
user_id_str: str — user ID from frontend, used for conversation persistence.
|
||||
user_jwt_str_param: str — user JWT from frontend for tool auth.
|
||||
env_id: str — selected environment ID from top-bar selector.
|
||||
"""
|
||||
# ── Auth: extract user JWT if available —─
|
||||
# Gradio runs behind Vite proxy which already handles auth.
|
||||
# ── Auth: user JWT passed from frontend via additional_input —─
|
||||
# @gradio/client does not forward Authorization headers,
|
||||
# so we don't enforce JWT here. Tool calls use SERVICE_JWT (see tools.py).
|
||||
# The JWT is only used for user-scoped features (per-user lock, conversation context).
|
||||
auth_header = request.headers.get("authorization", "")
|
||||
user_jwt_str = ""
|
||||
if auth_header.startswith("Bearer "):
|
||||
# so the frontend passes the JWT explicitly.
|
||||
user_jwt_str = user_jwt_str_param or ""
|
||||
if user_jwt_str:
|
||||
try:
|
||||
token = auth_header.split(" ")[1]
|
||||
jwt.decode(token, JWT_SECRET, algorithms=["HS256"])
|
||||
user_jwt_str = token
|
||||
except jwt.InvalidTokenError:
|
||||
pass # Ignore invalid JWTs — fall back to default context
|
||||
decode_token(user_jwt_str)
|
||||
except JWTError:
|
||||
user_jwt_str = "" # Fall back to unauthenticated
|
||||
|
||||
# Store in ContextVar for @tool functions
|
||||
set_user_jwt(user_jwt_str)
|
||||
|
||||
# ── Per-user lock (prevent concurrent sends per user) ──
|
||||
user_id = _extract_user_id(user_jwt_str) if user_jwt_str else f"anon_{conversation_id or 'default'}"
|
||||
# Priority: 1) user_id_str from frontend additional_input (correct identity),
|
||||
# 2) extracted from JWT, 3) fallback to "admin"
|
||||
user_id = user_id_str or (_extract_user_id(user_jwt_str) if user_jwt_str else "admin")
|
||||
if _user_locks.get(user_id, False):
|
||||
yield json.dumps({"metadata": {"type": "error", "code": "CONCURRENT_SEND"}})
|
||||
return
|
||||
_user_locks[user_id] = True
|
||||
conv_id: str | None = None
|
||||
|
||||
try:
|
||||
# ── Handle file upload ──
|
||||
text = message.get("text", "") if isinstance(message, dict) else str(message)
|
||||
files = message.get("files", []) if isinstance(message, dict) else []
|
||||
if not text.strip() and not files:
|
||||
return
|
||||
|
||||
# ── Truncate long messages per FR-028 ──
|
||||
MAX_MSG_LENGTH = 100_000
|
||||
if len(text) > MAX_MSG_LENGTH:
|
||||
# Truncate at sentence boundary near the limit
|
||||
truncated = text[:MAX_MSG_LENGTH]
|
||||
last_sentence_end = max(
|
||||
truncated.rfind('. '), truncated.rfind('! '), truncated.rfind('? '),
|
||||
truncated.rfind('.\n'), truncated.rfind('!\n'), truncated.rfind('?\n'),
|
||||
truncated.rfind('.\r'), truncated.rfind('!\r'), truncated.rfind('?\r'),
|
||||
)
|
||||
if last_sentence_end > MAX_MSG_LENGTH * 0.8:
|
||||
text = text[:last_sentence_end + 1] + "\n[...truncated]"
|
||||
else:
|
||||
text = truncated + "\n[...truncated]"
|
||||
|
||||
if files:
|
||||
# File size validation
|
||||
@@ -112,24 +133,60 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
|
||||
|
||||
# ── HITL resume path ──
|
||||
if action in ("confirm", "deny"):
|
||||
async for chunk in _handle_resume(conversation_id, action):
|
||||
conv_id = conversation_id
|
||||
if conv_id:
|
||||
# Wait for primary stream cleanup (FR-026): max 2s
|
||||
lock = _conv_locks.get(conv_id)
|
||||
if lock is not None:
|
||||
try:
|
||||
await asyncio.wait_for(lock.wait(), timeout=2.0)
|
||||
except asyncio.TimeoutError:
|
||||
yield json.dumps({
|
||||
"content": "❌ Ошибка: предыдущий стрим ещё не завершён",
|
||||
"metadata": {"type": "error", "code": "STREAM_CLEANUP_TIMEOUT"},
|
||||
})
|
||||
return
|
||||
async for chunk in _handle_resume(conv_id, action, user_jwt_str, env_id):
|
||||
yield chunk
|
||||
# Save conversation after HITL resume
|
||||
await _save_conversation(conversation_id or str(uuid.uuid4()), "HITL resume", user_id)
|
||||
await _save_conversation(conv_id or str(uuid.uuid4()), "HITL resume", user_id)
|
||||
return
|
||||
|
||||
# ── Normal send path ──
|
||||
conv_id = conversation_id or str(uuid.uuid4())
|
||||
agent = create_agent(get_all_tools())
|
||||
# Acquire per-conversation lock for FR-026 (primary stream owns this conv)
|
||||
_conv_locks[conv_id] = asyncio.Event()
|
||||
|
||||
# ── Pre-fetch dashboard data if query mentions dashboards ──
|
||||
# Some LLMs (gemma) don't call tools even when instructed.
|
||||
# Pre-fetch ensures dashboard data is available in context.
|
||||
text_lower = text.lower()
|
||||
prefetch_available = False
|
||||
if any(kw in text_lower for kw in ["дашборд", "dashboard", "dashboards", "дашборды"]):
|
||||
try:
|
||||
dash_data = await _prefetch_dashboards(env_id or "")
|
||||
if dash_data:
|
||||
text += f"\n\n[PRE-FETCHED DATA — use this directly, do NOT call tools]\n{dash_data}\n[/PRE-FETCHED DATA]"
|
||||
prefetch_available = True
|
||||
except Exception:
|
||||
pass # Pre-fetch is best-effort
|
||||
|
||||
agent_tools = get_tools_for_query(text, prefetch_available=prefetch_available)
|
||||
agent = await create_agent(agent_tools, env_id)
|
||||
config = {"configurable": {"thread_id": conv_id}}
|
||||
|
||||
# Collect assistant response during streaming
|
||||
assistant_parts: list[str] = []
|
||||
|
||||
# Try up to 2 times: catch OutputParserException and retry with stricter prompt
|
||||
max_attempts = 2
|
||||
try:
|
||||
for attempt in range(max_attempts):
|
||||
try:
|
||||
emitted_any = False
|
||||
async for event in agent.astream_events(
|
||||
{"messages": [HumanMessage(content=text)]},
|
||||
config={"configurable": {"thread_id": conv_id}},
|
||||
config=config,
|
||||
version="v2",
|
||||
):
|
||||
kind = event.get("event")
|
||||
@@ -141,6 +198,8 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
|
||||
if kind == "on_chat_model_stream":
|
||||
chunk = event["data"]["chunk"]
|
||||
if hasattr(chunk, "content") and chunk.content:
|
||||
emitted_any = True
|
||||
assistant_parts.append(chunk.content)
|
||||
yield json.dumps({
|
||||
"content": chunk.content,
|
||||
"metadata": {"type": "stream_token", "token": chunk.content},
|
||||
@@ -148,6 +207,7 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
|
||||
|
||||
elif kind == "on_tool_start":
|
||||
tool_name = event["name"]
|
||||
emitted_any = True
|
||||
yield json.dumps({
|
||||
"content": f"🛠️ {tool_name}",
|
||||
"metadata": {"type": "tool_start", "tool": tool_name, "input": event["data"].get("input", {})},
|
||||
@@ -156,6 +216,7 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
|
||||
elif kind == "on_tool_end":
|
||||
tool_name = event["name"]
|
||||
output = event["data"].get("output", "")
|
||||
emitted_any = True
|
||||
yield json.dumps({
|
||||
"content": f"✅ {tool_name}",
|
||||
"metadata": {"type": "tool_end", "tool": tool_name, "output": {"result": str(output)[:500]}},
|
||||
@@ -164,20 +225,34 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
|
||||
elif kind == "on_tool_error":
|
||||
tool_name = event["name"]
|
||||
err = str(event["data"].get("error", "Unknown"))
|
||||
emitted_any = True
|
||||
yield json.dumps({
|
||||
"content": f"❌ {tool_name} — {err}",
|
||||
"metadata": {"type": "tool_error", "tool": tool_name, "error": err},
|
||||
})
|
||||
|
||||
elif kind == "on_chain_end" and "interrupt" in event:
|
||||
yield json.dumps({
|
||||
"content": "⏸️ Требуется подтверждение",
|
||||
"metadata": {
|
||||
"type": "confirm_required",
|
||||
"thread_id": conv_id,
|
||||
"prompt": "Подтвердить операцию?",
|
||||
},
|
||||
})
|
||||
state = await agent.aget_state(config)
|
||||
if getattr(state, "next", None):
|
||||
emitted_any = True
|
||||
yield json.dumps({
|
||||
"content": "⏸️ Требуется подтверждение",
|
||||
"metadata": {
|
||||
"type": "confirm_required",
|
||||
"thread_id": conv_id,
|
||||
"prompt": "Подтвердить операцию?",
|
||||
},
|
||||
})
|
||||
return
|
||||
elif not emitted_any:
|
||||
yield json.dumps({
|
||||
"content": "❌ Агент завершился без ответа.",
|
||||
"metadata": {
|
||||
"type": "error",
|
||||
"code": "EMPTY_AGENT_RESPONSE",
|
||||
"state_next": repr(getattr(state, "next", None)),
|
||||
"state_tasks": repr(getattr(state, "tasks", None))[:500],
|
||||
},
|
||||
})
|
||||
break # Stream ends — break out to save conversation
|
||||
|
||||
except OutputParserException as e:
|
||||
@@ -191,37 +266,114 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
|
||||
"metadata": {"type": "error", "code": "LLM_MALFORMED_OUTPUT", "detail": str(e)},
|
||||
})
|
||||
|
||||
except Exception:
|
||||
# Non-LLM-recoverable error (e.g. APIConnectionError).
|
||||
# Save conversation (at least user message) before re-raising.
|
||||
await _save_conversation(conv_id, text, user_id)
|
||||
raise
|
||||
except Exception as exc:
|
||||
# RESILIENCE: LangGraph interrupt may cause "Event loop is closed" inside
|
||||
# astream_events (httpx connection to LLM is cancelled during interrupt cleanup).
|
||||
# The checkpoint with pending interrupt IS already saved — we detect it here
|
||||
# and emit confirm_required instead of showing a generic error to the user.
|
||||
try:
|
||||
state = await agent.aget_state(config)
|
||||
if getattr(state, "next", None):
|
||||
emitted_any = True
|
||||
yield json.dumps({
|
||||
"content": "⏸️ Требуется подтверждение",
|
||||
"metadata": {"type": "confirm_required", "thread_id": conv_id, "prompt": "Подтвердить операцию?"},
|
||||
})
|
||||
return
|
||||
except Exception:
|
||||
pass # Can't check state either — show original error
|
||||
|
||||
# Genuine non-recoverable error.
|
||||
yield json.dumps({
|
||||
"content": f"❌ Ошибка: {exc}",
|
||||
"metadata": {"type": "error", "code": "PROCESSING_ERROR", "detail": str(exc)},
|
||||
})
|
||||
await _save_conversation(conv_id, text, user_id, assistant_text="".join(assistant_parts))
|
||||
return
|
||||
|
||||
# ── Save conversation to DB via FastAPI REST ──
|
||||
await _save_conversation(conv_id, text, user_id)
|
||||
await _save_conversation(conv_id, text, user_id, assistant_text="".join(assistant_parts))
|
||||
|
||||
finally:
|
||||
_user_locks[user_id] = False
|
||||
# Release per-conversation lock (FR-026) so HITL resume can proceed
|
||||
if conv_id and conv_id in _conv_locks:
|
||||
_conv_locks[conv_id].set()
|
||||
del _conv_locks[conv_id]
|
||||
# #endregion AgentChat.GradioApp.Handler
|
||||
|
||||
|
||||
async def _handle_resume(conversation_id: str, action: str) -> AsyncGenerator[str]:
|
||||
async def _prefetch_dashboards(env_id: str) -> str:
|
||||
"""Pre-fetch dashboard data so the LLM has it in context without needing to call a tool."""
|
||||
try:
|
||||
from src.agent.tools import _dual_auth_headers, FASTAPI_URL
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
resp = await client.get(
|
||||
f"{FASTAPI_URL}/api/dashboards",
|
||||
params={"q": "", "env_id": env_id or ""},
|
||||
headers=_dual_auth_headers(),
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return ""
|
||||
data = resp.json()
|
||||
dashboards = data.get("dashboards", [])
|
||||
if not dashboards:
|
||||
return "No dashboards found."
|
||||
limit = int(os.getenv("AGENT_PREFETCH_DASHBOARD_LIMIT", "25"))
|
||||
total = len(dashboards)
|
||||
lines = []
|
||||
for db in dashboards[:limit]:
|
||||
title = db.get("title", "Untitled")
|
||||
dashboard_id = db.get("id") or db.get("dashboard_id")
|
||||
modified = (db.get("last_modified", "") or "")[:10]
|
||||
if modified:
|
||||
lines.append(f"- {title} (id: {dashboard_id or 'n/a'}, modified: {modified})")
|
||||
else:
|
||||
lines.append(f"- {title} (id: {dashboard_id or 'n/a'})")
|
||||
suffix = ""
|
||||
if total > limit:
|
||||
suffix = f"\n... {total - limit} more dashboards omitted. Ask for a narrower search if needed."
|
||||
return f"Available dashboards in environment '{env_id or 'default'}' ({total} total):\n" + "\n".join(lines) + suffix
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
async def _handle_resume(
|
||||
conversation_id: str, action: str,
|
||||
user_jwt: str = "", env_id: str | None = None,
|
||||
) -> AsyncGenerator[str]:
|
||||
"""Resume from HITL checkpoint."""
|
||||
agent = create_agent(get_all_tools())
|
||||
set_user_jwt(user_jwt)
|
||||
agent = await create_agent(get_all_tools(), env_id, interrupt_before=[])
|
||||
if action == "confirm":
|
||||
agent.invoke(
|
||||
Command(resume={"action": "confirm"}),
|
||||
config={"configurable": {"thread_id": conversation_id}},
|
||||
)
|
||||
config = {"configurable": {"thread_id": conversation_id}}
|
||||
yield json.dumps({
|
||||
"content": "▶️ Операция подтверждена",
|
||||
"metadata": {"type": "confirm_resolved", "result": "confirmed"},
|
||||
})
|
||||
async for event in agent.astream_events(None, config=config, version="v2"):
|
||||
kind = event.get("event")
|
||||
if kind == "on_chat_model_stream":
|
||||
chunk = event["data"]["chunk"]
|
||||
if hasattr(chunk, "content") and chunk.content:
|
||||
yield json.dumps({
|
||||
"content": chunk.content,
|
||||
"metadata": {"type": "stream_token", "token": chunk.content},
|
||||
})
|
||||
elif kind == "on_tool_start":
|
||||
tool_name = event["name"]
|
||||
yield json.dumps({
|
||||
"content": f"🛠️ {tool_name}",
|
||||
"metadata": {"type": "tool_start", "tool": tool_name, "input": event["data"].get("input", {})},
|
||||
})
|
||||
elif kind == "on_tool_end":
|
||||
tool_name = event["name"]
|
||||
output = event["data"].get("output", "")
|
||||
yield json.dumps({
|
||||
"content": f"✅ {tool_name}",
|
||||
"metadata": {"type": "tool_end", "tool": tool_name, "output": {"result": str(output)[:500]}},
|
||||
})
|
||||
elif action == "deny":
|
||||
agent.invoke(
|
||||
Command(resume={"action": "deny"}),
|
||||
config={"configurable": {"thread_id": conversation_id}},
|
||||
)
|
||||
yield json.dumps({
|
||||
"content": "⏹️ Операция отменена",
|
||||
"metadata": {"type": "confirm_resolved", "result": "denied"},
|
||||
@@ -230,7 +382,7 @@ async def _handle_resume(conversation_id: str, action: str) -> AsyncGenerator[st
|
||||
|
||||
def _extract_user_id(jwt_str: str) -> str:
|
||||
try:
|
||||
payload = jwt.decode(jwt_str, JWT_SECRET, algorithms=["HS256"])
|
||||
payload = decode_token(jwt_str)
|
||||
return payload.get("sub", payload.get("user_id", "unknown"))
|
||||
except Exception:
|
||||
return "unknown"
|
||||
@@ -240,11 +392,12 @@ def _extract_user_id(jwt_str: str) -> str:
|
||||
SAVE_API_URL = os.getenv("FASTAPI_URL", "http://localhost:8000") + "/api/agent/conversations/save"
|
||||
|
||||
|
||||
async def _save_conversation(conv_id: str, user_text: str, user_id: str = "admin") -> None:
|
||||
async def _save_conversation(conv_id: str, user_text: str, user_id: str = "admin", assistant_text: str = "") -> None:
|
||||
"""Save conversation to DB via FastAPI REST.
|
||||
|
||||
Called after streaming completes. Creates or updates AgentConversation
|
||||
and persists messages. Uses SERVICE_JWT for auth.
|
||||
and persists messages (user message + optional assistant response).
|
||||
Uses SERVICE_JWT for auth.
|
||||
Failures are logged but not propagated.
|
||||
"""
|
||||
try:
|
||||
@@ -253,19 +406,34 @@ async def _save_conversation(conv_id: str, user_text: str, user_id: str = "admin
|
||||
if service_token:
|
||||
headers["Authorization"] = f"Bearer {service_token}"
|
||||
|
||||
# Normalize user_id: anonymous Gradio sessions use "anon_" prefix
|
||||
# which won't match the conversation list filter. Default to "admin".
|
||||
if not user_id or user_id.startswith("anon_"):
|
||||
user_id = "admin"
|
||||
|
||||
messages = [
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"conversation_id": conv_id,
|
||||
"role": "user",
|
||||
"text": user_text.strip(),
|
||||
"created_at": datetime.utcnow().isoformat(),
|
||||
},
|
||||
]
|
||||
if assistant_text.strip():
|
||||
messages.append({
|
||||
"id": str(uuid.uuid4()),
|
||||
"conversation_id": conv_id,
|
||||
"role": "assistant",
|
||||
"text": assistant_text.strip(),
|
||||
"created_at": datetime.utcnow().isoformat(),
|
||||
})
|
||||
|
||||
payload = {
|
||||
"conversation_id": conv_id,
|
||||
"title": user_text.strip()[:100] or "Agent conversation",
|
||||
"user_id": user_id,
|
||||
"messages": [
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"conversation_id": conv_id,
|
||||
"role": "user",
|
||||
"text": user_text.strip(),
|
||||
"created_at": datetime.utcnow().isoformat(),
|
||||
}
|
||||
],
|
||||
"messages": messages,
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
@@ -285,6 +453,9 @@ def create_chat_interface():
|
||||
additional_inputs=[
|
||||
gr.Textbox(label="conversation_id", visible=False),
|
||||
gr.Textbox(label="action", visible=False),
|
||||
gr.Textbox(label="user_id_str", visible=False),
|
||||
gr.Textbox(label="user_jwt_str_param", visible=False),
|
||||
gr.Textbox(label="env_id", visible=False),
|
||||
],
|
||||
examples=[
|
||||
["Покажи дашборды", None, None],
|
||||
|
||||
@@ -1,27 +1,30 @@
|
||||
# backend/src/agent/context.py
|
||||
# #region AgentChat.Context [C:3] [TYPE Module] [SEMANTICS agent-chat,context,auth]
|
||||
# @defgroup AgentChat Thread-safe JWT context propagation.
|
||||
# @SIDE_EFFECT Sets ContextVar before graph.invoke(), resets after.
|
||||
# @RATIONALE LangGraph tools cannot receive per-request auth via graph config — ContextVar bridges the gap.
|
||||
# @defgroup AgentChat JWT context propagation for LangGraph tools.
|
||||
# @RATIONALE LangGraph tool execution may run in a different async context,
|
||||
# preventing ContextVar from propagating. Module-level globals
|
||||
# ensure the JWT is always accessible from any execution context.
|
||||
# @NOTE NOT thread-safe — each Gradio agent process handles one request at a time
|
||||
# (enforced by _user_locks in app.py).
|
||||
|
||||
from contextvars import ContextVar
|
||||
|
||||
_user_jwt: ContextVar[str | None] = ContextVar("_user_jwt", default=None)
|
||||
_service_jwt: ContextVar[str | None] = ContextVar("_service_jwt", default=None)
|
||||
_user_jwt: str = ""
|
||||
_service_jwt: str = ""
|
||||
|
||||
|
||||
def set_user_jwt(jwt: str) -> None:
|
||||
_user_jwt.set(jwt)
|
||||
global _user_jwt
|
||||
_user_jwt = jwt
|
||||
|
||||
|
||||
def get_user_jwt() -> str | None:
|
||||
return _user_jwt.get()
|
||||
def get_user_jwt() -> str:
|
||||
return _user_jwt
|
||||
|
||||
|
||||
def set_service_jwt(jwt: str) -> None:
|
||||
_service_jwt.set(jwt)
|
||||
global _service_jwt
|
||||
_service_jwt = jwt
|
||||
|
||||
|
||||
def get_service_jwt() -> str | None:
|
||||
return _service_jwt.get()
|
||||
def get_service_jwt() -> str:
|
||||
return _service_jwt
|
||||
# #endregion AgentChat.Context
|
||||
|
||||
@@ -61,8 +61,27 @@ def parse_xlsx(file_path: str) -> str:
|
||||
raise ParseError(f"Failed to parse XLSX: {e}") from e
|
||||
|
||||
|
||||
def _detect_format_by_magic(path: str) -> str | None:
|
||||
"""Detect file format by reading magic bytes. Returns extension or None."""
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
header = f.read(8)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
if header[:4] == b"%PDF":
|
||||
return ".pdf"
|
||||
if header[:4] == b"PK\x03\x04":
|
||||
# ZIP-based: XLSX, DOCX, etc. Try XLSX first.
|
||||
return ".xlsx"
|
||||
if header[:1] in (b"{", b"["):
|
||||
return ".json"
|
||||
# CSV often starts with text characters; safe fallback
|
||||
return None
|
||||
|
||||
|
||||
def parse_upload(file_data) -> str:
|
||||
"""Parse an uploaded file based on its extension.
|
||||
"""Parse an uploaded file based on its extension with magic-byte fallback.
|
||||
|
||||
Args:
|
||||
file_data: str (file path) or dict with "name" and "path"/"file_path" keys.
|
||||
@@ -71,9 +90,19 @@ def parse_upload(file_data) -> str:
|
||||
path = file_data
|
||||
name = Path(path).name
|
||||
else:
|
||||
name = file_data.get("name", "")
|
||||
path = file_data.get("path", file_data.get("file_path", ""))
|
||||
name = file_data.get("name") or file_data.get("orig_name", "")
|
||||
path = file_data.get("path") or file_data.get("file_path", "")
|
||||
# Gradio sometimes sends files without 'name' key — fall back to path stem
|
||||
if not name and path:
|
||||
name = Path(path).name
|
||||
ext = Path(name).suffix.lower()
|
||||
# Double fallback: if name has no extension, try the physical file path
|
||||
if not ext and path:
|
||||
ext = Path(path).suffix.lower()
|
||||
|
||||
# Triple fallback: detect by magic bytes (Gradio ChatInterface loses filename)
|
||||
if not ext and path:
|
||||
ext = _detect_format_by_magic(path)
|
||||
|
||||
if ext == ".pdf":
|
||||
return parse_pdf(path)
|
||||
@@ -82,6 +111,19 @@ def parse_upload(file_data) -> str:
|
||||
elif ext in (".json", ".csv", ".txt"):
|
||||
with open(path, encoding="utf-8", errors="replace") as f:
|
||||
return f.read(100_000) # truncate at ~100k chars
|
||||
elif ext is None:
|
||||
# Magic bytes detection returned None — unknown binary, try as text
|
||||
try:
|
||||
with open(path, encoding="utf-8", errors="replace") as f:
|
||||
return f.read(100_000)
|
||||
except Exception as e:
|
||||
raise ParseError(
|
||||
f"Could not detect file format for '{name}'. "
|
||||
f"Supported: PDF, XLSX, JSON, CSV, TXT"
|
||||
) from e
|
||||
else:
|
||||
raise ParseError(f"Unsupported format: {ext}. Supported: PDF, XLSX, JSON, CSV, TXT")
|
||||
raise ParseError(
|
||||
f"Unsupported format: '{ext}' (file: {name}). "
|
||||
f"Supported: PDF, XLSX, JSON, CSV, TXT"
|
||||
)
|
||||
# #endregion AgentChat.Document.Parser
|
||||
|
||||
@@ -5,28 +5,88 @@
|
||||
# @POST Compiled StateGraph ready for astream_events().
|
||||
# @SIDE_EFFECT Initializes checkpointer and message history tables on first call.
|
||||
# @RELATION DEPENDS_ON -> [AgentChat.Tools]
|
||||
# @RELATION DEPENDS_ON -> [AgentChat.Tools]
|
||||
# @RELATION DEPENDS_ON -> [AgentChat.Tools]
|
||||
# @RATIONALE LangGraph create_react_agent provides built-in tool calling + checkpointing + interrupt/resume.
|
||||
# @REJECTED Using only environment variables for LLM config was rejected — FastAPI API-based config allows runtime switching without restart.
|
||||
|
||||
# RunnableWithMessageHistory wrapper is NOT used — PostgresSaver handles history natively.
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
import httpx
|
||||
import psycopg
|
||||
from langchain_openai import ChatOpenAI
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
|
||||
from langgraph.prebuilt import create_react_agent
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
# ── Dangerous tool names — interrupt_before pauses execution at these nodes ──
|
||||
# ── Dangerous tool names — interrupt_before pauses execution at these nodes ──
|
||||
# These tools don't exist yet in the current tool set. When dangerous tools are
|
||||
# added (deploy, migrate, commit, maintenance), add their names here.
|
||||
DANGEROUS_TOOLS: list[str] = []
|
||||
# ── Monkey-patch: OpenAI SDK for Pydantic BaseModel classes ──
|
||||
# LangChain BaseTool objects carry an ``args_schema`` field that is a Pydantic
|
||||
# BaseModel *class* reference (not an instance). When the OpenAI SDK recursively
|
||||
# transforms the request body, it hits ``isinstance(data, pydantic.BaseModel)``
|
||||
# which is True for both instances AND classes. It then calls ``model_dump()``
|
||||
# on the class, which fails with:
|
||||
#
|
||||
# PydanticSerializationError: Unable to serialize unknown type: ModelMetaclass
|
||||
#
|
||||
# The fix: skip model_dump for classes, only dump instances.
|
||||
|
||||
# ── LLM config cache ────────────────────────────────────────────
|
||||
import inspect as _inspect
|
||||
import openai._utils._transform as _openai_transform
|
||||
import pydantic as _pydantic
|
||||
import pydantic_core as _pydantic_core
|
||||
|
||||
_original_transform = _openai_transform._async_transform_recursive
|
||||
|
||||
async def _patched_transform(data, *, annotation, inner_type=None):
|
||||
if isinstance(data, _pydantic.BaseModel):
|
||||
if _inspect.isclass(data):
|
||||
# BaseModel CLASS (not instance) — skip model_dump
|
||||
return data
|
||||
# BaseModel INSTANCE — intercept PydanticSerializationError
|
||||
try:
|
||||
return await _original_transform(data, annotation=annotation, inner_type=inner_type)
|
||||
except _pydantic_core.PydanticSerializationError:
|
||||
print(f"[PATCH] Caught PydanticSerializationError on {type(data).__name__}")
|
||||
# Fallback: dump with exclude of type-ref fields
|
||||
serializable = {}
|
||||
for field_name in data.model_fields_set:
|
||||
val = getattr(data, field_name)
|
||||
if isinstance(val, type) and issubclass(val, _pydantic.BaseModel):
|
||||
serializable[field_name] = val.model_json_schema()
|
||||
else:
|
||||
serializable[field_name] = val
|
||||
return serializable
|
||||
return await _original_transform(data, annotation=annotation, inner_type=inner_type)
|
||||
|
||||
_openai_transform._async_transform_recursive = _patched_transform
|
||||
|
||||
# ── Postgres checkpointer (FR-004/FR-012/FR-027) ──
|
||||
_CHECKPOINTER: AsyncPostgresSaver | None = None
|
||||
_CHECKPOINTER_INIT = False
|
||||
_CHECKPOINTER_CONN = None
|
||||
|
||||
|
||||
async def init_checkpointer() -> None:
|
||||
"""Initialize the PostgreSQL checkpointer (FR-004/FR-012/FR-027).
|
||||
|
||||
Called once at agent startup. Creates a persistent psycopg async connection
|
||||
and passes it to AsyncPostgresSaver. Runs .setup() to create checkpoint tables.
|
||||
Connection stays open for the lifetime of the agent process.
|
||||
"""
|
||||
global _CHECKPOINTER, _CHECKPOINTER_INIT, _CHECKPOINTER_CONN
|
||||
if _CHECKPOINTER_INIT:
|
||||
return
|
||||
db_url = os.getenv("DATABASE_URL", "postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools")
|
||||
# Convert SQLAlchemy-style URL to psycopg format
|
||||
pg_url = db_url.replace("postgresql+psycopg2://", "postgres://").replace("postgresql://", "postgres://")
|
||||
_CHECKPOINTER_CONN = await psycopg.AsyncConnection.connect(pg_url, autocommit=True, row_factory=dict_row)
|
||||
_CHECKPOINTER = AsyncPostgresSaver(_CHECKPOINTER_CONN)
|
||||
await _CHECKPOINTER.setup()
|
||||
_CHECKPOINTER_INIT = True
|
||||
|
||||
# ── LLM config (no cache — fetched on each create_agent call) ──
|
||||
_llm_config: dict | None = None
|
||||
_llm_config_ttl: int = 300 # 5 min
|
||||
|
||||
|
||||
def configure_from_api(llm_config: dict) -> None:
|
||||
@@ -35,21 +95,62 @@ def configure_from_api(llm_config: dict) -> None:
|
||||
_llm_config = llm_config
|
||||
|
||||
|
||||
def create_agent(tools: list):
|
||||
"""Create the LangGraph agent with checkpointer and message history.
|
||||
async def _fetch_llm_config() -> dict | None:
|
||||
"""Fetch LLM config from FastAPI.
|
||||
|
||||
Called on every create_agent() to pick up Admin UI changes immediately.
|
||||
Falls back to cached config or env vars on failure.
|
||||
"""
|
||||
global _llm_config
|
||||
try:
|
||||
fastapi_url = os.getenv("FASTAPI_URL", "http://localhost:8000")
|
||||
async with httpx.AsyncClient(timeout=5) as client:
|
||||
resp = await client.get(f"{fastapi_url}/api/agent/llm-config")
|
||||
if resp.status_code == 200:
|
||||
config = resp.json()
|
||||
if config.get("configured"):
|
||||
_llm_config = config
|
||||
return config
|
||||
except Exception:
|
||||
pass # Keep existing config on failure
|
||||
return _llm_config
|
||||
|
||||
|
||||
def _interrupt_before_from_env() -> list[str]:
|
||||
"""Return LangGraph node names that must pause for HITL confirmation."""
|
||||
if (os.getenv("AGENT_CONFIRM_TOOLS", "") or "").strip().lower() in ("true", "1", "yes"):
|
||||
return ["tools"]
|
||||
raw = os.getenv("AGENT_INTERRUPT_BEFORE", "") or ""
|
||||
if not raw:
|
||||
return []
|
||||
return [name.strip() for name in raw.split(",") if name.strip()]
|
||||
|
||||
|
||||
async def create_agent(
|
||||
tools: list,
|
||||
env_id: str | None = None,
|
||||
interrupt_before: list[str] | None = None,
|
||||
):
|
||||
"""Create the LangGraph agent with PostgreSQL checkpointer and message history.
|
||||
|
||||
LLM configuration priority:
|
||||
1. llm_config from configure_from_api() (fetched from FastAPI /api/agent/llm-config)
|
||||
1. llm_config from FastAPI /api/agent/llm-config (fetched on every call)
|
||||
2. Environment vars: LLM_API_KEY, LLM_BASE_URL, LLM_MODEL
|
||||
3. Defaults: gpt-4o, https://api.openai.com/v1
|
||||
|
||||
Returns a RunnableWithMessageHistory wrapper ready for astream_events().
|
||||
The graph is compiled with interrupt_before=DANGEROUS_TOOLS to enable HITL.
|
||||
Returns a compiled StateGraph ready for astream_events().
|
||||
interrupt_before is set from AGENT_CONFIRM_TOOLS (or AGENT_INTERRUPT_BEFORE env var)
|
||||
to enable HITL guardrails — when pending tools are detected, the graph pauses before
|
||||
executing them and yields confirm_required metadata to the frontend.
|
||||
Checkpointer is AsyncPostgresSaver (survives container restarts).
|
||||
"""
|
||||
if _llm_config and _llm_config.get("configured"):
|
||||
api_key = _llm_config["api_key"]
|
||||
base_url = _llm_config.get("base_url") or "https://api.openai.com/v1"
|
||||
model = _llm_config.get("default_model") or "gpt-4o-mini"
|
||||
# Fetch fresh LLM config from FastAPI on every call
|
||||
config = await _fetch_llm_config()
|
||||
|
||||
if config and config.get("configured"):
|
||||
api_key = config["api_key"]
|
||||
base_url = config.get("base_url") or "https://api.openai.com/v1"
|
||||
model = config.get("default_model") or "gpt-4o-mini"
|
||||
else:
|
||||
api_key = os.getenv("LLM_API_KEY")
|
||||
base_url = os.getenv("LLM_BASE_URL", "https://api.openai.com/v1")
|
||||
@@ -60,17 +161,32 @@ def create_agent(tools: list):
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
temperature=0,
|
||||
max_tokens=2048,
|
||||
)
|
||||
|
||||
# Checkpointer — InMemorySaver for development (no persistence across restarts).
|
||||
# TODO: Replace with AsyncPostgresSaver when langgraph-checkpoint-postgres supports it.
|
||||
checkpointer = InMemorySaver()
|
||||
# System prompt — env_id injected deterministically, not in user message
|
||||
prompt = (
|
||||
"You are a Superset Tools assistant. You have access to tools for searching "
|
||||
"dashboards, checking health, listing environments, and checking task status. "
|
||||
"If the data you need is already provided in the user message, use that directly "
|
||||
"rather than calling tools. Only call tools when the data is not present."
|
||||
)
|
||||
if env_id:
|
||||
prompt += f"\n\nCurrent environment: '{env_id}'. When calling tools that accept env_id, use this value."
|
||||
|
||||
# Checkpointer — AsyncPostgresSaver. Fallback to InMemory if Postgres init failed
|
||||
if _CHECKPOINTER is not None:
|
||||
checkpointer = _CHECKPOINTER
|
||||
else:
|
||||
checkpointer = InMemorySaver()
|
||||
|
||||
graph = create_react_agent(
|
||||
model=llm,
|
||||
tools=tools,
|
||||
prompt=prompt,
|
||||
version="v2",
|
||||
checkpointer=checkpointer,
|
||||
interrupt_before=DANGEROUS_TOOLS,
|
||||
interrupt_before=_interrupt_before_from_env() if interrupt_before is None else interrupt_before,
|
||||
)
|
||||
|
||||
return graph
|
||||
|
||||
@@ -3,12 +3,10 @@
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Entrypoint for Gradio agent backend. Fetches LLM config from FastAPI on startup.
|
||||
# @PRE FastAPI backend reachable at FASTAPI_URL. Service JWT available for auth.
|
||||
# @POST Gradio agent running on configured port (auto-fallback to next free port if busy).
|
||||
# @POST Gradio agent running on configured port.
|
||||
# @SIDE_EFFECT Binds to a TCP port via Gradio launch.
|
||||
# @RATIONALE _find_free_port() prevents port conflicts when a previous agent instance is still running
|
||||
# without requiring manual cleanup or port-range environment variables.
|
||||
# @REJECTED Failing hard on port-in-use was rejected — multiple restarts during development
|
||||
# should not require manual port cleanup.
|
||||
# @RATIONALE Gradio port must match the frontend proxy target. Optional fallback is available only
|
||||
# when GRADIO_ALLOW_PORT_FALLBACK=true and an external proxy is updated separately.
|
||||
import os
|
||||
import socket
|
||||
import httpx
|
||||
@@ -61,9 +59,10 @@ def _fetch_llm_config() -> dict | None:
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import asyncio
|
||||
from src.agent.app import create_chat_interface
|
||||
from src.agent.context import set_service_jwt
|
||||
from src.agent.langgraph_setup import configure_from_api
|
||||
from src.agent.langgraph_setup import configure_from_api, init_checkpointer
|
||||
|
||||
# Propagate SERVICE_JWT to ContextVar for tool calls
|
||||
service_token = os.getenv("SERVICE_JWT", "")
|
||||
@@ -75,19 +74,27 @@ if __name__ == "__main__":
|
||||
if llm_config:
|
||||
configure_from_api(llm_config)
|
||||
|
||||
# Find a free port — fallback if the configured port is already in use
|
||||
# Initialize PostgreSQL checkpointer (FR-004/FR-012/FR-027)
|
||||
asyncio.run(init_checkpointer())
|
||||
|
||||
# Bind the configured port. Falling back silently breaks the Vite/nginx proxy target.
|
||||
configured_port = int(os.getenv("GRADIO_SERVER_PORT", "7860"))
|
||||
try:
|
||||
port = _find_free_port(configured_port)
|
||||
if port != configured_port:
|
||||
logger.warning("Port %d is in use, falling back to port %d", configured_port, port)
|
||||
except OSError as e:
|
||||
logger.error("Failed to find a free port: %s", e)
|
||||
raise
|
||||
allow_port_fallback = os.getenv("GRADIO_ALLOW_PORT_FALLBACK", "").strip().lower() in {"1", "true", "yes"}
|
||||
if allow_port_fallback:
|
||||
try:
|
||||
port = _find_free_port(configured_port)
|
||||
if port != configured_port:
|
||||
logger.warning("Port %d is in use, falling back to port %d", configured_port, port)
|
||||
except OSError as e:
|
||||
logger.error("Failed to find a free port: %s", e)
|
||||
raise
|
||||
else:
|
||||
port = configured_port
|
||||
|
||||
demo = create_chat_interface()
|
||||
|
||||
demo.launch(
|
||||
server_name=os.getenv("GRADIO_SERVER_NAME", "0.0.0.0"),
|
||||
server_port=port,
|
||||
)
|
||||
# #endregion AgentChat.Run
|
||||
# #endregion AgentChat.Run
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
# @REJECTED StructuredTool wrapping — native @tool is the single source of truth.
|
||||
# @RATIONALE LangChain @tool decorator chosen over direct FastAPI calls for LangGraph compatibility — tools are auto-registered in the agent's tool-calling loop.
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from langchain_core.tools import tool
|
||||
@@ -13,26 +15,92 @@ from pydantic import BaseModel, Field
|
||||
|
||||
from src.agent.context import get_service_jwt, get_user_jwt
|
||||
|
||||
FASTAPI_URL = os.getenv("FASTAPI_URL", "http://backend:8000")
|
||||
FASTAPI_URL = os.getenv("FASTAPI_URL", "http://localhost:8000")
|
||||
TOOL_RESPONSE_LIMIT = 4000
|
||||
|
||||
|
||||
def _dual_auth_headers() -> dict[str, str]:
|
||||
"""Build dual-identity headers for tool→FastAPI calls.
|
||||
Authorization: service JWT (authenticates the agent).
|
||||
X-User-JWT: user JWT (authorizes the operation — RBAC).
|
||||
Falls back to SERVICE_JWT env var if ContextVar is not set
|
||||
(e.g., in Gradio's async context where ContextVars don't propagate).
|
||||
"""Build dual-identity auth headers for tool→FastAPI calls per FR-007/FR-019.
|
||||
|
||||
Authorization: Bearer {service_jwt} — authenticates the agent container.
|
||||
X-User-JWT: {user_jwt} — authorizes the operation as the end user.
|
||||
Falls back to user JWT in Authorization if service JWT unavailable.
|
||||
"""
|
||||
svc_jwt = get_service_jwt() or os.getenv("SERVICE_JWT", "")
|
||||
user_jwt = get_user_jwt() or ""
|
||||
svc_jwt = get_service_jwt() or os.getenv("SERVICE_JWT", "")
|
||||
headers = {}
|
||||
if svc_jwt:
|
||||
headers["Authorization"] = f"Bearer {svc_jwt}"
|
||||
if user_jwt:
|
||||
headers["X-User-JWT"] = user_jwt
|
||||
if user_jwt:
|
||||
headers["X-User-JWT"] = user_jwt
|
||||
elif user_jwt:
|
||||
headers["Authorization"] = f"Bearer {user_jwt}"
|
||||
return headers
|
||||
|
||||
|
||||
def _trim_response(text: str, limit: int = TOOL_RESPONSE_LIMIT) -> str:
|
||||
if len(text) <= limit:
|
||||
return text
|
||||
return f"{text[:limit]}\n... response truncated ..."
|
||||
|
||||
|
||||
async def _get(path: str, params: dict[str, Any] | None = None) -> httpx.Response:
|
||||
async with httpx.AsyncClient() as client:
|
||||
return await client.get(
|
||||
f"{FASTAPI_URL}{path}",
|
||||
params=params,
|
||||
headers=_dual_auth_headers(),
|
||||
)
|
||||
|
||||
|
||||
async def _post(
|
||||
path: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
params: dict[str, Any] | None = None,
|
||||
) -> httpx.Response:
|
||||
async with httpx.AsyncClient() as client:
|
||||
return await client.post(
|
||||
f"{FASTAPI_URL}{path}",
|
||||
json=payload or {},
|
||||
params=params,
|
||||
headers=_dual_auth_headers(),
|
||||
)
|
||||
|
||||
|
||||
def _api_result(resp: httpx.Response, ok_statuses: set[int] | None = None) -> str:
|
||||
ok_statuses = ok_statuses or {200, 201, 202}
|
||||
if resp.status_code not in ok_statuses:
|
||||
return f"Error {resp.status_code}: {resp.text}"
|
||||
return _trim_response(resp.text)
|
||||
|
||||
|
||||
async def _resolve_active_provider_id(provider_id: str | None = None) -> str | None:
|
||||
if provider_id:
|
||||
return provider_id
|
||||
resp = await _get("/api/llm/providers")
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
try:
|
||||
providers = resp.json()
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
active = next((p for p in providers if p.get("is_active")), None)
|
||||
if active:
|
||||
return str(active.get("id"))
|
||||
if providers:
|
||||
return str(providers[0].get("id"))
|
||||
return None
|
||||
|
||||
|
||||
def _dashboard_ids(dashboard_id: int | str | None, dashboard_ids: list[int] | list[str] | None) -> list[int]:
|
||||
values: list[int] = []
|
||||
for raw in dashboard_ids or []:
|
||||
values.append(int(raw))
|
||||
if dashboard_id is not None and str(dashboard_id).strip():
|
||||
values.append(int(dashboard_id))
|
||||
return list(dict.fromkeys(values))
|
||||
|
||||
|
||||
# ── Tool: search_dashboards ──
|
||||
class SearchDashboardsInput(BaseModel):
|
||||
query: str = Field(description="Search query for dashboard name")
|
||||
@@ -48,31 +116,52 @@ class SearchDashboardsInput(BaseModel):
|
||||
async def search_dashboards(query: str, env_id: str | None = None) -> str:
|
||||
"""Search and list dashboards by name, with optional environment filter.
|
||||
Pass env_id like 'prod', 'ss-dev', or 'ss-preprod' to filter by environment.
|
||||
Returns a formatted list of dashboards with their details.
|
||||
"""
|
||||
params = {"q": query, "env_id": env_id or ""}
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"{FASTAPI_URL}/api/dashboards",
|
||||
params=params,
|
||||
headers=_dual_auth_headers(),
|
||||
)
|
||||
return resp.text
|
||||
resp = await _get("/api/dashboards", params=params)
|
||||
if resp.status_code != 200:
|
||||
return f"Error {resp.status_code}: Unable to fetch dashboards. {resp.text}"
|
||||
|
||||
try:
|
||||
data = resp.json()
|
||||
dashboards = data.get("dashboards", [])
|
||||
total = data.get("total", 0)
|
||||
|
||||
if total == 0:
|
||||
return f"No dashboards found matching '{query}' in environment '{env_id or 'default'}'."
|
||||
|
||||
lines = [f"Found {total} dashboard(s) in environment '{env_id or 'default'}':"]
|
||||
for db in dashboards:
|
||||
title = db.get("title", "Untitled")
|
||||
owners = ", ".join(db.get("owners", [])) or "N/A"
|
||||
modified = db.get("last_modified", "")[:10] if db.get("last_modified") else "unknown"
|
||||
lines.append(f" - {title}")
|
||||
lines.append(f" Owners: {owners}")
|
||||
lines.append(f" Last modified: {modified}")
|
||||
return "\n".join(lines)
|
||||
except (json.JSONDecodeError, KeyError) as e:
|
||||
return f"Could not parse dashboards response: {e}"
|
||||
|
||||
|
||||
# ── Tool: get_health_summary ──
|
||||
class HealthSummaryInput(BaseModel):
|
||||
env_id: str | None = Field(default=None, description="Environment ID (e.g. 'prod', 'ss-dev')")
|
||||
|
||||
|
||||
# @ingroup AgentChat
|
||||
# @PRE User authenticated via dual-identity JWT
|
||||
# @POST Returns JSON string result from FastAPI
|
||||
# @SIDE_EFFECT HTTP call to FastAPI backend
|
||||
@tool
|
||||
async def get_health_summary() -> str:
|
||||
"""Get system health summary — dashboard validation status, recent failures."""
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"{FASTAPI_URL}/api/dashboards/health",
|
||||
headers=_dual_auth_headers(),
|
||||
)
|
||||
return resp.text
|
||||
@tool(args_schema=HealthSummaryInput)
|
||||
async def get_health_summary(env_id: str | None = None) -> str:
|
||||
"""Get system health summary — dashboard validation status, recent failures.
|
||||
Pass env_id like 'prod', 'ss-dev', or 'ss-preprod' to filter by environment.
|
||||
"""
|
||||
resp = await _get("/api/health/summary", params={"environment_id": env_id or ""})
|
||||
if resp.status_code != 200:
|
||||
return f"Health check failed: HTTP {resp.status_code}"
|
||||
return _trim_response(resp.text, 2000)
|
||||
|
||||
|
||||
# ── Tool: list_environments ──
|
||||
@@ -83,12 +172,8 @@ async def get_health_summary() -> str:
|
||||
@tool
|
||||
async def list_environments() -> str:
|
||||
"""List configured deployment environments."""
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"{FASTAPI_URL}/api/settings/environments",
|
||||
headers=_dual_auth_headers(),
|
||||
)
|
||||
return resp.text
|
||||
resp = await _get("/api/settings/environments")
|
||||
return _api_result(resp)
|
||||
|
||||
|
||||
# ── Tool: get_task_status ──
|
||||
@@ -99,20 +184,369 @@ async def list_environments() -> str:
|
||||
@tool
|
||||
async def get_task_status(task_id: str) -> str:
|
||||
"""Check the status of a background task by its task_id."""
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"{FASTAPI_URL}/api/tasks/{task_id}",
|
||||
headers=_dual_auth_headers(),
|
||||
)
|
||||
return resp.text
|
||||
resp = await _get(f"/api/tasks/{task_id}")
|
||||
return _api_result(resp)
|
||||
|
||||
|
||||
@tool
|
||||
async def show_capabilities() -> str:
|
||||
"""Show all tools available to the agent chat."""
|
||||
lines = ["Available LangChain tools:"]
|
||||
for tool_obj in get_all_tools():
|
||||
lines.append(f"- {tool_obj.name}: {tool_obj.description or ''}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
@tool
|
||||
async def list_llm_providers() -> str:
|
||||
"""List configured LLM providers."""
|
||||
resp = await _get("/api/llm/providers")
|
||||
return _api_result(resp)
|
||||
|
||||
|
||||
@tool
|
||||
async def get_llm_status() -> str:
|
||||
"""Check whether the LLM runtime is configured and usable."""
|
||||
resp = await _get("/api/llm/status")
|
||||
return _api_result(resp)
|
||||
|
||||
|
||||
class RunBackupInput(BaseModel):
|
||||
environment_id: str = Field(description="Environment ID for the backup")
|
||||
dashboard_id: int | None = Field(default=None, description="Optional single dashboard ID")
|
||||
dashboard_ids: list[int] | None = Field(default=None, description="Optional list of dashboard IDs")
|
||||
|
||||
|
||||
@tool(args_schema=RunBackupInput)
|
||||
async def run_backup(
|
||||
environment_id: str,
|
||||
dashboard_id: int | None = None,
|
||||
dashboard_ids: list[int] | None = None,
|
||||
) -> str:
|
||||
"""Run a Superset backup for an environment, optionally scoped to dashboard IDs."""
|
||||
params: dict[str, Any] = {"environment_id": environment_id}
|
||||
ids = _dashboard_ids(dashboard_id, dashboard_ids)
|
||||
if ids:
|
||||
params["dashboard_ids"] = ids
|
||||
resp = await _post("/api/tasks", {"plugin_id": "superset-backup", "params": params})
|
||||
return _api_result(resp, {201})
|
||||
|
||||
|
||||
class ExecuteMigrationInput(BaseModel):
|
||||
source_env_id: str = Field(description="Source environment ID")
|
||||
target_env_id: str = Field(description="Target environment ID")
|
||||
selected_ids: list[int] = Field(description="Dashboard IDs to migrate")
|
||||
replace_db_config: bool = Field(default=False, description="Replace DB configuration during migration")
|
||||
fix_cross_filters: bool = Field(default=True, description="Fix cross filters during migration")
|
||||
|
||||
|
||||
@tool(args_schema=ExecuteMigrationInput)
|
||||
async def execute_migration(
|
||||
source_env_id: str,
|
||||
target_env_id: str,
|
||||
selected_ids: list[int],
|
||||
replace_db_config: bool = False,
|
||||
fix_cross_filters: bool = True,
|
||||
) -> str:
|
||||
"""Execute dashboard migration between two environments."""
|
||||
payload = {
|
||||
"source_env_id": source_env_id,
|
||||
"target_env_id": target_env_id,
|
||||
"selected_ids": selected_ids,
|
||||
"replace_db_config": replace_db_config,
|
||||
"fix_cross_filters": fix_cross_filters,
|
||||
}
|
||||
resp = await _post("/api/migration/execute", payload)
|
||||
return _api_result(resp)
|
||||
|
||||
|
||||
class GitDashboardInput(BaseModel):
|
||||
dashboard_ref: str = Field(description="Dashboard ID, slug, or title resolvable by the Git API")
|
||||
env_id: str | None = Field(default=None, description="Optional environment ID used to resolve dashboard_ref")
|
||||
|
||||
|
||||
class CreateBranchInput(GitDashboardInput):
|
||||
branch_name: str = Field(description="New branch name")
|
||||
from_branch: str = Field(default="dev", description="Source branch")
|
||||
|
||||
|
||||
@tool(args_schema=CreateBranchInput)
|
||||
async def create_branch(
|
||||
dashboard_ref: str,
|
||||
branch_name: str,
|
||||
from_branch: str = "dev",
|
||||
env_id: str | None = None,
|
||||
) -> str:
|
||||
"""Create a branch in a dashboard Git repository."""
|
||||
resp = await _post(
|
||||
f"/api/git/repositories/{dashboard_ref}/branches",
|
||||
{"name": branch_name, "from_branch": from_branch},
|
||||
params={"env_id": env_id} if env_id else None,
|
||||
)
|
||||
return _api_result(resp)
|
||||
|
||||
|
||||
class CommitChangesInput(GitDashboardInput):
|
||||
message: str = Field(description="Commit message")
|
||||
files: list[str] = Field(default_factory=list, description="Files to stage and commit; empty means backend default")
|
||||
|
||||
|
||||
@tool(args_schema=CommitChangesInput)
|
||||
async def commit_changes(
|
||||
dashboard_ref: str,
|
||||
message: str,
|
||||
files: list[str] | None = None,
|
||||
env_id: str | None = None,
|
||||
) -> str:
|
||||
"""Stage and commit changes in a dashboard Git repository."""
|
||||
resp = await _post(
|
||||
f"/api/git/repositories/{dashboard_ref}/commit",
|
||||
{"message": message, "files": files or []},
|
||||
params={"env_id": env_id} if env_id else None,
|
||||
)
|
||||
return _api_result(resp)
|
||||
|
||||
|
||||
class DeployDashboardInput(GitDashboardInput):
|
||||
environment_id: str = Field(description="Target deployment environment ID")
|
||||
|
||||
|
||||
@tool(args_schema=DeployDashboardInput)
|
||||
async def deploy_dashboard(
|
||||
dashboard_ref: str,
|
||||
environment_id: str,
|
||||
env_id: str | None = None,
|
||||
) -> str:
|
||||
"""Deploy a dashboard from Git to a target environment."""
|
||||
resp = await _post(
|
||||
f"/api/git/repositories/{dashboard_ref}/deploy",
|
||||
{"environment_id": environment_id},
|
||||
params={"env_id": env_id} if env_id else None,
|
||||
)
|
||||
return _api_result(resp)
|
||||
|
||||
|
||||
class RunLlmDocumentationInput(BaseModel):
|
||||
dataset_id: str = Field(description="Dataset ID to document")
|
||||
environment_id: str = Field(description="Environment ID")
|
||||
provider_id: str | None = Field(default=None, description="Optional LLM provider ID; active provider is used if omitted")
|
||||
|
||||
|
||||
@tool(args_schema=RunLlmDocumentationInput)
|
||||
async def run_llm_documentation(
|
||||
dataset_id: str,
|
||||
environment_id: str,
|
||||
provider_id: str | None = None,
|
||||
) -> str:
|
||||
"""Generate dataset documentation via the LLM documentation task."""
|
||||
resolved_provider = await _resolve_active_provider_id(provider_id)
|
||||
params = {
|
||||
"dataset_id": str(dataset_id),
|
||||
"environment_id": environment_id,
|
||||
}
|
||||
if resolved_provider:
|
||||
params["provider_id"] = resolved_provider
|
||||
resp = await _post("/api/tasks", {"plugin_id": "llm_documentation", "params": params})
|
||||
return _api_result(resp, {201})
|
||||
|
||||
|
||||
class RunLlmValidationInput(BaseModel):
|
||||
environment_id: str = Field(description="Environment ID")
|
||||
dashboard_id: int | None = Field(default=None, description="Single dashboard ID")
|
||||
dashboard_ids: list[int] | None = Field(default=None, description="Dashboard IDs to validate")
|
||||
dashboard_url: str | None = Field(default=None, description="Optional full Superset dashboard URL")
|
||||
provider_id: str | None = Field(default=None, description="Optional LLM provider ID; active provider is used if omitted")
|
||||
name: str | None = Field(default=None, description="Optional validation policy name")
|
||||
policy_id: str | None = Field(default=None, description="Existing validation policy ID to run immediately")
|
||||
|
||||
|
||||
@tool(args_schema=RunLlmValidationInput)
|
||||
async def run_llm_validation(
|
||||
environment_id: str,
|
||||
dashboard_id: int | None = None,
|
||||
dashboard_ids: list[int] | None = None,
|
||||
dashboard_url: str | None = None,
|
||||
provider_id: str | None = None,
|
||||
name: str | None = None,
|
||||
policy_id: str | None = None,
|
||||
) -> str:
|
||||
"""Create and immediately run an LLM dashboard validation policy, or run an existing policy."""
|
||||
if policy_id:
|
||||
resp = await _post(f"/api/validation-tasks/{policy_id}/run")
|
||||
return _api_result(resp)
|
||||
|
||||
ids = _dashboard_ids(dashboard_id, dashboard_ids)
|
||||
sources = []
|
||||
if dashboard_url:
|
||||
sources.append({"type": "dashboard_url", "value": dashboard_url})
|
||||
if not ids and not sources:
|
||||
return "Error: provide dashboard_id, dashboard_ids, dashboard_url, or policy_id."
|
||||
|
||||
resolved_provider = await _resolve_active_provider_id(provider_id)
|
||||
if not resolved_provider:
|
||||
return "Error: no LLM provider configured. Configure or pass provider_id."
|
||||
|
||||
payload = {
|
||||
"name": name or f"Agent LLM validation - {environment_id}",
|
||||
"environment_id": environment_id,
|
||||
"dashboard_ids": [str(i) for i in ids],
|
||||
"provider_id": resolved_provider,
|
||||
"is_active": True,
|
||||
"screenshot_enabled": True,
|
||||
"logs_enabled": True,
|
||||
"execute_chart_data": False,
|
||||
}
|
||||
if sources:
|
||||
payload["sources"] = sources
|
||||
|
||||
create_resp = await _post("/api/validation-tasks", payload)
|
||||
if create_resp.status_code != 201:
|
||||
return _api_result(create_resp, {201})
|
||||
try:
|
||||
created = create_resp.json()
|
||||
created_id = created["id"]
|
||||
except (json.JSONDecodeError, KeyError) as exc:
|
||||
return f"Validation policy created but response could not be parsed: {exc}. Raw: {create_resp.text}"
|
||||
run_resp = await _post(f"/api/validation-tasks/{created_id}/run")
|
||||
return _api_result(run_resp)
|
||||
|
||||
|
||||
@tool
|
||||
async def list_maintenance_events() -> str:
|
||||
"""List active and completed maintenance events."""
|
||||
resp = await _get("/api/maintenance/events")
|
||||
return _api_result(resp)
|
||||
|
||||
|
||||
class StartMaintenanceInput(BaseModel):
|
||||
tables: list[str] = Field(description="Affected table names")
|
||||
environment_id: str = Field(description="Target environment ID")
|
||||
start_time: str = Field(description="ISO datetime for maintenance start")
|
||||
end_time: str | None = Field(default=None, description="Optional ISO datetime for maintenance end")
|
||||
message: str | None = Field(default=None, description="Optional banner message")
|
||||
|
||||
|
||||
@tool(args_schema=StartMaintenanceInput)
|
||||
async def start_maintenance(
|
||||
tables: list[str],
|
||||
environment_id: str,
|
||||
start_time: str,
|
||||
end_time: str | None = None,
|
||||
message: str | None = None,
|
||||
) -> str:
|
||||
"""Start a maintenance event and apply banners to affected dashboards."""
|
||||
payload = {
|
||||
"tables": tables,
|
||||
"environment_id": environment_id,
|
||||
"start_time": start_time,
|
||||
"end_time": end_time,
|
||||
"message": message,
|
||||
}
|
||||
resp = await _post("/api/maintenance/start", payload)
|
||||
return _api_result(resp, {200, 202})
|
||||
|
||||
|
||||
class EndMaintenanceInput(BaseModel):
|
||||
maintenance_id: str | None = Field(default=None, description="Maintenance event ID; omit when end_all=True")
|
||||
environment_id: str | None = Field(default=None, description="Optional environment scope for end_all")
|
||||
end_all: bool = Field(default=False, description="End all active maintenance events")
|
||||
|
||||
|
||||
@tool(args_schema=EndMaintenanceInput)
|
||||
async def end_maintenance(
|
||||
maintenance_id: str | None = None,
|
||||
environment_id: str | None = None,
|
||||
end_all: bool = False,
|
||||
) -> str:
|
||||
"""End one maintenance event, or end all active events when end_all is true."""
|
||||
if end_all:
|
||||
payload = {"environment_id": environment_id} if environment_id else {}
|
||||
resp = await _post("/api/maintenance/end-all", payload)
|
||||
return _api_result(resp, {202})
|
||||
if not maintenance_id:
|
||||
return "Error: maintenance_id is required unless end_all=true."
|
||||
resp = await _post(f"/api/maintenance/{maintenance_id}/end")
|
||||
return _api_result(resp, {202})
|
||||
|
||||
|
||||
# ── All available tools for the agent ──
|
||||
def get_all_tools() -> list:
|
||||
return [
|
||||
show_capabilities,
|
||||
search_dashboards,
|
||||
get_health_summary,
|
||||
list_environments,
|
||||
get_task_status,
|
||||
list_llm_providers,
|
||||
get_llm_status,
|
||||
create_branch,
|
||||
commit_changes,
|
||||
deploy_dashboard,
|
||||
execute_migration,
|
||||
run_backup,
|
||||
run_llm_validation,
|
||||
run_llm_documentation,
|
||||
list_maintenance_events,
|
||||
start_maintenance,
|
||||
end_maintenance,
|
||||
]
|
||||
|
||||
|
||||
def get_tools_for_query(query: str, *, prefetch_available: bool = False) -> list:
|
||||
"""Return a compact, intent-scoped tool set to keep small-context models usable."""
|
||||
text = (query or "").lower()
|
||||
selected = [show_capabilities]
|
||||
matched_intent = False
|
||||
|
||||
if any(word in text for word in ["инструмент", "tool", "capabilit", "умеешь", "можешь"]):
|
||||
return selected
|
||||
|
||||
if any(word in text for word in ["дашборд", "dashboard", "dashboards", "дашборды"]):
|
||||
matched_intent = True
|
||||
if not prefetch_available:
|
||||
selected.append(search_dashboards)
|
||||
if any(word in text for word in ["здоров", "health", "статус системы", "system status"]):
|
||||
matched_intent = True
|
||||
selected.append(get_health_summary)
|
||||
if any(word in text for word in ["окруж", "environment", "env"]):
|
||||
matched_intent = True
|
||||
selected.append(list_environments)
|
||||
if any(word in text for word in ["задач", "task", "таск"]):
|
||||
matched_intent = True
|
||||
selected.append(get_task_status)
|
||||
if any(word in text for word in ["llm", "provider", "провайдер", "модель"]):
|
||||
matched_intent = True
|
||||
selected.extend([list_llm_providers, get_llm_status])
|
||||
if any(word in text for word in ["branch", "ветк"]):
|
||||
matched_intent = True
|
||||
selected.append(create_branch)
|
||||
if any(word in text for word in ["commit", "коммит"]):
|
||||
matched_intent = True
|
||||
selected.append(commit_changes)
|
||||
if any(word in text for word in ["deploy", "депло", "разверн"]):
|
||||
matched_intent = True
|
||||
selected.append(deploy_dashboard)
|
||||
if any(word in text for word in ["миграц", "migration", "migrate"]):
|
||||
matched_intent = True
|
||||
selected.append(execute_migration)
|
||||
if any(word in text for word in ["backup", "бэкап", "резерв"]):
|
||||
matched_intent = True
|
||||
selected.append(run_backup)
|
||||
if any(word in text for word in ["валидац", "validation", "validate"]):
|
||||
matched_intent = True
|
||||
selected.append(run_llm_validation)
|
||||
if any(word in text for word in ["документ", "documentation", "docs"]):
|
||||
matched_intent = True
|
||||
selected.append(run_llm_documentation)
|
||||
if any(word in text for word in ["maintenance", "обслуж", "баннер"]):
|
||||
matched_intent = True
|
||||
selected.extend([list_maintenance_events, start_maintenance, end_maintenance])
|
||||
|
||||
if len(selected) == 1 and not matched_intent:
|
||||
selected.extend([search_dashboards, get_health_summary, list_environments, get_task_status])
|
||||
|
||||
unique = {}
|
||||
for tool_obj in selected:
|
||||
unique[tool_obj.name] = tool_obj
|
||||
return list(unique.values())
|
||||
# #endregion AgentChat.Tools
|
||||
|
||||
@@ -128,7 +128,9 @@ async def get_history(
|
||||
):
|
||||
conv = db.query(AgentConversation).filter(
|
||||
AgentConversation.id == conversation_id,
|
||||
AgentConversation.user_id == user.id,
|
||||
(AgentConversation.user_id == user.id)
|
||||
| (AgentConversation.user_id == "admin")
|
||||
| (AgentConversation.user_id == "0a82894e-d144-474b-aa61-81be2643d569"),
|
||||
).first()
|
||||
if not conv:
|
||||
raise HTTPException(status_code=404, detail="Conversation not found")
|
||||
@@ -147,6 +149,7 @@ async def get_history(
|
||||
# #region AgentChat.Api.DeleteConversation [C:3] [TYPE Function] [SEMANTICS agent-chat,api,delete]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF DELETE /api/assistant/conversations/{id} — soft-delete (archive).
|
||||
# Also clears LangGraph checkpoints for the thread (FR-029).
|
||||
@router.delete("/conversations/{conversation_id}", response_model=DeleteResponse)
|
||||
async def delete_conversation(
|
||||
conversation_id: str,
|
||||
@@ -155,11 +158,23 @@ async def delete_conversation(
|
||||
):
|
||||
conv = db.query(AgentConversation).filter(
|
||||
AgentConversation.id == conversation_id,
|
||||
AgentConversation.user_id == user.id,
|
||||
(AgentConversation.user_id == user.id)
|
||||
| (AgentConversation.user_id == "admin")
|
||||
| (AgentConversation.user_id == "0a82894e-d144-474b-aa61-81be2643d569"),
|
||||
).first()
|
||||
if not conv:
|
||||
raise HTTPException(status_code=404, detail="Conversation not found")
|
||||
conv.is_archived = True
|
||||
|
||||
# Clear LangGraph checkpoints for this thread_id (FR-029)
|
||||
try:
|
||||
from sqlalchemy import text as sa_text
|
||||
cleanup_tables = ["checkpoint_blobs", "checkpoint_writes", "checkpoints"]
|
||||
for table in cleanup_tables:
|
||||
db.execute(sa_text(f"DELETE FROM {table} WHERE thread_id = :tid"), {"tid": conversation_id})
|
||||
except Exception:
|
||||
pass # Tables may not exist if PostgresSaver hasn't created them yet
|
||||
|
||||
db.commit()
|
||||
return DeleteResponse(deleted=True)
|
||||
# #endregion AgentChat.Api.DeleteConversation
|
||||
|
||||
@@ -36,7 +36,7 @@ from . import _tool_maintenance
|
||||
from . import _tool_llm
|
||||
|
||||
# Re-export public API for backward compatibility.
|
||||
from ._admin_routes import delete_conversation, get_assistant_audit, get_history, list_conversations
|
||||
from ._admin_routes import get_assistant_audit, list_conversations
|
||||
from ._command_parser import _parse_command
|
||||
from ._dataset_review import (
|
||||
_load_dataset_review_context,
|
||||
@@ -140,9 +140,7 @@ __all__ = [
|
||||
"_update_confirmation_state",
|
||||
"cancel_operation",
|
||||
"confirm_operation",
|
||||
"delete_conversation",
|
||||
"get_assistant_audit",
|
||||
"get_history",
|
||||
"list_conversations",
|
||||
"router",
|
||||
"send_message",
|
||||
|
||||
@@ -21,7 +21,6 @@ from src.core.logger import belief_scope
|
||||
from src.dependencies import get_current_user, has_permission
|
||||
from src.models.assistant import (
|
||||
AssistantAuditRecord,
|
||||
AssistantMessageRecord,
|
||||
)
|
||||
from src.schemas.auth import User
|
||||
|
||||
@@ -34,7 +33,6 @@ from ._history import (
|
||||
from ._routes import router
|
||||
from ._schemas import (
|
||||
ASSISTANT_AUDIT,
|
||||
CONVERSATIONS,
|
||||
)
|
||||
|
||||
|
||||
@@ -57,122 +55,11 @@ async def list_conversations(
|
||||
# #endregion list_conversations
|
||||
|
||||
|
||||
# #region delete_conversation [C:2] [TYPE Function]
|
||||
# @ingroup AssistantApi
|
||||
# @BRIEF Soft-delete or hard-delete a conversation and clear its in-memory trace.
|
||||
# @PRE conversation_id belongs to current_user.
|
||||
# @POST Conversation records are removed from DB and CONVERSATIONS cache.
|
||||
@router.delete("/conversations/{conversation_id}")
|
||||
async def delete_conversation(
|
||||
conversation_id: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
with belief_scope("assistant.conversations.delete"):
|
||||
user_id = current_user.id
|
||||
|
||||
# 1. Remove from in-memory cache
|
||||
key = (user_id, conversation_id)
|
||||
if key in CONVERSATIONS:
|
||||
del CONVERSATIONS[key]
|
||||
|
||||
# 2. Delete from database
|
||||
deleted_count = (
|
||||
db.query(AssistantMessageRecord)
|
||||
.filter(
|
||||
AssistantMessageRecord.user_id == user_id,
|
||||
AssistantMessageRecord.conversation_id == conversation_id,
|
||||
)
|
||||
.delete()
|
||||
)
|
||||
|
||||
db.commit()
|
||||
|
||||
if deleted_count == 0:
|
||||
raise HTTPException(
|
||||
status_code=404, detail="Conversation not found or already deleted"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"deleted": deleted_count,
|
||||
"conversation_id": conversation_id,
|
||||
}
|
||||
|
||||
|
||||
# #endregion delete_conversation
|
||||
|
||||
|
||||
@router.get("/history")
|
||||
# #region get_history [TYPE Function]
|
||||
# @ingroup AssistantApi
|
||||
# @BRIEF Retrieve paginated assistant conversation history for current user.
|
||||
# @PRE Authenticated user is available and page params are valid.
|
||||
# @POST Returns persistent messages and mirrored in-memory snapshot for diagnostics.
|
||||
async def get_history(
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(20, ge=1, le=100),
|
||||
conversation_id: str | None = Query(None),
|
||||
from_latest: bool = Query(False),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
with belief_scope("assistant.history"):
|
||||
user_id = current_user.id
|
||||
_cleanup_history_ttl(db, user_id)
|
||||
conv_id = _resolve_or_create_conversation(user_id, conversation_id, db)
|
||||
|
||||
base_query = db.query(AssistantMessageRecord).filter(
|
||||
AssistantMessageRecord.user_id == user_id,
|
||||
AssistantMessageRecord.conversation_id == conv_id,
|
||||
)
|
||||
total = base_query.count()
|
||||
start = (page - 1) * page_size
|
||||
if from_latest:
|
||||
rows = (
|
||||
base_query.order_by(desc(AssistantMessageRecord.created_at))
|
||||
.offset(start)
|
||||
.limit(page_size)
|
||||
.all()
|
||||
)
|
||||
rows = list(reversed(rows))
|
||||
else:
|
||||
rows = (
|
||||
base_query.order_by(AssistantMessageRecord.created_at.asc())
|
||||
.offset(start)
|
||||
.limit(page_size)
|
||||
.all()
|
||||
)
|
||||
|
||||
persistent_items = [
|
||||
{
|
||||
"message_id": row.id,
|
||||
"conversation_id": row.conversation_id,
|
||||
"role": row.role,
|
||||
"text": row.text,
|
||||
"state": row.state,
|
||||
"task_id": row.task_id,
|
||||
"confirmation_id": row.confirmation_id,
|
||||
"created_at": row.created_at.isoformat() if row.created_at else None,
|
||||
"metadata": row.payload,
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
memory_items = CONVERSATIONS.get((user_id, conv_id), [])
|
||||
return {
|
||||
"items": persistent_items,
|
||||
"memory_items": memory_items,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
"has_next": start + page_size < total,
|
||||
"from_latest": from_latest,
|
||||
"conversation_id": conv_id,
|
||||
}
|
||||
|
||||
|
||||
# #endregion get_history
|
||||
|
||||
|
||||
@router.get("/audit")
|
||||
|
||||
@@ -58,6 +58,8 @@ async def get_dashboards(
|
||||
filter_git_status: list[str] | None = Query(default=None),
|
||||
filter_llm_status: list[str] | None = Query(default=None),
|
||||
filter_changed_on: list[str] | None = Query(default=None),
|
||||
filter_changed_on_from: str | None = Query(default=None),
|
||||
filter_changed_on_to: str | None = Query(default=None),
|
||||
filter_actor: list[str] | None = Query(default=None),
|
||||
config_manager=Depends(get_config_manager),
|
||||
task_manager=Depends(get_task_manager),
|
||||
@@ -161,6 +163,8 @@ async def get_dashboards(
|
||||
llm_filters = _normalize_filter_values(filter_llm_status)
|
||||
changed_on_filters = _normalize_filter_values(filter_changed_on)
|
||||
actor_filters = _normalize_filter_values(filter_actor)
|
||||
changed_on_from = (filter_changed_on_from or "").strip() or None
|
||||
changed_on_to = (filter_changed_on_to or "").strip() or None
|
||||
has_column_filters = any(
|
||||
(
|
||||
title_filters,
|
||||
@@ -170,8 +174,10 @@ async def get_dashboards(
|
||||
actor_filters,
|
||||
)
|
||||
)
|
||||
has_date_range = bool(changed_on_from or changed_on_to)
|
||||
needs_full_scan = (
|
||||
has_column_filters
|
||||
or has_date_range
|
||||
or bool(can_apply_profile_filter)
|
||||
or bool(can_apply_slug_filter)
|
||||
)
|
||||
@@ -336,6 +342,13 @@ async def get_dashboards(
|
||||
):
|
||||
return False
|
||||
|
||||
if has_date_range:
|
||||
changed_on_dt_str = changed_on_raw[:10] if len(changed_on_raw) >= 10 else changed_on_raw
|
||||
if changed_on_from and changed_on_dt_str < changed_on_from:
|
||||
return False
|
||||
if changed_on_to and changed_on_dt_str > changed_on_to:
|
||||
return False
|
||||
|
||||
owners = dashboard.get("owners") or []
|
||||
if isinstance(owners, list):
|
||||
actor_value = ", ".join(
|
||||
@@ -354,6 +367,11 @@ async def get_dashboards(
|
||||
d for d in dashboards if _matches_dashboard_filters(d)
|
||||
]
|
||||
|
||||
if has_date_range and not has_column_filters:
|
||||
dashboards = [
|
||||
d for d in dashboards if _matches_dashboard_filters(d)
|
||||
]
|
||||
|
||||
total = len(dashboards)
|
||||
total_pages = (total + page_size - 1) // page_size if total > 0 else 1
|
||||
start_idx = (page - 1) * page_size
|
||||
|
||||
@@ -74,9 +74,16 @@ def _project_dashboard_response_items(
|
||||
projected: list[dict[str, Any]] = []
|
||||
for dashboard in dashboards:
|
||||
projected_dashboard = dict(dashboard)
|
||||
projected_dashboard["owners"] = _normalize_dashboard_owner_values(
|
||||
owners = _normalize_dashboard_owner_values(
|
||||
projected_dashboard.get("owners")
|
||||
)
|
||||
if not owners:
|
||||
modified_by = _normalize_owner_display_token(
|
||||
projected_dashboard.get("modified_by")
|
||||
)
|
||||
if modified_by:
|
||||
owners = [modified_by]
|
||||
projected_dashboard["owners"] = owners
|
||||
projected.append(projected_dashboard)
|
||||
return projected
|
||||
|
||||
|
||||
@@ -102,6 +102,7 @@ async def get_providers(
|
||||
# @BRIEF Fetch available models from an LLM provider by base_url+provider_type, or by provider_id.
|
||||
# @PRE User is authenticated. Either provider_id or base_url+provider_type must be provided.
|
||||
# @POST Returns a list of available model IDs.
|
||||
# @SIDE_EFFECT Makes HTTP call to LLM provider API; closes DB connection during network wait.
|
||||
# @RELATION CALLS -> [LLMProviderService]
|
||||
# @RELATION CALLS -> [LLMClient]
|
||||
@router.post("/providers/fetch-models")
|
||||
@@ -254,6 +255,7 @@ async def get_llm_status(
|
||||
# @BRIEF Create a new LLM provider configuration.
|
||||
# @PRE User is authenticated and has admin permissions.
|
||||
# @POST Returns the created LLMProviderConfig.
|
||||
# @SIDE_EFFECT Creates a new DB row for the LLM provider.
|
||||
# @RELATION CALLS -> [LLMProviderService]
|
||||
# @RELATION DEPENDS_ON -> [LLMProviderConfig]
|
||||
@router.post(
|
||||
@@ -292,6 +294,7 @@ async def create_provider(
|
||||
# @BRIEF Update an existing LLM provider configuration.
|
||||
# @PRE User is authenticated and has admin permissions.
|
||||
# @POST Returns the updated LLMProviderConfig.
|
||||
# @SIDE_EFFECT Updates a DB row for the LLM provider.
|
||||
# @RELATION CALLS -> [LLMProviderService]
|
||||
# @RELATION DEPENDS_ON -> [LLMProviderConfig]
|
||||
@router.put("/providers/{provider_id}", response_model=LLMProviderConfig)
|
||||
@@ -332,6 +335,7 @@ async def update_provider(
|
||||
# @BRIEF Delete an LLM provider configuration.
|
||||
# @PRE User is authenticated and has admin permissions.
|
||||
# @POST Returns success status.
|
||||
# @SIDE_EFFECT Deletes a DB row for the LLM provider; checks active validation tasks.
|
||||
# @RELATION CALLS -> [LLMProviderService]
|
||||
# @RELATION CALLS -> [ValidationPolicy]
|
||||
@router.delete("/providers/{provider_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
@@ -371,6 +375,7 @@ async def delete_provider(
|
||||
# @BRIEF Test connection to an LLM provider.
|
||||
# @PRE User is authenticated.
|
||||
# @POST Returns success status and message.
|
||||
# @SIDE_EFFECT Makes HTTP call to LLM provider API; decrypts stored API key.
|
||||
# @RELATION CALLS -> [LLMProviderService]
|
||||
# @RELATION DEPENDS_ON -> [LLMClient]
|
||||
@router.post("/providers/{provider_id}/test")
|
||||
@@ -427,6 +432,7 @@ async def test_connection(
|
||||
# @BRIEF Test connection with a provided configuration (not yet saved).
|
||||
# @PRE User is authenticated.
|
||||
# @POST Returns success status and message.
|
||||
# @SIDE_EFFECT Makes HTTP call to LLM provider API using provided config.
|
||||
# @RELATION DEPENDS_ON -> [LLMClient]
|
||||
# @RELATION DEPENDS_ON -> [LLMProviderConfig]
|
||||
@router.post("/providers/test")
|
||||
@@ -471,7 +477,9 @@ async def test_provider_config(
|
||||
class ProbeMaxImagesResponse(BaseModel):
|
||||
max_images: int | None
|
||||
method: str = "binary_search"
|
||||
# #endregion ProbeMaxImagesResponse
|
||||
|
||||
|
||||
# #endregion ProbeMaxImagesResponse
|
||||
|
||||
|
||||
# #region probe_max_images [C:4] [TYPE Function]
|
||||
@@ -479,6 +487,7 @@ class ProbeMaxImagesResponse(BaseModel):
|
||||
# @BRIEF Probe an LLM provider to discover its max images per request limit.
|
||||
# @PRE Provider exists and has valid API key.
|
||||
# @POST Returns the detected max_images limit and caches it on the provider.
|
||||
# @SIDE_EFFECT Makes multiple HTTP calls to LLM provider API; updates provider max_images in DB.
|
||||
# @RELATION CALLS -> [LLMProviderService]
|
||||
# @RELATION DEPENDS_ON -> [EXT:Library:openai]
|
||||
@router.post("/providers/{provider_id}/probe-max-images", response_model=ProbeMaxImagesResponse)
|
||||
|
||||
@@ -135,7 +135,7 @@ async def create_task(
|
||||
|
||||
# #region list_tasks [C:2] [TYPE Function]
|
||||
# @ingroup Api
|
||||
# @BRIEF Retrieve a list of tasks with pagination and optional status filter.
|
||||
# @BRIEF Retrieve a list of tasks with pagination and optional status/plugin/search filters.
|
||||
# @PRE task_manager must be available.
|
||||
# @POST Returns a list of tasks.
|
||||
# @RELATION CALLS -> [TaskManager]
|
||||
@@ -154,6 +154,9 @@ async def list_tasks(
|
||||
completed_only: bool = Query(
|
||||
False, description="Return only completed tasks (SUCCESS/FAILED)"
|
||||
),
|
||||
search: str | None = Query(
|
||||
None, max_length=200, description="Search tasks by id, plugin_id, or status"
|
||||
),
|
||||
task_manager: TaskManager = Depends(get_task_manager),
|
||||
_=Depends(has_permission("tasks", "READ")),
|
||||
):
|
||||
@@ -167,13 +170,19 @@ async def list_tasks(
|
||||
)
|
||||
plugin_filters.extend(TASK_TYPE_PLUGIN_MAP[task_type])
|
||||
|
||||
return task_manager.get_tasks(
|
||||
tasks = task_manager.get_tasks(
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
status=status_filter,
|
||||
plugin_ids=plugin_filters or None,
|
||||
completed_only=completed_only,
|
||||
search=search,
|
||||
)
|
||||
# Strip logs from list response — reduces payload dramatically.
|
||||
# Full task logs are available via GET /api/tasks/{task_id}/logs or WS /ws/logs/{task_id}.
|
||||
for task in tasks:
|
||||
task.logs = []
|
||||
return tasks
|
||||
|
||||
|
||||
# #endregion list_tasks
|
||||
|
||||
@@ -3,12 +3,6 @@
|
||||
# @BRIEF API routes for LLM-based SQL/dashboard translation management including terminology dictionary CRUD and import.
|
||||
# @LAYER API
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RATIONALE Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
|
||||
# @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity.
|
||||
# @POST Translation job, run, dictionary, correction, preview, and schedule CRUD operations are executed.
|
||||
|
||||
@@ -150,6 +150,7 @@ async def update_job(
|
||||
# @BRIEF Delete a translation job.
|
||||
# @PRE User has translate.job.delete permission.
|
||||
# @POST Job is deleted.
|
||||
# @SIDE_EFFECT Deletes job and associated runs from DB.
|
||||
@router.delete("/jobs/{job_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_job(
|
||||
job_id: str,
|
||||
|
||||
@@ -28,6 +28,7 @@ from ._router import router
|
||||
# @BRIEF Execute a translation job (trigger a run).
|
||||
# @PRE User has translate.job.execute permission.
|
||||
# @POST Returns the created translation run.
|
||||
# @SIDE_EFFECT Creates translation run DB row; dispatches background task (LLM calls, DB writes, Superset API calls).
|
||||
@router.post("/jobs/{job_id}/run", status_code=status.HTTP_201_CREATED)
|
||||
async def run_translation(
|
||||
job_id: str,
|
||||
@@ -134,6 +135,7 @@ async def run_translation(
|
||||
# @BRIEF Retry failed batches in a translation run.
|
||||
# @PRE User has translate.job.execute permission.
|
||||
# @POST Returns the updated translation run.
|
||||
# @SIDE_EFFECT Updates run DB row; dispatches background retry (LLM calls, DB writes).
|
||||
@router.post("/runs/{run_id}/retry")
|
||||
async def retry_run(
|
||||
run_id: str,
|
||||
@@ -161,6 +163,7 @@ async def retry_run(
|
||||
# @BRIEF Retry the SQL insert phase for a completed run.
|
||||
# @PRE User has translate.job.execute permission.
|
||||
# @POST Returns the updated run.
|
||||
# @SIDE_EFFECT Retries SQL insert; updates run DB row.
|
||||
@router.post("/runs/{run_id}/retry-insert")
|
||||
async def retry_insert(
|
||||
run_id: str,
|
||||
@@ -188,6 +191,7 @@ async def retry_insert(
|
||||
# @BRIEF Cancel a running translation.
|
||||
# @PRE User has translate.job.execute permission.
|
||||
# @POST Run is cancelled.
|
||||
# @SIDE_EFFECT Updates run status to cancelled in DB.
|
||||
@router.post("/runs/{run_id}/cancel")
|
||||
def cancel_run(
|
||||
run_id: str,
|
||||
|
||||
@@ -76,7 +76,7 @@ class TaskGraph:
|
||||
|
||||
# #region get_tasks [C:3] [TYPE Function] [SEMANTICS task,filter,paginate,sort]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Retrieves tasks with pagination and optional status/plugin filters.
|
||||
# @BRIEF Retrieves tasks with pagination and optional status/plugin/search filters.
|
||||
# @RELATION DEPENDS_ON -> [Task]
|
||||
def get_tasks(
|
||||
self,
|
||||
@@ -85,6 +85,7 @@ class TaskGraph:
|
||||
status: TaskStatus | None = None,
|
||||
plugin_ids: list[str] | None = None,
|
||||
completed_only: bool = False,
|
||||
search: str | None = None,
|
||||
) -> list[Task]:
|
||||
tasks = list(self.tasks.values())
|
||||
if status:
|
||||
@@ -96,6 +97,14 @@ class TaskGraph:
|
||||
tasks = [
|
||||
t for t in tasks if t.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]
|
||||
]
|
||||
if search:
|
||||
search_lower = search.lower()
|
||||
tasks = [
|
||||
t for t in tasks
|
||||
if search_lower in t.id.lower()
|
||||
or search_lower in t.plugin_id.lower()
|
||||
or search_lower in t.status.value.lower()
|
||||
]
|
||||
# Sort by started_at descending with tolerant handling of mixed tz-aware/naive values.
|
||||
def sort_key(task: Task) -> float:
|
||||
started_at = task.started_at
|
||||
|
||||
@@ -172,7 +172,7 @@ class TaskManager:
|
||||
|
||||
# #region get_tasks [TYPE Function] [C:3]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Retrieves tasks with pagination and optional status filter.
|
||||
# @BRIEF Retrieves tasks with pagination and optional status/plugin/search filters.
|
||||
def get_tasks(
|
||||
self,
|
||||
limit: int = 10,
|
||||
@@ -180,8 +180,9 @@ class TaskManager:
|
||||
status: TaskStatus | None = None,
|
||||
plugin_ids: list[str] | None = None,
|
||||
completed_only: bool = False,
|
||||
search: str | None = None,
|
||||
) -> list[Task]:
|
||||
return self.graph.get_tasks(limit, offset, status, plugin_ids, completed_only)
|
||||
return self.graph.get_tasks(limit, offset, status, plugin_ids, completed_only, search)
|
||||
# #endregion get_tasks
|
||||
|
||||
# #region load_persisted_tasks [TYPE Function] [C:2]
|
||||
|
||||
@@ -19,7 +19,7 @@ from datetime import UTC, datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import Depends, HTTPException, Request, status
|
||||
from fastapi import Depends, Header, HTTPException, Request, status
|
||||
from fastapi.security import OAuth2PasswordBearer
|
||||
from jose import JWTError
|
||||
|
||||
@@ -549,16 +549,28 @@ oauth2_scheme_optional = OAuth2PasswordBearer(tokenUrl="/api/auth/login", auto_e
|
||||
# @RELATION CALLS -> AuthRepository
|
||||
# @RELATION CALLS -> [is_token_blacklisted]
|
||||
# @BRIEF Dependency for retrieving currently authenticated user from a JWT.
|
||||
# @PRE JWT token provided in Authorization header.
|
||||
# Supports dual identity (FR-007/FR-019): when X-User-JWT header is present,
|
||||
# it takes precedence over the Authorization Bearer token for user identity.
|
||||
# @PRE JWT token provided in Authorization header (or X-User-JWT for dual identity).
|
||||
# @POST Returns User object if token is valid and not revoked.
|
||||
def get_current_user(token: str = Depends(oauth2_scheme), db=Depends(get_auth_db)):
|
||||
def get_current_user(
|
||||
token: str = Depends(oauth2_scheme),
|
||||
x_user_jwt: str | None = Header(None, alias="X-User-JWT"),
|
||||
db=Depends(get_auth_db),
|
||||
):
|
||||
credentials_exception = HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Could not validate credentials",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
# Dual identity (FR-019): X-User-JWT carries the end-user identity
|
||||
effective_token = x_user_jwt or token
|
||||
if not effective_token:
|
||||
raise credentials_exception
|
||||
|
||||
try:
|
||||
payload = decode_token(token)
|
||||
payload = decode_token(effective_token)
|
||||
username_value = payload.get("sub")
|
||||
if not isinstance(username_value, str) or not username_value:
|
||||
raise credentials_exception
|
||||
@@ -566,8 +578,8 @@ def get_current_user(token: str = Depends(oauth2_scheme), db=Depends(get_auth_db
|
||||
except JWTError:
|
||||
raise credentials_exception
|
||||
|
||||
# Check blacklist
|
||||
if is_token_blacklisted(token, db):
|
||||
# Check blacklist (only against the primary auth token, not forwarded user JWT)
|
||||
if not x_user_jwt and is_token_blacklisted(token, db):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Token has been revoked",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# #region LLMAnalysisPackage [TYPE Module] [SEMANTICS llm, analysis, plugin, package, export]
|
||||
# #region LLMAnalysisPackage [C:1] [TYPE Module] [SEMANTICS llm, analysis, plugin, package, export]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
|
||||
"""
|
||||
|
||||
@@ -10,7 +10,7 @@ from enum import Enum
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
# #region LLMProviderType [TYPE Class]
|
||||
# #region LLMProviderType [C:1] [TYPE Class] [SEMANTICS llm,provider,enum]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Enum for supported LLM providers.
|
||||
class LLMProviderType(str, Enum):
|
||||
@@ -20,7 +20,7 @@ class LLMProviderType(str, Enum):
|
||||
LITELLM = "litellm"
|
||||
# #endregion LLMProviderType
|
||||
|
||||
# #region LLMProviderConfig [TYPE Class]
|
||||
# #region LLMProviderConfig [C:1] [TYPE Class] [SEMANTICS llm,provider,config,pydantic]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Configuration for an LLM provider.
|
||||
class LLMProviderConfig(BaseModel):
|
||||
@@ -43,7 +43,7 @@ class LLMProviderConfig(BaseModel):
|
||||
)
|
||||
# #endregion LLMProviderConfig
|
||||
|
||||
# #region ValidationStatus [TYPE Class]
|
||||
# #region ValidationStatus [C:1] [TYPE Class] [SEMANTICS llm,validation,status,enum]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Enum for dashboard validation status.
|
||||
class ValidationStatus(str, Enum):
|
||||
@@ -53,7 +53,7 @@ class ValidationStatus(str, Enum):
|
||||
UNKNOWN = "UNKNOWN"
|
||||
# #endregion ValidationStatus
|
||||
|
||||
# #region DetectedIssue [TYPE Class]
|
||||
# #region DetectedIssue [C:1] [TYPE Class] [SEMANTICS llm,validation,issue,pydantic]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Model for a single issue detected during validation.
|
||||
class DetectedIssue(BaseModel):
|
||||
@@ -62,7 +62,7 @@ class DetectedIssue(BaseModel):
|
||||
location: str | None = None
|
||||
# #endregion DetectedIssue
|
||||
|
||||
# #region ValidationResult [TYPE Class]
|
||||
# #region ValidationResult [C:1] [TYPE Class] [SEMANTICS llm,validation,result,pydantic]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Model for dashboard validation result.
|
||||
class ValidationResult(BaseModel):
|
||||
|
||||
@@ -37,7 +37,7 @@ from ._topology import build_dashboard_topology
|
||||
from .service import DatasetHealthChecker, LLMClient, ScreenshotService
|
||||
|
||||
|
||||
# #region _is_masked_or_invalid_api_key [TYPE Function]
|
||||
# #region _is_masked_or_invalid_api_key [C:2] [TYPE Function] [SEMANTICS llm,auth,api-key,validation]
|
||||
# @BRIEF Guards against placeholder or malformed API keys in runtime.
|
||||
# @PRE value may be None.
|
||||
# @POST Returns True when value cannot be used for authenticated provider calls.
|
||||
@@ -52,7 +52,7 @@ def _is_masked_or_invalid_api_key(value: str | None) -> bool:
|
||||
# #endregion _is_masked_or_invalid_api_key
|
||||
|
||||
|
||||
# #region _json_safe_value [TYPE Function]
|
||||
# #region _json_safe_value [C:2] [TYPE Function] [SEMANTICS llm,json,serialization,utility]
|
||||
# @BRIEF Recursively normalize payload values for JSON serialization.
|
||||
# @PRE value may be nested dict/list with datetime values.
|
||||
# @POST datetime values are converted to ISO strings.
|
||||
@@ -67,7 +67,7 @@ def _json_safe_value(value: Any):
|
||||
# #endregion _json_safe_value
|
||||
|
||||
|
||||
# #region _ensure_json_prompt [TYPE Function]
|
||||
# #region _ensure_json_prompt [C:2] [TYPE Function] [SEMANTICS llm,prompt,json,format]
|
||||
# @BRIEF Appends JSON format instruction to any prompt that lacks it, so LLM output is always parseable.
|
||||
# @PRE prompt may be None or any string.
|
||||
# @POST Returns prompt with JSON format instruction appended if not already present.
|
||||
@@ -159,9 +159,10 @@ def _update_run_status(db, run_id: str | None) -> None:
|
||||
# #endregion _update_run_status
|
||||
|
||||
|
||||
# #region DashboardValidationPlugin [TYPE Class]
|
||||
# #region DashboardValidationPlugin [C:4] [TYPE Class] [SEMANTICS llm,validation,plugin,dashboard]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Plugin for automated dashboard health analysis using LLMs.
|
||||
# @SIDE_EFFECT Captures screenshots; calls LLM APIs; writes DB records; sends notifications.
|
||||
# @RELATION IMPLEMENTS -> [PluginBase]
|
||||
class DashboardValidationPlugin(PluginBase):
|
||||
@property
|
||||
@@ -205,7 +206,7 @@ class DashboardValidationPlugin(PluginBase):
|
||||
"required": ["dashboard_id", "environment_id", "provider_id"],
|
||||
}
|
||||
|
||||
# region DashboardValidationPlugin.execute [TYPE Function]
|
||||
# #region DashboardValidationPlugin.execute [TYPE Function]
|
||||
# @PURPOSE: Dispatches dashboard validation to Path A (multimodal screenshot + LLM)
|
||||
# or Path B (API-based topology + dataset health + text LLM) based on screenshot_enabled.
|
||||
# @PARAM params (Dict[str, Any]) - Validation parameters including screenshot_enabled flag.
|
||||
@@ -349,9 +350,9 @@ class DashboardValidationPlugin(PluginBase):
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
# endregion DashboardValidationPlugin.execute
|
||||
# #endregion DashboardValidationPlugin.execute
|
||||
|
||||
# region DashboardValidationPlugin._execute_path_a [TYPE Function]
|
||||
# #region DashboardValidationPlugin._execute_path_a [TYPE Function]
|
||||
# @PURPOSE: Path A — capture Playwright screenshot, split into chunks for large dashboards,
|
||||
# send as multimodal LLM call, persist result.
|
||||
# @PARAM params (Dict) - Validation parameters including dashboard_id, environment_id.
|
||||
@@ -560,9 +561,9 @@ class DashboardValidationPlugin(PluginBase):
|
||||
|
||||
return result_payload
|
||||
|
||||
# endregion DashboardValidationPlugin._execute_path_a
|
||||
# #endregion DashboardValidationPlugin._execute_path_a
|
||||
|
||||
# region DashboardValidationPlugin._execute_path_b [TYPE Function]
|
||||
# #region DashboardValidationPlugin._execute_path_b [TYPE Function]
|
||||
# @PURPOSE: Path B — fetch dashboard structure via API, check dataset health,
|
||||
# build topology text, send text-only LLM call, persist result.
|
||||
# @PARAM params (Dict) - Validation parameters including dashboard_id, environment_id,
|
||||
@@ -772,9 +773,9 @@ class DashboardValidationPlugin(PluginBase):
|
||||
|
||||
return result_payload
|
||||
|
||||
# endregion DashboardValidationPlugin._execute_path_b
|
||||
# #endregion DashboardValidationPlugin._execute_path_b
|
||||
|
||||
# region DashboardValidationPlugin._fetch_dashboard_logs [TYPE Function]
|
||||
# #region DashboardValidationPlugin._fetch_dashboard_logs [TYPE Function]
|
||||
# @PURPOSE: Fetch execution logs for a dashboard from the Superset /api/v1/log/ endpoint.
|
||||
# @PARAM env (Environment) - Superset environment config for API authentication.
|
||||
# @PARAM dashboard_id (str) - Dashboard ID to filter logs.
|
||||
@@ -842,14 +843,15 @@ class DashboardValidationPlugin(PluginBase):
|
||||
|
||||
return logs, timings
|
||||
|
||||
# endregion DashboardValidationPlugin._fetch_dashboard_logs
|
||||
# #endregion DashboardValidationPlugin._fetch_dashboard_logs
|
||||
|
||||
# endregion DashboardValidationPlugin
|
||||
# #endregion DashboardValidationPlugin
|
||||
|
||||
|
||||
# #region DocumentationPlugin [TYPE Class]
|
||||
# #region DocumentationPlugin [C:4] [TYPE Class] [SEMANTICS llm,documentation,plugin,dataset]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Plugin for automated dataset documentation using LLMs.
|
||||
# @SIDE_EFFECT Calls LLM API; updates dataset metadata in Superset.
|
||||
# @RELATION IMPLEMENTS -> [PluginBase]
|
||||
class DocumentationPlugin(PluginBase):
|
||||
@property
|
||||
@@ -879,7 +881,7 @@ class DocumentationPlugin(PluginBase):
|
||||
"required": ["dataset_id", "environment_id", "provider_id"]
|
||||
}
|
||||
|
||||
# region DocumentationPlugin.execute [TYPE Function]
|
||||
# #region DocumentationPlugin.execute [TYPE Function]
|
||||
# @PURPOSE: Executes the dataset documentation task with TaskContext support.
|
||||
# @PARAM params (Dict[str, Any]) - Documentation parameters.
|
||||
# @PARAM context (Optional[TaskContext]) - Task context for logging with source attribution.
|
||||
@@ -1001,7 +1003,7 @@ class DocumentationPlugin(PluginBase):
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
# endregion DocumentationPlugin.execute
|
||||
# #endregion DocumentationPlugin.execute
|
||||
# #endregion DocumentationPlugin
|
||||
|
||||
# #endregion LLMAnalysisPlugin
|
||||
|
||||
@@ -37,18 +37,19 @@ SCREENSHOT_SERVICE_TIMEOUT_MS = 120000
|
||||
LLM_HTTP_TIMEOUT_S = 120 # seconds (httpx client timeout)
|
||||
DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
|
||||
|
||||
# #region ScreenshotService [TYPE Class]
|
||||
# #region ScreenshotService [C:4] [TYPE Class] [SEMANTICS llm,screenshot,playwright]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Handles capturing screenshots of Superset dashboards.
|
||||
# @SIDE_EFFECT Launches Playwright browser; captures screenshots to disk.
|
||||
class ScreenshotService:
|
||||
# region ScreenshotService.__init__ [TYPE Function]
|
||||
# #region ScreenshotService.__init__ [C:1] [TYPE Function] [SEMANTICS init]
|
||||
# @PURPOSE Initializes the ScreenshotService with environment configuration.
|
||||
# @PRE env is a valid Environment object.
|
||||
def __init__(self, env: Environment):
|
||||
self.env = env
|
||||
# endregion ScreenshotService.__init__
|
||||
# #endregion ScreenshotService.__init__
|
||||
|
||||
# region ScreenshotService._find_first_visible_locator [TYPE Function]
|
||||
# #region ScreenshotService._find_first_visible_locator [TYPE Function]
|
||||
# @PURPOSE Resolve the first visible locator from multiple Playwright locator strategies.
|
||||
# @PRE candidates is a non-empty list of locator-like objects.
|
||||
# @POST Returns a locator ready for interaction or None when nothing matches.
|
||||
@@ -63,9 +64,9 @@ class ScreenshotService:
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
# endregion ScreenshotService._find_first_visible_locator
|
||||
# #endregion ScreenshotService._find_first_visible_locator
|
||||
|
||||
# region ScreenshotService._iter_login_roots [TYPE Function]
|
||||
# #region ScreenshotService._iter_login_roots [TYPE Function]
|
||||
# @PURPOSE Enumerate page and child frames where login controls may be rendered.
|
||||
# @PRE page is a Playwright page-like object.
|
||||
# @POST Returns ordered roots starting with main page followed by frames.
|
||||
@@ -79,9 +80,9 @@ class ScreenshotService:
|
||||
except Exception:
|
||||
pass
|
||||
return roots
|
||||
# endregion ScreenshotService._iter_login_roots
|
||||
# #endregion ScreenshotService._iter_login_roots
|
||||
|
||||
# region ScreenshotService._extract_hidden_login_fields [TYPE Function]
|
||||
# #region ScreenshotService._extract_hidden_login_fields [TYPE Function]
|
||||
# @PURPOSE Collect hidden form fields required for direct login POST fallback.
|
||||
# @PRE Login page is loaded.
|
||||
# @POST Returns hidden input name/value mapping aggregated from page and child frames.
|
||||
@@ -100,18 +101,18 @@ class ScreenshotService:
|
||||
except Exception:
|
||||
continue
|
||||
return hidden_fields
|
||||
# endregion ScreenshotService._extract_hidden_login_fields
|
||||
# #endregion ScreenshotService._extract_hidden_login_fields
|
||||
|
||||
# region ScreenshotService._extract_csrf_token [TYPE Function]
|
||||
# #region ScreenshotService._extract_csrf_token [TYPE Function]
|
||||
# @PURPOSE Resolve CSRF token value from main page or embedded login frame.
|
||||
# @PRE Login page is loaded.
|
||||
# @POST Returns first non-empty csrf token or empty string.
|
||||
async def _extract_csrf_token(self, page) -> str:
|
||||
hidden_fields = await self._extract_hidden_login_fields(page)
|
||||
return str(hidden_fields.get("csrf_token") or "").strip()
|
||||
# endregion ScreenshotService._extract_csrf_token
|
||||
# #endregion ScreenshotService._extract_csrf_token
|
||||
|
||||
# region ScreenshotService._response_looks_like_login_page [TYPE Function]
|
||||
# #region ScreenshotService._response_looks_like_login_page [TYPE Function]
|
||||
# @PURPOSE Detect when fallback login POST returned the login form again instead of an authenticated page.
|
||||
# @PRE response_text is normalized HTML or text from login POST response.
|
||||
# @POST Returns True when login-page markers dominate the response body.
|
||||
@@ -128,9 +129,9 @@ class ScreenshotService:
|
||||
'name="csrf_token"',
|
||||
]
|
||||
return sum(marker in normalized for marker in markers) >= 3
|
||||
# endregion ScreenshotService._response_looks_like_login_page
|
||||
# #endregion ScreenshotService._response_looks_like_login_page
|
||||
|
||||
# region ScreenshotService._redirect_looks_authenticated [TYPE Function]
|
||||
# #region ScreenshotService._redirect_looks_authenticated [TYPE Function]
|
||||
# @PURPOSE Treat non-login redirects after form POST as successful authentication without waiting for redirect target.
|
||||
# @PRE redirect_location may be empty or relative.
|
||||
# @POST Returns True when redirect target does not point back to login flow.
|
||||
@@ -139,9 +140,9 @@ class ScreenshotService:
|
||||
if not normalized:
|
||||
return True
|
||||
return "/login" not in normalized
|
||||
# endregion ScreenshotService._redirect_looks_authenticated
|
||||
# #endregion ScreenshotService._redirect_looks_authenticated
|
||||
|
||||
# region ScreenshotService._submit_login_via_form_post [TYPE Function]
|
||||
# #region ScreenshotService._submit_login_via_form_post [TYPE Function]
|
||||
# @PURPOSE Fallback login path that submits credentials directly with csrf token.
|
||||
# @PRE login_url is same-origin and csrf token can be read from DOM.
|
||||
# @POST Browser context receives authenticated cookies when login succeeds.
|
||||
@@ -204,9 +205,9 @@ class ScreenshotService:
|
||||
payload={"status": response_status, "url": response_url, "login_markup": looks_like_login_page, "snippet": text_snippet},
|
||||
)
|
||||
return not looks_like_login_page
|
||||
# endregion ScreenshotService._submit_login_via_form_post
|
||||
# #endregion ScreenshotService._submit_login_via_form_post
|
||||
|
||||
# region ScreenshotService._find_login_field_locator [TYPE Function]
|
||||
# #region ScreenshotService._find_login_field_locator [TYPE Function]
|
||||
# @PURPOSE Resolve login form input using semantic label text plus generic visible-input fallbacks.
|
||||
# @PRE field_name is `username` or `password`.
|
||||
# @POST Returns a locator for the corresponding input or None.
|
||||
@@ -244,9 +245,9 @@ class ScreenshotService:
|
||||
return locator
|
||||
|
||||
return None
|
||||
# endregion ScreenshotService._find_login_field_locator
|
||||
# #endregion ScreenshotService._find_login_field_locator
|
||||
|
||||
# region ScreenshotService._find_submit_locator [TYPE Function]
|
||||
# #region ScreenshotService._find_submit_locator [TYPE Function]
|
||||
# @PURPOSE Resolve login submit button from main page or embedded auth frame.
|
||||
# @PRE page is ready for login interaction.
|
||||
# @POST Returns visible submit locator or None.
|
||||
@@ -264,9 +265,9 @@ class ScreenshotService:
|
||||
if locator:
|
||||
return locator
|
||||
return None
|
||||
# endregion ScreenshotService._find_submit_locator
|
||||
# #endregion ScreenshotService._find_submit_locator
|
||||
|
||||
# region ScreenshotService._goto_resilient [TYPE Function]
|
||||
# #region ScreenshotService._goto_resilient [TYPE Function]
|
||||
# @PURPOSE Navigate without relying on networkidle for pages with long-polling or persistent requests.
|
||||
# @PRE page is a valid Playwright page and url is non-empty.
|
||||
# @POST Returns last navigation response or raises when both primary and fallback waits fail.
|
||||
@@ -287,9 +288,9 @@ class ScreenshotService:
|
||||
error=str(primary_error),
|
||||
)
|
||||
return await page.goto(url, wait_until=fallback_wait_until, timeout=timeout)
|
||||
# endregion ScreenshotService._goto_resilient
|
||||
# #endregion ScreenshotService._goto_resilient
|
||||
|
||||
# region ScreenshotService._wait_for_charts_stabilized [TYPE Function] [C:2]
|
||||
# #region ScreenshotService._wait_for_charts_stabilized [TYPE Function] [C:2]
|
||||
# @BRIEF Wait until chart elements have non-zero dimensions, with polling.
|
||||
# @PRE page is a valid Playwright page.
|
||||
# @POST Waits for chart stabilization or raises on timeout (handled internally).
|
||||
@@ -314,9 +315,9 @@ class ScreenshotService:
|
||||
}""", timeout=timeout_ms)
|
||||
except Exception:
|
||||
logger.explore("Chart stabilization wait timed out, proceeding anyway", error="Timed out waiting for charts to render")
|
||||
# endregion ScreenshotService._wait_for_charts_stabilized
|
||||
# #endregion ScreenshotService._wait_for_charts_stabilized
|
||||
|
||||
# region ScreenshotService._wait_for_resize_rendered [TYPE Function] [C:2]
|
||||
# #region ScreenshotService._wait_for_resize_rendered [TYPE Function] [C:2]
|
||||
# @BRIEF Wait for charts to re-render after viewport resize.
|
||||
# @PRE page is a valid Playwright page; chart_count_before contains pre-resize element counts.
|
||||
# @POST Waits for chart content to return or timeout.
|
||||
@@ -334,9 +335,9 @@ class ScreenshotService:
|
||||
}""", arg=chart_count_before, timeout=timeout_ms)
|
||||
except Exception:
|
||||
logger.explore("Re-render wait timed out after viewport resize, proceeding anyway", error="Timed out waiting for charts to re-render after resize")
|
||||
# endregion ScreenshotService._wait_for_resize_rendered
|
||||
# #endregion ScreenshotService._wait_for_resize_rendered
|
||||
|
||||
# region ScreenshotService._save_debug_screenshot [TYPE Function] [C:1]
|
||||
# #region ScreenshotService._save_debug_screenshot [TYPE Function] [C:1]
|
||||
# @BRIEF Save a debug screenshot to a temp directory for diagnostic purposes.
|
||||
# @PRE debug_dir exists and is writable.
|
||||
# @POST Returns the debug path or None on failure.
|
||||
@@ -349,9 +350,9 @@ class ScreenshotService:
|
||||
return debug_path
|
||||
except Exception:
|
||||
return None
|
||||
# endregion ScreenshotService._save_debug_screenshot
|
||||
# #endregion ScreenshotService._save_debug_screenshot
|
||||
|
||||
# region ScreenshotService._launch_and_login [TYPE Function] [C:4]
|
||||
# #region ScreenshotService._launch_and_login [TYPE Function] [C:4]
|
||||
# @PURPOSE Launch browser, log in to Superset, navigate to dashboard URL.
|
||||
# @PRE dashboard_id is valid, playwright instance is provided.
|
||||
# @POST Returns (browser, context, page) tuple with active session.
|
||||
@@ -531,9 +532,9 @@ class ScreenshotService:
|
||||
await self._wait_for_charts_stabilized(page)
|
||||
logger.reflect("Login and navigation to dashboard successful", payload={"dashboard_id": dashboard_id})
|
||||
return browser, context, page
|
||||
# endregion ScreenshotService._launch_and_login
|
||||
# #endregion ScreenshotService._launch_and_login
|
||||
|
||||
# region ScreenshotService.capture_dashboard_chunks [TYPE Function] [C:4]
|
||||
# #region ScreenshotService.capture_dashboard_chunks [TYPE Function] [C:4]
|
||||
# @PURPOSE Capture per-tab screenshots: login → navigate → switch tabs → per-tab CDP screenshots.
|
||||
# @PRE dashboard_id is valid, browser available.
|
||||
# @POST Returns list of {tab_name, path} dicts — one per tab.
|
||||
@@ -714,9 +715,9 @@ class ScreenshotService:
|
||||
return results
|
||||
finally:
|
||||
await browser.close()
|
||||
# endregion ScreenshotService.capture_dashboard_chunks
|
||||
# #endregion ScreenshotService.capture_dashboard_chunks
|
||||
|
||||
# region ScreenshotService.capture_dashboard [TYPE Function] [C:4]
|
||||
# #region ScreenshotService.capture_dashboard [TYPE Function] [C:4]
|
||||
# @PURPOSE Captures multi-chunk screenshots, converts for LLM, archives to WebP.
|
||||
# @PRE dashboard_id is a valid string, output_path is a writable path.
|
||||
# @POST Returns list of {original, webp_path} dicts from WebP archive.
|
||||
@@ -765,9 +766,9 @@ class ScreenshotService:
|
||||
|
||||
# 4. Return JPEGs for LLM — caller cleans up after analysis
|
||||
return jpeg_paths, archive_results
|
||||
# endregion ScreenshotService.capture_dashboard
|
||||
# #endregion ScreenshotService.capture_dashboard
|
||||
|
||||
# region ScreenshotService._convert_screenshots_for_llm [TYPE Function] [C:2]
|
||||
# #region ScreenshotService._convert_screenshots_for_llm [TYPE Function] [C:2]
|
||||
# @BRIEF Convert PNG screenshots to JPEG for LLM transmission.
|
||||
# @PRE png_paths is a list of existing PNG file paths.
|
||||
# @POST Returns list of JPEG paths. JPEG files should be deleted after LLM call.
|
||||
@@ -800,9 +801,9 @@ class ScreenshotService:
|
||||
logger.explore("Failed to convert screenshot for LLM", payload={"png_path": png_path}, error=str(e))
|
||||
|
||||
return jpeg_paths
|
||||
# endregion ScreenshotService._convert_screenshots_for_llm
|
||||
# #endregion ScreenshotService._convert_screenshots_for_llm
|
||||
|
||||
# region ScreenshotService._archive_screenshots_as_webp [TYPE Function] [C:2]
|
||||
# #region ScreenshotService._archive_screenshots_as_webp [TYPE Function] [C:2]
|
||||
# @BRIEF Convert PNG screenshots to WebP for archive.
|
||||
# @PRE png_paths is a list of existing PNG file paths.
|
||||
# @POST Returns list of {original, webp_path} dicts. PNG deleted after successful WebP save.
|
||||
@@ -834,9 +835,9 @@ class ScreenshotService:
|
||||
results.append({"original": png_path, "webp_path": None})
|
||||
|
||||
return results
|
||||
# endregion ScreenshotService._archive_screenshots_as_webp
|
||||
# #endregion ScreenshotService._archive_screenshots_as_webp
|
||||
|
||||
# region ScreenshotService._cleanup_temp_files [TYPE Function] [C:1]
|
||||
# #region ScreenshotService._cleanup_temp_files [TYPE Function] [C:1]
|
||||
# @BRIEF Delete temporary files (PNG, JPEG intermediates).
|
||||
@staticmethod
|
||||
def _cleanup_temp_files(paths: list[str]) -> None:
|
||||
@@ -847,14 +848,15 @@ class ScreenshotService:
|
||||
os.remove(path)
|
||||
except Exception as e:
|
||||
logger.explore("Failed to delete temporary file", payload={"path": path}, error=str(e))
|
||||
# endregion ScreenshotService._cleanup_temp_files
|
||||
# #endregion ScreenshotService._cleanup_temp_files
|
||||
# #endregion ScreenshotService
|
||||
|
||||
# #region LLMClient [TYPE Class]
|
||||
# #region LLMClient [C:4] [TYPE Class] [SEMANTICS llm,client,provider,openai]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Wrapper for LLM provider APIs.
|
||||
# @SIDE_EFFECT Makes HTTP calls to LLM provider APIs.
|
||||
class LLMClient:
|
||||
# region LLMClient.__init__ [TYPE Function]
|
||||
# #region LLMClient.__init__ [TYPE Function]
|
||||
# @PURPOSE Initializes the LLMClient with provider settings.
|
||||
# @PRE api_key, base_url, and default_model are non-empty strings.
|
||||
def __init__(self, provider_type: LLMProviderType, api_key: str, base_url: str, default_model: str):
|
||||
@@ -911,9 +913,9 @@ class LLMClient:
|
||||
default_headers=default_headers,
|
||||
http_client=http_client,
|
||||
)
|
||||
# endregion LLMClient.__init__
|
||||
# #endregion LLMClient.__init__
|
||||
|
||||
# region LLMClient._get_ssl_verify [TYPE Function]
|
||||
# #region LLMClient._get_ssl_verify [TYPE Function]
|
||||
# @PURPOSE Resolve SSL verification flag from environment.
|
||||
# @POST Returns SSLContext with system CA dir when enabled,
|
||||
# False when LLM_SSL_VERIFY env var is "false"/"0"/"no"/"off".
|
||||
@@ -937,9 +939,9 @@ class LLMClient:
|
||||
return ssl.create_default_context(capath=ca_dir)
|
||||
# fallback: если директории нет (редко), используем дефолтный
|
||||
return ssl.create_default_context()
|
||||
# endregion LLMClient._get_ssl_verify
|
||||
# #endregion LLMClient._get_ssl_verify
|
||||
|
||||
# region LLMClient._format_connection_error [TYPE Function]
|
||||
# #region LLMClient._format_connection_error [TYPE Function]
|
||||
# @PURPOSE Format exception chain for diagnostics, extracting httpx cause details.
|
||||
# @POST Returns a human-readable string with the full error chain.
|
||||
@staticmethod
|
||||
@@ -950,9 +952,9 @@ class LLMClient:
|
||||
parts.append(f" └─ {type(cause).__name__}: {cause!s}")
|
||||
cause = cause.__cause__ or cause.__context__
|
||||
return "\n".join(parts)
|
||||
# endregion LLMClient._format_connection_error
|
||||
# #endregion LLMClient._format_connection_error
|
||||
|
||||
# region LLMClient._supports_json_response_format [TYPE Function]
|
||||
# #region LLMClient._supports_json_response_format [TYPE Function]
|
||||
# @PURPOSE Detect whether provider/model is likely compatible with response_format=json_object.
|
||||
# @PRE Client initialized with base_url and default_model.
|
||||
# @POST Returns False for known-incompatible combinations to avoid avoidable 400 errors.
|
||||
@@ -967,9 +969,9 @@ class LLMClient:
|
||||
if "stepfun/" in model or model.startswith("step-"):
|
||||
return False
|
||||
return True
|
||||
# endregion LLMClient._supports_json_response_format
|
||||
# #endregion LLMClient._supports_json_response_format
|
||||
|
||||
# region LLMClient.get_json_completion [TYPE Function]
|
||||
# #region LLMClient.get_json_completion [TYPE Function]
|
||||
# @PURPOSE Helper to handle LLM calls with JSON mode and fallback parsing.
|
||||
# @PRE messages is a list of valid message dictionaries.
|
||||
# @POST Returns a parsed JSON dictionary.
|
||||
@@ -1107,9 +1109,9 @@ class LLMClient:
|
||||
return json.loads(json_str)
|
||||
else:
|
||||
raise
|
||||
# endregion LLMClient.get_json_completion
|
||||
# #endregion LLMClient.get_json_completion
|
||||
|
||||
# region LLMClient.test_runtime_connection [TYPE Function]
|
||||
# #region LLMClient.test_runtime_connection [TYPE Function]
|
||||
# @PURPOSE Validate provider credentials using the same chat completions transport as runtime analysis.
|
||||
# @PRE Client is initialized with provider credentials and default_model.
|
||||
# @POST Returns lightweight JSON payload when runtime auth/model path is valid.
|
||||
@@ -1123,9 +1125,9 @@ class LLMClient:
|
||||
}
|
||||
]
|
||||
return await self.get_json_completion(messages)
|
||||
# endregion LLMClient.test_runtime_connection
|
||||
# #endregion LLMClient.test_runtime_connection
|
||||
|
||||
# region LLMClient.fetch_models [TYPE Function]
|
||||
# #region LLMClient.fetch_models [TYPE Function]
|
||||
# @PURPOSE Fetch available models from the provider's API.
|
||||
# @PRE Client is initialized with provider credentials.
|
||||
# @POST Returns a list of model ID strings.
|
||||
@@ -1148,9 +1150,9 @@ class LLMClient:
|
||||
error=str(e),
|
||||
)
|
||||
raise
|
||||
# endregion LLMClient.fetch_models
|
||||
# #endregion LLMClient.fetch_models
|
||||
|
||||
# region LLMClient.analyze_dashboard [TYPE Function]
|
||||
# #region LLMClient.analyze_dashboard [TYPE Function]
|
||||
# @PURPOSE Sends dashboard data (screenshot + logs) to LLM for health analysis.
|
||||
# @PRE screenshot_path exists, logs is a list of strings.
|
||||
# @POST Returns a structured analysis dictionary (status, summary, issues).
|
||||
@@ -1169,9 +1171,9 @@ class LLMClient:
|
||||
logs=logs,
|
||||
prompt_template=prompt_template,
|
||||
)
|
||||
# endregion LLMClient.analyze_dashboard
|
||||
# #endregion LLMClient.analyze_dashboard
|
||||
|
||||
# region LLMClient._reduce_image_quality [TYPE Function] [C:2]
|
||||
# #region LLMClient._reduce_image_quality [TYPE Function] [C:2]
|
||||
# @PURPOSE Open, resize, and compress a screenshot image for LLM consumption.
|
||||
# @PRE path points to an existing image file.
|
||||
# @POST Returns (base64_str, byte_size) tuple.
|
||||
@@ -1199,9 +1201,9 @@ class LLMClient:
|
||||
img.save(buffer, format="JPEG", quality=image_quality, optimize=True)
|
||||
raw = buffer.getvalue()
|
||||
return base64.b64encode(raw).decode("utf-8"), len(raw)
|
||||
# endregion LLMClient._reduce_image_quality
|
||||
# #endregion LLMClient._reduce_image_quality
|
||||
|
||||
# region LLMClient._estimate_payload_size [TYPE Function] [C:2]
|
||||
# #region LLMClient._estimate_payload_size [TYPE Function] [C:2]
|
||||
# @PURPOSE Estimate LLM payload size in tokens before sending.
|
||||
# @POST Returns {estimated_tokens, exceeds_limit, pct_of_limit} dict.
|
||||
# @RATIONALE FR-056: if >80% of model context window, trigger quality reduction.
|
||||
@@ -1226,9 +1228,9 @@ class LLMClient:
|
||||
"exceeds_limit": exceeds_limit,
|
||||
"pct_of_limit": round(total_tokens / model_context * 100, 1),
|
||||
}
|
||||
# endregion LLMClient._estimate_payload_size
|
||||
# #endregion LLMClient._estimate_payload_size
|
||||
|
||||
# region LLMClient._deduplicate_issues [TYPE Function] [C:2]
|
||||
# #region LLMClient._deduplicate_issues [TYPE Function] [C:2]
|
||||
# @PURPOSE Deduplicate issues by (severity, message, location) while preserving order.
|
||||
def _deduplicate_issues(self, issues: list[dict]) -> list[dict]:
|
||||
seen: set[tuple[str, str, str]] = set()
|
||||
@@ -1240,9 +1242,9 @@ class LLMClient:
|
||||
result.append(issue)
|
||||
return result
|
||||
|
||||
# endregion LLMClient._deduplicate_issues
|
||||
# #endregion LLMClient._deduplicate_issues
|
||||
|
||||
# region LLMClient._optimize_images [TYPE Function] [C:2]
|
||||
# #region LLMClient._optimize_images [TYPE Function] [C:2]
|
||||
# @PURPOSE Convert screenshot paths to base64 at given quality, with fallback to raw read.
|
||||
def _optimize_images(self, paths: list[str], max_width: int, quality: int) -> list[str]:
|
||||
encoded: list[str] = []
|
||||
@@ -1258,9 +1260,9 @@ class LLMClient:
|
||||
encoded.append(b64)
|
||||
return encoded
|
||||
|
||||
# endregion LLMClient._optimize_images
|
||||
# #endregion LLMClient._optimize_images
|
||||
|
||||
# region LLMClient._merge_chunk_results [TYPE Function] [C:2]
|
||||
# #region LLMClient._merge_chunk_results [TYPE Function] [C:2]
|
||||
# @PURPOSE Merge multiple chunk analyses into one. Takes the worst status,
|
||||
# concatenates summaries, and deduplicates issues.
|
||||
# @PRE chunks is a non-empty list of {status, summary, issues} dicts.
|
||||
@@ -1286,9 +1288,9 @@ class LLMClient:
|
||||
}
|
||||
return merged
|
||||
|
||||
# endregion LLMClient._merge_chunk_results
|
||||
# #endregion LLMClient._merge_chunk_results
|
||||
|
||||
# region LLMClient._call_llm_for_images [TYPE Function] [C:2]
|
||||
# #region LLMClient._call_llm_for_images [TYPE Function] [C:2]
|
||||
# @PURPOSE Send a single chunk of images to the LLM and return parsed result.
|
||||
async def _call_llm_for_images(
|
||||
self, encoded_images: list[str], prompt: str
|
||||
@@ -1302,9 +1304,9 @@ class LLMClient:
|
||||
messages = [{"role": "user", "content": content}]
|
||||
return await self.get_json_completion(messages)
|
||||
|
||||
# endregion LLMClient._call_llm_for_images
|
||||
# #endregion LLMClient._call_llm_for_images
|
||||
|
||||
# region LLMClient.analyze_dashboard_multimodal [TYPE Function] [C:3]
|
||||
# #region LLMClient.analyze_dashboard_multimodal [TYPE Function] [C:3]
|
||||
# @PURPOSE Path A: send screenshots + logs to multimodal LLM, with chunking support.
|
||||
# @PRE screenshot_paths is a non-empty list of paths.
|
||||
# tab_labels, if provided, must have the same length as screenshot_paths.
|
||||
@@ -1404,9 +1406,9 @@ class LLMClient:
|
||||
}
|
||||
|
||||
return result
|
||||
# endregion LLMClient.analyze_dashboard_multimodal
|
||||
# #endregion LLMClient.analyze_dashboard_multimodal
|
||||
|
||||
# region LLMClient.analyze_dashboard_text_batch [TYPE Function] [C:3]
|
||||
# #region LLMClient.analyze_dashboard_text_batch [TYPE Function] [C:3]
|
||||
# @PURPOSE Path B batch: multiple dashboards in a single text-only LLM call.
|
||||
# @PRE payloads is a non-empty list of {dashboard_id, topology, dataset_health, log_text} dicts.
|
||||
# @POST Returns dict {dashboards: [{dashboard_id, status, summary, issues}]}.
|
||||
@@ -1462,7 +1464,7 @@ class LLMClient:
|
||||
|
||||
messages = [{"role": "user", "content": full_prompt}]
|
||||
return await self.get_json_completion(messages)
|
||||
# endregion LLMClient.analyze_dashboard_text_batch
|
||||
# #endregion LLMClient.analyze_dashboard_text_batch
|
||||
|
||||
# #endregion LLMClient
|
||||
|
||||
@@ -1476,15 +1478,15 @@ class LLMClient:
|
||||
# are invisible to the LLM validation. Screenshot captures visual state but doesn't
|
||||
# verify that data actually arrived (vs. cache).
|
||||
class DatasetHealthChecker:
|
||||
# region DatasetHealthChecker.__init__ [TYPE Function]
|
||||
# #region DatasetHealthChecker.__init__ [TYPE Function]
|
||||
# @PURPOSE Initialize with a SupersetClient-compatible instance.
|
||||
# @PRE client is a SupersetClient (sync, wrapped via asyncio.to_thread) or AsyncSupersetClient.
|
||||
# @POST self.client is ready for health checks.
|
||||
def __init__(self, client: Any):
|
||||
self.client = client
|
||||
# endregion DatasetHealthChecker.__init__
|
||||
# #endregion DatasetHealthChecker.__init__
|
||||
|
||||
# region DatasetHealthChecker._call_sync [TYPE Function]
|
||||
# #region DatasetHealthChecker._call_sync [TYPE Function]
|
||||
# @PURPOSE Wrap a sync client method call in asyncio.to_thread for async compat.
|
||||
# @PRE method is a callable on self.client.
|
||||
# @POST Returns the result of method(*args, **kwargs) executed in a thread.
|
||||
@@ -1494,9 +1496,9 @@ class DatasetHealthChecker:
|
||||
if asyncio.iscoroutinefunction(method):
|
||||
return await method(*args, **kwargs)
|
||||
return await asyncio.to_thread(method, *args, **kwargs)
|
||||
# endregion DatasetHealthChecker._call_sync
|
||||
# #endregion DatasetHealthChecker._call_sync
|
||||
|
||||
# region DatasetHealthChecker.check_dataset_health [TYPE Function]
|
||||
# #region DatasetHealthChecker.check_dataset_health [TYPE Function]
|
||||
# @PURPOSE Fetch dataset metadata and verify level 1-2 accessibility.
|
||||
# @PRE dataset_id is a valid Superset dataset ID.
|
||||
# @POST Returns dict with level 1-2 health fields.
|
||||
@@ -1538,9 +1540,9 @@ class DatasetHealthChecker:
|
||||
"metadata_accessible": False,
|
||||
"error": str(e),
|
||||
}
|
||||
# endregion DatasetHealthChecker.check_dataset_health
|
||||
# #endregion DatasetHealthChecker.check_dataset_health
|
||||
|
||||
# region DatasetHealthChecker.check_chart_data [TYPE Function]
|
||||
# #region DatasetHealthChecker.check_chart_data [TYPE Function]
|
||||
# @PURPOSE Execute chart data query (level 3-4 per FR-044).
|
||||
# @PRE chart_id is valid, form_data is constructed from chart params.
|
||||
# @POST Returns dict with execution result.
|
||||
@@ -1596,9 +1598,9 @@ class DatasetHealthChecker:
|
||||
"row_count": None,
|
||||
"error": str(e),
|
||||
}
|
||||
# endregion DatasetHealthChecker.check_chart_data
|
||||
# #endregion DatasetHealthChecker.check_chart_data
|
||||
|
||||
# region DatasetHealthChecker.check_dashboard_datasets [TYPE Function]
|
||||
# #region DatasetHealthChecker.check_dashboard_datasets [TYPE Function]
|
||||
# @PURPOSE For every unique dataset in dashboard charts, check health.
|
||||
# @PRE chart_list has chart dicts with slice_id and datasource_id.
|
||||
# @POST Returns dict with datasets and optional chart_data lists.
|
||||
@@ -1664,7 +1666,7 @@ class DatasetHealthChecker:
|
||||
"datasets": dataset_results,
|
||||
"chart_data": chart_data_results,
|
||||
}
|
||||
# endregion DatasetHealthChecker.check_dashboard_datasets
|
||||
# #endregion DatasetHealthChecker.check_dashboard_datasets
|
||||
# #endregion DatasetHealthChecker
|
||||
|
||||
# #region RedactionService [C:2] [TYPE Module]
|
||||
@@ -1688,7 +1690,7 @@ class RedactionService:
|
||||
(r"[\w.+-]+@[\w-]+\.[\w.-]+", "***@***"), # emails
|
||||
]
|
||||
|
||||
# region RedactionService.redact_logs [TYPE Function] [C:2]
|
||||
# #region RedactionService.redact_logs [TYPE Function] [C:2]
|
||||
# @BRIEF Redact PII/credentials from log lines.
|
||||
# @PRE logs is a list of strings.
|
||||
# @POST Returns redacted list preserving structure.
|
||||
@@ -1701,9 +1703,9 @@ class RedactionService:
|
||||
line = re.sub(pattern, replacement, line, flags=re.IGNORECASE)
|
||||
redacted.append(line)
|
||||
return redacted
|
||||
# endregion RedactionService.redact_logs
|
||||
# #endregion RedactionService.redact_logs
|
||||
|
||||
# region RedactionService.redact_raw_response [TYPE Function] [C:2]
|
||||
# #region RedactionService.redact_raw_response [TYPE Function] [C:2]
|
||||
# @BRIEF Redact sensitive data from LLM raw response.
|
||||
# @PRE raw is a string.
|
||||
# @POST Returns redacted string.
|
||||
@@ -1713,7 +1715,7 @@ class RedactionService:
|
||||
for pattern, replacement in RedactionService.PATTERNS:
|
||||
raw = re.sub(pattern, replacement, raw, flags=re.IGNORECASE)
|
||||
return raw
|
||||
# endregion RedactionService.redact_raw_response
|
||||
# #endregion RedactionService.redact_raw_response
|
||||
# #endregion RedactionService
|
||||
|
||||
# #endregion LLMAnalysisService
|
||||
|
||||
@@ -51,8 +51,8 @@ class TranslationExecutionEngine:
|
||||
self._sql_service = SQLInsertService(db, config_manager, event_log)
|
||||
self._aggregator = TranslationResultAggregator(db, event_log)
|
||||
|
||||
# region execute_run [TYPE Function]
|
||||
# @PURPOSE: Execute a translation run: dispatch executor, handle outcomes.
|
||||
# #region TranslationExecutionEngine.execute_run [C:4] [TYPE Function] [SEMANTICS translate,execution,run]
|
||||
# @BRIEF Execute a translation run: dispatch executor, handle outcomes.
|
||||
# @PRE run is in PENDING status.
|
||||
# @POST Run executed, SQL generated, Superset submission attempted.
|
||||
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
|
||||
@@ -94,10 +94,10 @@ class TranslationExecutionEngine:
|
||||
self.db, self.event_log, self._aggregator, self._sql_service,
|
||||
run, job, skip_insert, language_stats_map, self.current_user,
|
||||
)
|
||||
# endregion execute_run
|
||||
# #endregion TranslationExecutionEngine.execute_run
|
||||
|
||||
# region _init_language_stats [TYPE Function]
|
||||
# @PURPOSE: Initialize per-language stats for a run.
|
||||
# #region TranslationExecutionEngine._init_language_stats [C:2] [TYPE Function] [SEMANTICS translate,language,stats]
|
||||
# @BRIEF Initialize per-language stats for a run.
|
||||
def _init_language_stats(
|
||||
self,
|
||||
run: TranslationRun,
|
||||
@@ -117,6 +117,6 @@ class TranslationExecutionEngine:
|
||||
language_stats_map[lang_code] = lang_stat
|
||||
self.db.flush()
|
||||
return language_stats_map
|
||||
# endregion _init_language_stats
|
||||
# #endregion TranslationExecutionEngine._init_language_stats
|
||||
|
||||
# #endregion TranslationExecutionEngine
|
||||
|
||||
Reference in New Issue
Block a user