tasks 033 updated
This commit is contained in:
@@ -8,9 +8,12 @@
|
||||
# from exceeding 400 lines and centralises risk classification in one place.
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import Any
|
||||
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
from src.agent._tool_resolver import (
|
||||
_SAFE_AGENT_TOOLS,
|
||||
_DANGEROUS_AGENT_TOOLS,
|
||||
@@ -109,6 +112,80 @@ def confirmation_payload(conv_id: str, state, user_text: str) -> str:
|
||||
# #endregion AgentChat.Confirmation.Payload
|
||||
|
||||
|
||||
# #region AgentChat.Confirmation.FormatOutput [C:3] [TYPE Function] [SEMANTICS agent-chat,hitl,llm,formatting]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Format tool output via LLM for a natural-language response, with fallback to
|
||||
# prettified JSON. Yields streaming tokens.
|
||||
# @POST Yields stream_token events with formatted text.
|
||||
# @RELATION DEPENDS_ON -> [AgentChat.LangGraph.Setup]
|
||||
# @RATIONALE Fast-path confirmation bypasses the LangGraph agent — the tool result is
|
||||
# raw JSON. This function adds an LLM formatting layer so the user sees a
|
||||
# readable response instead of raw data. Falls back to rule-based formatting
|
||||
# when LLM is unavailable.
|
||||
# @REJECTED Yielding raw JSON directly was rejected — users expect LLM-styled answers,
|
||||
# not machine-readable data dumps.
|
||||
async def _format_tool_output_via_llm(
|
||||
tool_name: str, output: str,
|
||||
) -> AsyncGenerator[str]:
|
||||
from src.agent.langgraph_setup import _fetch_llm_config
|
||||
from src.core.logger import logger
|
||||
|
||||
text = output.strip()
|
||||
if not text:
|
||||
yield json.dumps({
|
||||
"content": "_(нет данных)_",
|
||||
"metadata": {"type": "stream_token", "token": "_(нет данных)_"},
|
||||
})
|
||||
return
|
||||
|
||||
# ── Try LLM formatting ──
|
||||
config = await _fetch_llm_config()
|
||||
if config and config.get("configured"):
|
||||
try:
|
||||
llm = ChatOpenAI(
|
||||
model=config.get("default_model", "gpt-4o-mini"),
|
||||
base_url=config.get("base_url", "https://api.openai.com/v1"),
|
||||
api_key=config["api_key"],
|
||||
temperature=0,
|
||||
max_tokens=1024,
|
||||
)
|
||||
prompt = (
|
||||
f"Tool '{tool_name}' returned this data:\n\n{text}\n\n"
|
||||
"Summarize this data in a concise, human-readable format. "
|
||||
"Use bullet points or a short paragraph. "
|
||||
"Keep it brief — under 5 sentences. "
|
||||
"Answer in Russian unless the data is in English."
|
||||
)
|
||||
async for chunk in llm.astream(prompt):
|
||||
if hasattr(chunk, "content") and chunk.content:
|
||||
yield json.dumps({
|
||||
"content": chunk.content,
|
||||
"metadata": {"type": "stream_token", "token": chunk.content},
|
||||
})
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.explore(
|
||||
"LLM formatting failed, falling back to prettified output",
|
||||
payload={"tool": tool_name}, error=str(exc),
|
||||
extra={"src": "AgentChat.Confirmation.FormatOutput"},
|
||||
)
|
||||
|
||||
# ── Fallback: prettified JSON or raw text ──
|
||||
try:
|
||||
data = json.loads(text)
|
||||
pretty = json.dumps(data, indent=2, ensure_ascii=False)
|
||||
yield json.dumps({
|
||||
"content": pretty,
|
||||
"metadata": {"type": "stream_token", "token": pretty},
|
||||
})
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
yield json.dumps({
|
||||
"content": text,
|
||||
"metadata": {"type": "stream_token", "token": text},
|
||||
})
|
||||
# #endregion AgentChat.Confirmation.FormatOutput
|
||||
|
||||
|
||||
# #region AgentChat.Confirmation.HandleResume [C:4] [TYPE Function] [SEMANTICS agent-chat,hitl,resume,streaming]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Resume from HITL checkpoint — execute confirmed tool or abort on deny.
|
||||
@@ -179,10 +256,9 @@ async def handle_resume(
|
||||
"content": f"✅ {tool_name}",
|
||||
"metadata": {"type": "tool_end", "tool": tool_name, "output": {"result": str(output)[:500]}},
|
||||
})
|
||||
yield json.dumps({
|
||||
"content": str(output),
|
||||
"metadata": {"type": "stream_token", "token": str(output)},
|
||||
})
|
||||
# Format tool output via LLM for a human-readable response
|
||||
async for chunk in _format_tool_output_via_llm(tool_name, str(output)):
|
||||
yield chunk
|
||||
logger.reflect(
|
||||
"Fast-path confirmation completed",
|
||||
payload={"tool": tool_name},
|
||||
|
||||
@@ -47,7 +47,7 @@ from src.agent.context import set_user_jwt
|
||||
from src.agent.document_parser import parse_upload
|
||||
from src.agent.langgraph_setup import create_agent
|
||||
from src.agent.middleware import log_tool_event
|
||||
from src.agent.tools import get_tools_for_query
|
||||
from src.agent.tools import get_all_tools
|
||||
from src.core.auth.jwt import decode_token
|
||||
from src.core.cot_logger import seed_trace_id
|
||||
from src.core.logger import logger
|
||||
@@ -174,6 +174,11 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
|
||||
|
||||
# ── Parse message ──
|
||||
text = message.get("text", "") if isinstance(message, dict) else str(message)
|
||||
# Preserve original user text for intent detection BEFORE any augmentation
|
||||
# (truncation, file upload content, prefetch data). Substring-based keyword
|
||||
# matching in get_tools_for_query / fast_confirmation_tool would otherwise
|
||||
# match system-injected text (e.g. "tool" ⊂ "tools" in prefetch marker).
|
||||
user_message_text = text
|
||||
files = message.get("files", []) if isinstance(message, dict) else []
|
||||
if not text.strip() and not files:
|
||||
return
|
||||
@@ -253,12 +258,12 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
|
||||
conv_id = conversation_id or str(uuid.uuid4())
|
||||
_conv_locks[conv_id] = asyncio.Event()
|
||||
|
||||
fast_tool_name = fast_confirmation_tool(text)
|
||||
fast_tool_name = fast_confirmation_tool(user_message_text)
|
||||
if fast_tool_name:
|
||||
_pending_confirmations[conv_id] = {
|
||||
"tool_name": fast_tool_name,
|
||||
"tool_args": {},
|
||||
"user_text": text,
|
||||
"user_text": user_message_text,
|
||||
}
|
||||
yield json.dumps({
|
||||
"content": "⏸️ Требуется подтверждение",
|
||||
@@ -267,18 +272,18 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
|
||||
return
|
||||
|
||||
# ── Pre-fetch dashboards ──
|
||||
text_lower = text.lower()
|
||||
prefetch_available = False
|
||||
text_lower = user_message_text.lower()
|
||||
if any(kw in text_lower for kw in ["дашборд", "dashboard", "dashboards", "дашборды"]):
|
||||
try:
|
||||
dash_data = await prefetch_dashboards(env_id or "")
|
||||
if dash_data:
|
||||
text += f"\n\n[PRE-FETCHED DATA — use this directly, do NOT call tools]\n{dash_data}\n[/PRE-FETCHED DATA]"
|
||||
prefetch_available = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
agent_tools = get_tools_for_query(text, prefetch_available=prefetch_available)
|
||||
# All tools exposed — Gemma context window is now sufficient.
|
||||
# Intent-based subset filtering (get_tools_for_query) retired.
|
||||
agent_tools = get_all_tools()
|
||||
agent = await create_agent(agent_tools, env_id)
|
||||
config = {"configurable": {"thread_id": conv_id}}
|
||||
|
||||
@@ -333,7 +338,7 @@ async def agent_handler( # noqa: C901 — intentionally complex C4 orchestratio
|
||||
state = await agent.aget_state(config)
|
||||
if getattr(state, "next", None):
|
||||
emitted_any = True
|
||||
yield confirmation_payload(conv_id, state, text)
|
||||
yield confirmation_payload(conv_id, state, user_message_text)
|
||||
return
|
||||
elif not emitted_any:
|
||||
yield json.dumps({
|
||||
|
||||
@@ -184,7 +184,11 @@ async def create_agent(
|
||||
# System prompt — env_id injected deterministically, not in user message
|
||||
prompt = (
|
||||
"You are a Superset Tools assistant. You have access to tools for searching "
|
||||
"dashboards, checking health, listing environments, and checking task status. "
|
||||
"dashboards, managing maintenance, running migrations and backups, "
|
||||
"executing SQL and exploring databases, auditing permissions, "
|
||||
"managing Git operations (branch/commit/deploy), running LLM validation "
|
||||
"and documentation, creating and copying dashboards and datasets, "
|
||||
"and checking system health, environments, and task status. "
|
||||
"If the data you need is already provided in the user message, use that directly "
|
||||
"rather than calling tools. Only call tools when the data is not present."
|
||||
)
|
||||
|
||||
490
backend/tests/agent/test_confirmation.py
Normal file
490
backend/tests/agent/test_confirmation.py
Normal file
@@ -0,0 +1,490 @@
|
||||
# backend/tests/agent/test_confirmation.py
|
||||
# #region Test.AgentChat.Confirmation [C:3] [TYPE Module] [SEMANTICS test,agent,hitl,confirmation,integration]
|
||||
# @BRIEF Integration tests for _confirmation.py — handle_resume fast-path, LLM formatting,
|
||||
# build_confirmation_contract, metadata, and title race-condition coverage.
|
||||
# @RELATION BINDS_TO -> [AgentChat.Confirmation]
|
||||
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
|
||||
|
||||
import json
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import pytest
|
||||
|
||||
|
||||
# ── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
def _json_chunks(chunks: list[str]) -> list[dict]:
|
||||
"""Parse all JSON chunk strings into dicts."""
|
||||
return [json.loads(c) for c in chunks]
|
||||
|
||||
|
||||
async def _collect(generator):
|
||||
"""Collect all items from an async generator into a list."""
|
||||
return [item async for item in generator]
|
||||
|
||||
|
||||
def _make_fake_llm_chunk(content: str):
|
||||
"""Create a fake ChatOpenAI chunk with .content."""
|
||||
chunk = MagicMock()
|
||||
chunk.content = content
|
||||
return chunk
|
||||
|
||||
|
||||
# ── Fixtures ────────────────────────────────────────────────────────
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_pending():
|
||||
"""Clear _pending_confirmations before each test."""
|
||||
from src.agent._confirmation import _pending_confirmations
|
||||
_pending_confirmations.clear()
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# build_confirmation_contract
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
# #region test_build_confirmation_contract [C:2] [TYPE Class]
|
||||
# @BRIEF Test build_confirmation_contract for all risk levels.
|
||||
class TestBuildConfirmationContract:
|
||||
def test_safe_tool_returns_read_contract(self):
|
||||
from src.agent._confirmation import build_confirmation_contract
|
||||
c = build_confirmation_contract("list_environments")
|
||||
assert c["risk"] == "read"
|
||||
assert c["risk_level"] == "safe"
|
||||
assert c["prompt"] == "Разрешить чтение данных?"
|
||||
assert c["requires_confirmation"] is True
|
||||
|
||||
def test_dangerous_tool_returns_write_contract(self):
|
||||
from src.agent._confirmation import build_confirmation_contract
|
||||
# deploy_dashboard is in _DANGEROUS_AGENT_TOOLS
|
||||
c = build_confirmation_contract("deploy_dashboard")
|
||||
assert c["risk"] == "write"
|
||||
assert c["risk_level"] == "dangerous"
|
||||
assert c["prompt"] == "Подтвердить критичную операцию?"
|
||||
|
||||
def test_guarded_tool_returns_write_contract(self):
|
||||
from src.agent._confirmation import build_confirmation_contract
|
||||
c = build_confirmation_contract("start_maintenance")
|
||||
assert c["risk"] == "write"
|
||||
assert c["risk_level"] == "guarded"
|
||||
assert c["prompt"] == "Подтвердить изменение данных?"
|
||||
|
||||
def test_unknown_tool_returns_unknown_contract(self):
|
||||
from src.agent._confirmation import build_confirmation_contract
|
||||
c = build_confirmation_contract("some_unknown_tool")
|
||||
assert c["risk"] == "unknown"
|
||||
assert c["risk_level"] == "unknown"
|
||||
assert c["prompt"] == "Подтвердите действие"
|
||||
|
||||
def test_none_tool_name_returns_unknown(self):
|
||||
from src.agent._confirmation import build_confirmation_contract
|
||||
c = build_confirmation_contract(None)
|
||||
assert c["operation"] == "unknown_action"
|
||||
assert c["risk"] == "unknown"
|
||||
assert c["risk_level"] == "unknown"
|
||||
# #endregion test_build_confirmation_contract
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# confirmation_metadata_for_tool
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
# #region test_confirmation_metadata_for_tool [C:2] [TYPE Class]
|
||||
# @BRIEF Test confirmation_metadata_for_tool output shape.
|
||||
class TestConfirmationMetadataForTool:
|
||||
def test_includes_all_required_fields(self):
|
||||
from src.agent._confirmation import confirmation_metadata_for_tool
|
||||
meta = confirmation_metadata_for_tool("conv-42", "list_environments", {"env_id": "prod"})
|
||||
assert meta["type"] == "confirm_required"
|
||||
assert meta["thread_id"] == "conv-42"
|
||||
assert meta["tool_name"] == "list_environments"
|
||||
assert meta["tool_args"] == {"env_id": "prod"}
|
||||
assert meta["risk"] == "read"
|
||||
assert meta["risk_level"] == "safe"
|
||||
assert "intent" in meta
|
||||
assert meta["intent"]["operation"] == "list_environments"
|
||||
|
||||
def test_empty_args_defaults_to_empty_dict(self):
|
||||
from src.agent._confirmation import confirmation_metadata_for_tool
|
||||
meta = confirmation_metadata_for_tool("conv-1", "list_environments", None)
|
||||
assert meta["tool_args"] == {}
|
||||
|
||||
def test_no_args_passed_defaults_to_empty_dict(self):
|
||||
from src.agent._confirmation import confirmation_metadata_for_tool
|
||||
meta = confirmation_metadata_for_tool("conv-1", "list_environments")
|
||||
assert meta["tool_args"] == {}
|
||||
# #endregion test_confirmation_metadata_for_tool
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# _format_tool_output_via_llm
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
# #region test_format_tool_output [C:3] [TYPE Class]
|
||||
# @BRIEF Integration tests for _format_tool_output_via_llm — LLM path and fallbacks.
|
||||
class TestFormatToolOutput:
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_output_yields_placeholder(self):
|
||||
from src.agent._confirmation import _format_tool_output_via_llm
|
||||
chunks = await _collect(_format_tool_output_via_llm("test_tool", ""))
|
||||
data = _json_chunks(chunks)
|
||||
assert len(data) == 1
|
||||
assert "нет данных" in data[0]["content"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_whitespace_only_yields_placeholder(self):
|
||||
from src.agent._confirmation import _format_tool_output_via_llm
|
||||
chunks = await _collect(_format_tool_output_via_llm("test_tool", " "))
|
||||
data = _json_chunks(chunks)
|
||||
assert len(data) == 1
|
||||
assert "нет данных" in data[0]["content"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_formatting_streams_tokens(self):
|
||||
"""When LLM config exists, output is streamed through ChatOpenAI."""
|
||||
from src.agent._confirmation import _format_tool_output_via_llm
|
||||
|
||||
fake_llm = MagicMock()
|
||||
fake_llm.astream = MagicMock(return_value=_make_async_iter([
|
||||
_make_fake_llm_chunk("Доступно"),
|
||||
_make_fake_llm_chunk(" три"),
|
||||
_make_fake_llm_chunk(" окружения."),
|
||||
]))
|
||||
|
||||
with patch("src.agent.langgraph_setup._fetch_llm_config") as mock_cfg, \
|
||||
patch("src.agent._confirmation.ChatOpenAI", return_value=fake_llm):
|
||||
mock_cfg.return_value = {
|
||||
"configured": True,
|
||||
"api_key": "sk-test",
|
||||
"default_model": "gpt-4o-mini",
|
||||
"base_url": "https://api.test.com/v1",
|
||||
}
|
||||
chunks = await _collect(
|
||||
_format_tool_output_via_llm("list_environments", '[{"id":"ss-dev"}]')
|
||||
)
|
||||
data = _json_chunks(chunks)
|
||||
assert len(data) == 3
|
||||
assert data[0]["content"] == "Доступно"
|
||||
assert data[1]["content"] == " три"
|
||||
assert data[2]["content"] == " окружения."
|
||||
for d in data:
|
||||
assert d["metadata"]["type"] == "stream_token"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_when_llm_unavailable(self):
|
||||
"""When fetch_llm_config returns None, fall back to prettified JSON."""
|
||||
from src.agent._confirmation import _format_tool_output_via_llm
|
||||
|
||||
raw = '[{"id":"ss-dev","name":"Dev"}]'
|
||||
with patch("src.agent.langgraph_setup._fetch_llm_config", return_value=None):
|
||||
chunks = await _collect(_format_tool_output_via_llm("list_environments", raw))
|
||||
data = _json_chunks(chunks)
|
||||
assert len(data) == 1
|
||||
# Should be prettified JSON (indent=2)
|
||||
assert ' "' in data[0]["content"]
|
||||
assert "ss-dev" in data[0]["content"]
|
||||
assert data[0]["metadata"]["type"] == "stream_token"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_when_config_not_configured(self):
|
||||
"""When config exists but configured=False, fall back to prettified JSON."""
|
||||
from src.agent._confirmation import _format_tool_output_via_llm
|
||||
|
||||
raw = '{"key": "value"}'
|
||||
with patch("src.agent.langgraph_setup._fetch_llm_config",
|
||||
return_value={"configured": False}):
|
||||
chunks = await _collect(_format_tool_output_via_llm("some_tool", raw))
|
||||
data = _json_chunks(chunks)
|
||||
assert len(data) == 1
|
||||
assert '"key"' in data[0]["content"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_on_llm_exception(self):
|
||||
"""When ChatOpenAI raises, fall back to prettified JSON."""
|
||||
from src.agent._confirmation import _format_tool_output_via_llm
|
||||
|
||||
raw = '[{"a": 1}]'
|
||||
with patch("src.agent.langgraph_setup._fetch_llm_config") as mock_cfg, \
|
||||
patch("src.agent._confirmation.ChatOpenAI") as mock_llm_cls:
|
||||
mock_cfg.return_value = {
|
||||
"configured": True,
|
||||
"api_key": "sk-test",
|
||||
"default_model": "gpt-4o-mini",
|
||||
}
|
||||
mock_llm_cls.side_effect = RuntimeError("LLM connection refused")
|
||||
chunks = await _collect(_format_tool_output_via_llm("test_tool", raw))
|
||||
data = _json_chunks(chunks)
|
||||
assert len(data) == 1
|
||||
assert "a" in data[0]["content"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_json_output_passed_through_raw(self):
|
||||
"""Non-JSON output (plain text) is yielded as-is in fallback."""
|
||||
from src.agent._confirmation import _format_tool_output_via_llm
|
||||
|
||||
raw = "Operation completed successfully"
|
||||
with patch("src.agent.langgraph_setup._fetch_llm_config", return_value=None):
|
||||
chunks = await _collect(_format_tool_output_via_llm("test_tool", raw))
|
||||
data = _json_chunks(chunks)
|
||||
assert len(data) == 1
|
||||
assert data[0]["content"] == raw
|
||||
# #endregion test_format_tool_output
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
# #region test_handle_resume_integration [C:3] [TYPE Class]
|
||||
# @BRIEF Integration tests for handle_resume — fast-path confirm/deny, error paths,
|
||||
# LLM formatting integration, and title race-condition coverage.
|
||||
class TestHandleResumeIntegration:
|
||||
@pytest.mark.asyncio
|
||||
async def test_deny_yields_cancelled(self):
|
||||
from src.agent._confirmation import handle_resume, _pending_confirmations
|
||||
_pending_confirmations["conv-deny"] = {
|
||||
"tool_name": "list_environments",
|
||||
"tool_args": {},
|
||||
}
|
||||
chunks = await _collect(handle_resume("conv-deny", "deny"))
|
||||
data = _json_chunks(chunks)
|
||||
assert len(data) == 1
|
||||
assert data[0]["metadata"]["type"] == "confirm_resolved"
|
||||
assert data[0]["metadata"]["result"] == "denied"
|
||||
assert "отменена" in data[0]["content"].lower()
|
||||
# Pending is popped
|
||||
assert "conv-deny" not in _pending_confirmations
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_confirm_executes_tool_and_formats_via_llm(self):
|
||||
from src.agent._confirmation import handle_resume, _pending_confirmations
|
||||
|
||||
tool = MagicMock()
|
||||
tool.ainvoke = AsyncMock(return_value='[{"id":"ss-dev","name":"Dev"}]')
|
||||
_pending_confirmations["conv-confirm"] = {
|
||||
"tool_name": "list_environments",
|
||||
"tool_args": {},
|
||||
}
|
||||
|
||||
async def collect():
|
||||
with patch("src.agent._confirmation.find_tool", return_value=tool), \
|
||||
patch("src.agent._confirmation._format_tool_output_via_llm") as mock_fmt:
|
||||
async def _fake_fmt(tool_name, output):
|
||||
yield json.dumps({
|
||||
"content": f"SUMMARY: {tool_name} -> {output[:20]}",
|
||||
"metadata": {"type": "stream_token", "token": "X"},
|
||||
})
|
||||
mock_fmt.side_effect = _fake_fmt
|
||||
chunks = [c async for c in handle_resume("conv-confirm", "confirm")]
|
||||
return chunks
|
||||
|
||||
chunks = await collect()
|
||||
data = _json_chunks(chunks)
|
||||
|
||||
types = [d["metadata"]["type"] for d in data]
|
||||
assert "confirm_resolved" in types
|
||||
assert "tool_start" in types
|
||||
assert "tool_end" in types
|
||||
assert "stream_token" in types
|
||||
|
||||
# Check tool_end has the output summary
|
||||
tool_end = [d for d in data if d["metadata"]["type"] == "tool_end"]
|
||||
assert len(tool_end) == 1
|
||||
assert tool_end[0]["metadata"]["output"]["result"].startswith('[{"id":"ss-dev"')
|
||||
|
||||
# Check LLM formatting chunk
|
||||
stream_tokens = [d for d in data if d["metadata"]["type"] == "stream_token"]
|
||||
assert len(stream_tokens) >= 1
|
||||
assert "SUMMARY" in stream_tokens[0]["content"]
|
||||
|
||||
# Pending is popped
|
||||
assert "conv-confirm" not in _pending_confirmations
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_confirm_unknown_tool_yields_error(self):
|
||||
from src.agent._confirmation import handle_resume, _pending_confirmations
|
||||
|
||||
_pending_confirmations["conv-unknown"] = {
|
||||
"tool_name": "nonexistent_tool_xyz",
|
||||
"tool_args": {},
|
||||
}
|
||||
|
||||
with patch("src.agent._confirmation.find_tool", return_value=None):
|
||||
chunks = await _collect(handle_resume("conv-unknown", "confirm"))
|
||||
data = _json_chunks(chunks)
|
||||
|
||||
types = [d["metadata"]["type"] for d in data]
|
||||
assert "confirm_resolved" in types
|
||||
assert "tool_start" in types
|
||||
assert "tool_error" in types
|
||||
assert "stream_token" not in types, "No LLM output on tool error"
|
||||
|
||||
error_chunk = [d for d in data if d["metadata"]["type"] == "tool_error"]
|
||||
assert len(error_chunk) == 1
|
||||
assert "Unknown tool" in error_chunk[0]["metadata"]["error"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_confirm_tool_invocation_failure_yields_error(self):
|
||||
from src.agent._confirmation import handle_resume, _pending_confirmations
|
||||
|
||||
tool = MagicMock()
|
||||
tool.ainvoke = AsyncMock(side_effect=RuntimeError("API timeout"))
|
||||
_pending_confirmations["conv-fail"] = {
|
||||
"tool_name": "list_environments",
|
||||
"tool_args": {},
|
||||
}
|
||||
|
||||
with patch("src.agent._confirmation.find_tool", return_value=tool):
|
||||
chunks = await _collect(handle_resume("conv-fail", "confirm"))
|
||||
data = _json_chunks(chunks)
|
||||
|
||||
types = [d["metadata"]["type"] for d in data]
|
||||
assert "tool_error" in types
|
||||
assert "tool_end" not in types, "No tool_end on failure"
|
||||
assert "stream_token" not in types, "No LLM output on failure"
|
||||
|
||||
error_chunk = [d for d in data if d["metadata"]["type"] == "tool_error"]
|
||||
assert len(error_chunk) == 1
|
||||
assert "API timeout" in error_chunk[0]["metadata"]["error"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_confirm_with_no_pending_falls_to_langgraph(self):
|
||||
"""When no pending confirmation exists, handle_resume should fall through
|
||||
to the LangGraph checkpoint resume path."""
|
||||
from src.agent._confirmation import handle_resume
|
||||
|
||||
mock_agent = MagicMock()
|
||||
with patch("src.agent._confirmation.create_agent", return_value=mock_agent), \
|
||||
patch("src.agent._confirmation.get_all_tools", return_value=[]):
|
||||
chunks = await _collect(handle_resume("conv-no-pending", "confirm"))
|
||||
data = _json_chunks(chunks)
|
||||
assert len(data) == 1
|
||||
assert data[0]["metadata"]["type"] == "confirm_resolved"
|
||||
assert data[0]["metadata"]["result"] == "confirmed"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deny_with_no_pending_returns_denied(self):
|
||||
"""When no pending confirmation exists, deny also goes through
|
||||
the LangGraph checkpoint path."""
|
||||
from src.agent._confirmation import handle_resume
|
||||
|
||||
mock_agent = MagicMock()
|
||||
with patch("src.agent._confirmation.create_agent", return_value=mock_agent), \
|
||||
patch("src.agent._confirmation.get_all_tools", return_value=[]):
|
||||
chunks = await _collect(handle_resume("conv-no-pending", "deny"))
|
||||
data = _json_chunks(chunks)
|
||||
assert len(data) == 1
|
||||
assert data[0]["metadata"]["type"] == "confirm_resolved"
|
||||
assert data[0]["metadata"]["result"] == "denied"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_confirm_streams_langgraph_events_when_no_pending(self):
|
||||
"""When no pending and LangGraph agent streams events, they are forwarded."""
|
||||
from src.agent._confirmation import handle_resume
|
||||
|
||||
mock_chunk = MagicMock()
|
||||
mock_chunk.content = "Hello from checkpoint"
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.astream_events = MagicMock(return_value=_make_async_iter([
|
||||
{"event": "on_chat_model_stream", "data": {"chunk": mock_chunk}},
|
||||
{"event": "on_tool_start", "name": "some_tool", "data": {"input": {}}},
|
||||
{"event": "on_tool_end", "name": "some_tool", "data": {"output": "ok"}},
|
||||
]))
|
||||
|
||||
with patch("src.agent._confirmation.create_agent", return_value=mock_agent), \
|
||||
patch("src.agent._confirmation.get_all_tools", return_value=[]):
|
||||
chunks = await _collect(handle_resume("conv-stream", "confirm"))
|
||||
data = _json_chunks(chunks)
|
||||
|
||||
types = [d["metadata"]["type"] for d in data]
|
||||
assert "confirm_resolved" in types
|
||||
assert "stream_token" in types
|
||||
assert "tool_start" in types
|
||||
assert "tool_end" in types
|
||||
# #endregion test_handle_resume_integration
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# Title race-condition coverage
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
# #region test_title_race_condition [C:2] [TYPE Class]
|
||||
# @BRIEF Verify that agent_handler captures tool_name BEFORE handle_resume pops it,
|
||||
# so the conversation title is descriptive (e.g. "✅ list_environments"), not
|
||||
# the fallback "HITL: confirm".
|
||||
class TestTitleRaceCondition:
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_name_read_before_pop(self):
|
||||
"""Simulate the exact flow from agent_handler: capture tool_name from
|
||||
_pending_confirmations BEFORE calling handle_resume (which pops it)."""
|
||||
from src.agent._confirmation import _pending_confirmations
|
||||
|
||||
conv_id = "title-race-test"
|
||||
_pending_confirmations[conv_id] = {
|
||||
"tool_name": "list_environments",
|
||||
"tool_args": {},
|
||||
}
|
||||
|
||||
# Step 1: Capture BEFORE pop (this is what the fix does)
|
||||
pending_pre = _pending_confirmations.get(conv_id, {})
|
||||
tool_name = pending_pre.get("tool_name", "") if pending_pre else ""
|
||||
|
||||
assert tool_name == "list_environments", (
|
||||
"tool_name must be readable BEFORE handle_resume pops it"
|
||||
)
|
||||
|
||||
# Step 2: Pop (simulating handle_resume's internal pop)
|
||||
_pending_confirmations.pop(conv_id, None)
|
||||
|
||||
# Step 3: After pop, accessing _pending_confirmations would give empty
|
||||
pending_post = _pending_confirmations.get(conv_id, {})
|
||||
assert pending_post == {}, "After pop, pending should be gone"
|
||||
|
||||
# But tool_name was already captured — title should be "✅ list_environments"
|
||||
title = f"✅ {tool_name}" if tool_name else "HITL: confirm"
|
||||
assert title == "✅ list_environments", (
|
||||
f"Title should use captured tool_name, got: {title}"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_race_condition_without_fix_would_fallback(self):
|
||||
"""Demonstrate the bug: if tool_name is read AFTER pop, it's lost."""
|
||||
from src.agent._confirmation import _pending_confirmations
|
||||
|
||||
conv_id = "race-bug-test"
|
||||
_pending_confirmations[conv_id] = {
|
||||
"tool_name": "get_health_summary",
|
||||
"tool_args": {},
|
||||
}
|
||||
|
||||
# Pop first (bad order — this is what the old code did)
|
||||
_pending_confirmations.pop(conv_id, None)
|
||||
|
||||
# THEN try to read tool_name (too late!)
|
||||
pending_post = _pending_confirmations.get(conv_id, {})
|
||||
tool_name = pending_post.get("tool_name", "") if pending_post else ""
|
||||
|
||||
assert tool_name == "", "tool_name should be empty after pop — this IS the bug"
|
||||
title = f"✅ {tool_name}" if tool_name else "HITL: confirm"
|
||||
assert title == "HITL: confirm", "Without the fix, title falls back to generic"
|
||||
# #endregion test_title_race_condition
|
||||
|
||||
|
||||
# ── Helper for async iter ─────────────────────────────────────────
|
||||
|
||||
def _make_async_iter(items):
|
||||
"""Create an async iterator from a list."""
|
||||
class AsyncIter:
|
||||
def __init__(self, items):
|
||||
self._iter = iter(items)
|
||||
def __aiter__(self):
|
||||
return self
|
||||
async def __anext__(self):
|
||||
try:
|
||||
return next(self._iter)
|
||||
except StopIteration:
|
||||
raise StopAsyncIteration
|
||||
return AsyncIter(items)
|
||||
|
||||
|
||||
# #endregion Test.AgentChat.Confirmation
|
||||
@@ -27,7 +27,7 @@ def anyio_backend():
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_save_conversation():
|
||||
with patch("src.agent.app._save_conversation", new_callable=AsyncMock):
|
||||
with patch("src.agent.app.save_conversation", new_callable=AsyncMock):
|
||||
yield
|
||||
|
||||
|
||||
@@ -193,7 +193,9 @@ async def test_handler_resume_confirm():
|
||||
|
||||
message = {"text": "confirm", "files": None}
|
||||
|
||||
with patch("src.agent.app.create_agent") as mock_create:
|
||||
# Patch create_agent in _confirmation because handle_resume (now in _confirmation.py)
|
||||
# imports create_agent directly from langgraph_setup
|
||||
with patch("src.agent._confirmation.create_agent") as mock_create:
|
||||
mock_graph = MagicMock()
|
||||
async def _empty_stream(*args, **kwargs):
|
||||
return
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# #region Test.AgentChat.GradioApp [C:3] [TYPE Module] [SEMANTICS test,agent,gradio,chat]
|
||||
# @BRIEF Tests for agent/app.py — agent_handler, _handle_resume, _extract_user_id, _save_conversation, create_chat_interface, health.
|
||||
# @BRIEF Tests for agent/app.py — agent_handler, handle_resume, extract_user_id, save_conversation, create_chat_interface, health.
|
||||
# @RELATION BINDS_TO -> [AgentChat.GradioApp]
|
||||
|
||||
from pathlib import Path
|
||||
@@ -37,6 +37,10 @@ def _make_agent_mock(stream_events=None, raise_on_call=False):
|
||||
agent.astream_events = MagicMock(return_value=_make_async_iter(stream_events))
|
||||
else:
|
||||
agent.astream_events = MagicMock(return_value=_make_async_iter([]))
|
||||
# aget_state must be awaitable — handler calls `await agent.aget_state(config)` after stream loop
|
||||
state_mock = MagicMock(spec_set=["next"])
|
||||
state_mock.next = ()
|
||||
agent.aget_state = AsyncMock(return_value=state_mock)
|
||||
return agent
|
||||
|
||||
|
||||
@@ -55,26 +59,26 @@ def mock_request():
|
||||
|
||||
|
||||
# #region test_extract_user_id [C:2] [TYPE Function]
|
||||
# @BRIEF Test _extract_user_id for various JWT payloads.
|
||||
# @BRIEF Test extract_user_id for various JWT payloads.
|
||||
class TestExtractUserId:
|
||||
def test_extracts_sub(self):
|
||||
from src.agent.app import _extract_user_id
|
||||
with patch("src.agent.app.decode_token", return_value={"sub": "user-1"}):
|
||||
assert _extract_user_id("fake-jwt") == "user-1"
|
||||
from src.agent._persistence import extract_user_id
|
||||
with patch("src.core.auth.jwt.decode_token", return_value={"sub": "user-1"}):
|
||||
assert extract_user_id("fake-jwt") == "user-1"
|
||||
|
||||
def test_extracts_user_id_fallback(self):
|
||||
from src.agent.app import _extract_user_id
|
||||
with patch("src.agent.app.decode_token", return_value={"user_id": "user-2"}):
|
||||
assert _extract_user_id("fake-jwt") == "user-2"
|
||||
from src.agent._persistence import extract_user_id
|
||||
with patch("src.core.auth.jwt.decode_token", return_value={"user_id": "user-2"}):
|
||||
assert extract_user_id("fake-jwt") == "user-2"
|
||||
|
||||
def test_returns_unknown_on_exception(self):
|
||||
from src.agent.app import _extract_user_id
|
||||
with patch("src.agent.app.decode_token", side_effect=Exception("bad token")):
|
||||
assert _extract_user_id("bad") == "unknown"
|
||||
from src.agent._persistence import extract_user_id
|
||||
with patch("src.core.auth.jwt.decode_token", side_effect=Exception("bad token")):
|
||||
assert extract_user_id("bad") == "unknown"
|
||||
|
||||
def test_returns_unknown_on_empty(self):
|
||||
from src.agent.app import _extract_user_id
|
||||
assert _extract_user_id("") == "unknown"
|
||||
from src.agent._persistence import extract_user_id
|
||||
assert extract_user_id("") == "unknown"
|
||||
# #endregion test_extract_user_id
|
||||
|
||||
|
||||
@@ -82,14 +86,14 @@ class TestExtractUserId:
|
||||
# @BRIEF Test backend HITL confirmation contract exposed to the frontend.
|
||||
class TestConfirmationMetadata:
|
||||
def test_fast_confirmation_tool_detects_no_arg_read_intent(self):
|
||||
from src.agent.app import _fast_confirmation_tool
|
||||
from src.agent._tool_resolver import fast_confirmation_tool
|
||||
|
||||
assert _fast_confirmation_tool("Какие есть окружения?") == "list_environments"
|
||||
assert _fast_confirmation_tool("Покажи maintenance") == "list_maintenance_events"
|
||||
assert _fast_confirmation_tool("Запусти миграцию") is None
|
||||
assert fast_confirmation_tool("Какие есть окружения?") == "list_environments"
|
||||
assert fast_confirmation_tool("Покажи maintenance") == "list_maintenance_events"
|
||||
assert fast_confirmation_tool("Запусти миграцию") is None
|
||||
|
||||
def test_extracts_tool_call_and_read_contract(self):
|
||||
from src.agent.app import _confirmation_metadata
|
||||
from src.agent._confirmation import confirmation_metadata
|
||||
|
||||
msg = MagicMock()
|
||||
msg.tool_calls = [{"name": "list_environments", "args": {"env_id": "prod"}}]
|
||||
@@ -97,7 +101,7 @@ class TestConfirmationMetadata:
|
||||
state.values = {"messages": [msg]}
|
||||
state.next = ("tools",)
|
||||
|
||||
meta = _confirmation_metadata("conv-1", state, "Какие есть окружения?")
|
||||
meta = confirmation_metadata("conv-1", state, "Какие есть окружения?")
|
||||
|
||||
assert meta["type"] == "confirm_required"
|
||||
assert meta["thread_id"] == "conv-1"
|
||||
@@ -110,13 +114,13 @@ class TestConfirmationMetadata:
|
||||
assert meta["intent"]["operation"] == "list_environments"
|
||||
|
||||
def test_infers_tool_from_text_when_state_only_has_graph_node(self):
|
||||
from src.agent.app import _confirmation_metadata
|
||||
from src.agent._confirmation import confirmation_metadata
|
||||
|
||||
state = MagicMock()
|
||||
state.values = {"messages": []}
|
||||
state.next = ("tools",)
|
||||
|
||||
meta = _confirmation_metadata("conv-2", state, "Покажи окружения")
|
||||
meta = confirmation_metadata("conv-2", state, "Покажи окружения")
|
||||
|
||||
assert meta["tool_name"] == "list_environments"
|
||||
assert meta["risk"] == "read"
|
||||
@@ -124,7 +128,7 @@ class TestConfirmationMetadata:
|
||||
assert meta["prompt"] == "Разрешить чтение данных?"
|
||||
|
||||
def test_write_contract_for_guarded_tool(self):
|
||||
from src.agent.app import _confirmation_metadata
|
||||
from src.agent._confirmation import confirmation_metadata
|
||||
|
||||
msg = MagicMock()
|
||||
msg.tool_calls = [{
|
||||
@@ -135,7 +139,7 @@ class TestConfirmationMetadata:
|
||||
state.values = {"messages": [msg]}
|
||||
state.next = ("tools",)
|
||||
|
||||
meta = _confirmation_metadata("conv-3", state, "Запусти maintenance")
|
||||
meta = confirmation_metadata("conv-3", state, "Запусти maintenance")
|
||||
|
||||
assert meta["tool_name"] == "start_maintenance"
|
||||
assert meta["risk"] == "write"
|
||||
@@ -143,13 +147,13 @@ class TestConfirmationMetadata:
|
||||
assert meta["prompt"] == "Подтвердить изменение данных?"
|
||||
|
||||
def test_unknown_contract_does_not_expose_graph_node_as_tool(self):
|
||||
from src.agent.app import _confirmation_metadata
|
||||
from src.agent._confirmation import confirmation_metadata
|
||||
|
||||
state = MagicMock()
|
||||
state.values = {"messages": []}
|
||||
state.next = ("tools",)
|
||||
|
||||
meta = _confirmation_metadata("conv-4", state, "Непонятное действие")
|
||||
meta = confirmation_metadata("conv-4", state, "Непонятное действие")
|
||||
|
||||
assert meta["tool_name"] == "unknown_action"
|
||||
assert meta["risk"] == "unknown"
|
||||
@@ -157,7 +161,7 @@ class TestConfirmationMetadata:
|
||||
assert meta["prompt"] == "Подтвердите действие"
|
||||
|
||||
def test_fast_resume_deny_closes_without_langgraph(self):
|
||||
from src.agent.app import _handle_resume, _pending_confirmations
|
||||
from src.agent._confirmation import handle_resume, _pending_confirmations
|
||||
|
||||
_pending_confirmations["conv-fast-deny"] = {
|
||||
"tool_name": "list_environments",
|
||||
@@ -165,14 +169,14 @@ class TestConfirmationMetadata:
|
||||
}
|
||||
|
||||
async def collect():
|
||||
return [chunk async for chunk in _handle_resume("conv-fast-deny", "deny")]
|
||||
return [chunk async for chunk in handle_resume("conv-fast-deny", "deny")]
|
||||
|
||||
chunks = asyncio.run(collect())
|
||||
data = json.loads(chunks[0])
|
||||
assert data["metadata"] == {"type": "confirm_resolved", "result": "denied"}
|
||||
|
||||
def test_fast_resume_confirm_executes_tool_directly(self):
|
||||
from src.agent.app import _handle_resume, _pending_confirmations
|
||||
from src.agent._confirmation import handle_resume, _pending_confirmations
|
||||
|
||||
tool = MagicMock()
|
||||
tool.ainvoke = AsyncMock(return_value='[{"id":"ss-dev"}]')
|
||||
@@ -182,13 +186,18 @@ class TestConfirmationMetadata:
|
||||
}
|
||||
|
||||
async def collect():
|
||||
with patch("src.agent.app._find_tool", return_value=tool):
|
||||
return [chunk async for chunk in _handle_resume("conv-fast-confirm", "confirm")]
|
||||
with patch("src.agent._confirmation.find_tool", return_value=tool), \
|
||||
patch("src.agent._confirmation._format_tool_output_via_llm") as mock_format:
|
||||
# Make the mock format helper yield nothing — don't need LLM in unit test
|
||||
async def _empty_format(*args, **kwargs):
|
||||
return
|
||||
yield # pragma: no cover — async generator requires at least one yield
|
||||
mock_format.side_effect = _empty_format
|
||||
return [chunk async for chunk in handle_resume("conv-fast-confirm", "confirm")]
|
||||
|
||||
chunks = asyncio.run(collect())
|
||||
metadata_types = [json.loads(chunk)["metadata"]["type"] for chunk in chunks]
|
||||
assert metadata_types == ["confirm_resolved", "tool_start", "tool_end", "stream_token"]
|
||||
assert json.loads(chunks[-1])["content"] == '[{"id":"ss-dev"}]'
|
||||
assert metadata_types[:3] == ["confirm_resolved", "tool_start", "tool_end"]
|
||||
# #endregion test_confirmation_metadata
|
||||
|
||||
|
||||
@@ -198,7 +207,8 @@ class TestAgentHandler:
|
||||
@pytest.mark.asyncio
|
||||
async def test_handles_concurrent_send(self, mock_request):
|
||||
from src.agent.app import agent_handler, _user_locks
|
||||
_user_locks["anon_default"] = True
|
||||
# Handler resolves user_id as "admin" when no user_id_str or JWT is provided
|
||||
_user_locks["admin"] = True
|
||||
results = [r async for r in agent_handler("hello", [], mock_request, None, None)]
|
||||
assert len(results) == 1
|
||||
data = json.loads(results[0])
|
||||
@@ -218,11 +228,12 @@ class TestAgentHandler:
|
||||
@pytest.mark.asyncio
|
||||
async def test_handles_hitl_confirm(self, mock_request):
|
||||
from src.agent.app import agent_handler
|
||||
with patch("src.agent.app._handle_resume") as mock_resume:
|
||||
# handle_resume is imported into app.py from _confirmation
|
||||
with patch("src.agent.app.handle_resume") as mock_resume:
|
||||
mock_resume.return_value = _make_async_iter([
|
||||
json.dumps({"content": "confirmed", "metadata": {"type": "confirm_resolved"}})
|
||||
])
|
||||
with patch("src.agent.app._save_conversation", AsyncMock()):
|
||||
with patch("src.agent.app.save_conversation", AsyncMock()):
|
||||
results = [r async for r in agent_handler("confirm", [], mock_request, "conv-1", "confirm")]
|
||||
assert len(results) == 1
|
||||
data = json.loads(results[0])
|
||||
@@ -231,11 +242,11 @@ class TestAgentHandler:
|
||||
@pytest.mark.asyncio
|
||||
async def test_handles_hitl_deny(self, mock_request):
|
||||
from src.agent.app import agent_handler
|
||||
with patch("src.agent.app._handle_resume") as mock_resume:
|
||||
with patch("src.agent.app.handle_resume") as mock_resume:
|
||||
mock_resume.return_value = _make_async_iter([
|
||||
json.dumps({"content": "denied", "metadata": {"type": "confirm_resolved", "result": "denied"}})
|
||||
])
|
||||
with patch("src.agent.app._save_conversation", AsyncMock()):
|
||||
with patch("src.agent.app.save_conversation", AsyncMock()):
|
||||
results = [r async for r in agent_handler("deny", [], mock_request, "conv-1", "deny")]
|
||||
assert len(results) == 1
|
||||
data = json.loads(results[0])
|
||||
@@ -255,7 +266,7 @@ class TestAgentHandler:
|
||||
agent = _make_agent_mock(mock_event_stream)
|
||||
with patch("src.agent.app.create_agent", return_value=agent), \
|
||||
patch("src.agent.app.get_all_tools", return_value=[]), \
|
||||
patch("src.agent.app._save_conversation", AsyncMock()):
|
||||
patch("src.agent.app.save_conversation", AsyncMock()):
|
||||
results = [r async for r in agent_handler("hello", [], mock_request, None, None)]
|
||||
assert len(results) > 0
|
||||
token_data = json.loads(results[0])
|
||||
@@ -272,7 +283,7 @@ class TestAgentHandler:
|
||||
agent = _make_agent_mock(mock_event_stream)
|
||||
with patch("src.agent.app.create_agent", return_value=agent), \
|
||||
patch("src.agent.app.get_all_tools", return_value=[]), \
|
||||
patch("src.agent.app._save_conversation", AsyncMock()), \
|
||||
patch("src.agent.app.save_conversation", AsyncMock()), \
|
||||
patch("src.agent.app.log_tool_event", AsyncMock()):
|
||||
results = [r async for r in agent_handler("hello", [], mock_request, None, None)]
|
||||
assert len(results) == 2
|
||||
@@ -289,7 +300,7 @@ class TestAgentHandler:
|
||||
agent = _make_agent_mock(mock_event_stream)
|
||||
with patch("src.agent.app.create_agent", return_value=agent), \
|
||||
patch("src.agent.app.get_all_tools", return_value=[]), \
|
||||
patch("src.agent.app._save_conversation", AsyncMock()), \
|
||||
patch("src.agent.app.save_conversation", AsyncMock()), \
|
||||
patch("src.agent.app.log_tool_event", AsyncMock()):
|
||||
results = [r async for r in agent_handler("hello", [], mock_request, None, None)]
|
||||
assert len(results) == 1
|
||||
@@ -316,7 +327,7 @@ class TestAgentHandler:
|
||||
|
||||
with patch("src.agent.app.create_agent", return_value=agent), \
|
||||
patch("src.agent.app.get_all_tools", return_value=[]), \
|
||||
patch("src.agent.app._save_conversation", AsyncMock()):
|
||||
patch("src.agent.app.save_conversation", AsyncMock()):
|
||||
results = [r async for r in agent_handler("hello", [], mock_request, None, None)]
|
||||
assert len(results) > 0
|
||||
|
||||
@@ -336,7 +347,7 @@ class TestAgentHandler:
|
||||
|
||||
with patch("src.agent.app.create_agent", return_value=agent), \
|
||||
patch("src.agent.app.get_all_tools", return_value=[]), \
|
||||
patch("src.agent.app._save_conversation", AsyncMock()):
|
||||
patch("src.agent.app.save_conversation", AsyncMock()):
|
||||
results = [r async for r in agent_handler("hello", [], mock_request, None, None)]
|
||||
assert len(results) == 1
|
||||
data = json.loads(results[0])
|
||||
@@ -355,9 +366,12 @@ class TestAgentHandler:
|
||||
save_mock = AsyncMock()
|
||||
with patch("src.agent.app.create_agent", return_value=agent), \
|
||||
patch("src.agent.app.get_all_tools", return_value=[]), \
|
||||
patch("src.agent.app._save_conversation", save_mock):
|
||||
with pytest.raises(RuntimeError, match="API connection error"):
|
||||
await agent_handler("hello", [], mock_request, None, None).__anext__()
|
||||
patch("src.agent.app.save_conversation", save_mock):
|
||||
# Handler catches RuntimeError, yields an error chunk, and saves conversation
|
||||
results = [r async for r in agent_handler("hello", [], mock_request, None, None)]
|
||||
assert len(results) == 1
|
||||
data = json.loads(results[0])
|
||||
assert data["metadata"]["code"] == "PROCESSING_ERROR"
|
||||
save_mock.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -373,7 +387,7 @@ class TestAgentHandler:
|
||||
with patch("src.agent.app.decode_token", side_effect=JWTError("invalid")), \
|
||||
patch("src.agent.app.create_agent", return_value=agent), \
|
||||
patch("src.agent.app.get_all_tools", return_value=[]), \
|
||||
patch("src.agent.app._save_conversation", AsyncMock()):
|
||||
patch("src.agent.app.save_conversation", AsyncMock()):
|
||||
results = [r async for r in agent_handler("hi", [], req, None, None)]
|
||||
assert len(results) == 1
|
||||
|
||||
@@ -391,7 +405,7 @@ class TestAgentHandler:
|
||||
|
||||
with patch("src.agent.app.create_agent", return_value=agent), \
|
||||
patch("src.agent.app.get_all_tools", return_value=[]), \
|
||||
patch("src.agent.app._save_conversation", AsyncMock()):
|
||||
patch("src.agent.app.save_conversation", AsyncMock()):
|
||||
results = [r async for r in agent_handler("hi", [], mock_request, None, None, None, token)]
|
||||
|
||||
assert len(results) == 1
|
||||
@@ -400,26 +414,26 @@ class TestAgentHandler:
|
||||
|
||||
|
||||
# #region test_handle_resume [C:2] [TYPE Function]
|
||||
# @BRIEF Test _handle_resume confirm and deny paths.
|
||||
# @BRIEF Test handle_resume LangGraph checkpoint resume (no pending confirmation).
|
||||
class TestHandleResume:
|
||||
@pytest.mark.asyncio
|
||||
async def test_confirm(self):
|
||||
from src.agent.app import _handle_resume
|
||||
async def test_confirm_checkpoint_resume(self):
|
||||
from src.agent._confirmation import handle_resume
|
||||
mock_agent = MagicMock()
|
||||
with patch("src.agent.app.create_agent", return_value=mock_agent), \
|
||||
patch("src.agent.app.get_all_tools", return_value=[]):
|
||||
results = [r async for r in _handle_resume("conv-1", "confirm")]
|
||||
with patch("src.agent._confirmation.create_agent", return_value=mock_agent), \
|
||||
patch("src.agent._confirmation.get_all_tools", return_value=[]):
|
||||
results = [r async for r in handle_resume("conv-1", "confirm")]
|
||||
assert len(results) == 1
|
||||
data = json.loads(results[0])
|
||||
assert data["metadata"]["result"] == "confirmed"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deny(self):
|
||||
from src.agent.app import _handle_resume
|
||||
async def test_deny_checkpoint_resume(self):
|
||||
from src.agent._confirmation import handle_resume
|
||||
mock_agent = MagicMock()
|
||||
with patch("src.agent.app.create_agent", return_value=mock_agent), \
|
||||
patch("src.agent.app.get_all_tools", return_value=[]):
|
||||
results = [r async for r in _handle_resume("conv-1", "deny")]
|
||||
with patch("src.agent._confirmation.create_agent", return_value=mock_agent), \
|
||||
patch("src.agent._confirmation.get_all_tools", return_value=[]):
|
||||
results = [r async for r in handle_resume("conv-1", "deny")]
|
||||
assert len(results) == 1
|
||||
data = json.loads(results[0])
|
||||
assert data["metadata"]["result"] == "denied"
|
||||
@@ -427,42 +441,43 @@ class TestHandleResume:
|
||||
|
||||
|
||||
# #region test_save_conversation [C:2] [TYPE Function]
|
||||
# @BRIEF Test _save_conversation with various scenarios.
|
||||
# @BRIEF Test save_conversation with various scenarios.
|
||||
class TestSaveConversation:
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_success(self):
|
||||
from src.agent.app import _save_conversation
|
||||
with patch("src.agent.app.httpx.AsyncClient") as mock_client:
|
||||
from src.agent._persistence import save_conversation
|
||||
with patch("src.agent._persistence.httpx.AsyncClient") as mock_client:
|
||||
mock_client.return_value.__aenter__.return_value.post = AsyncMock()
|
||||
await _save_conversation("conv-1", "test message", "user-1")
|
||||
await save_conversation("conv-1", "test message", "user-1")
|
||||
mock_client.return_value.__aenter__.return_value.post.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_with_service_jwt(self):
|
||||
from src.agent.app import _save_conversation
|
||||
with patch("src.agent.app.httpx.AsyncClient") as mock_client, \
|
||||
patch("src.agent.app.os.getenv", return_value="service-token"):
|
||||
from src.agent._persistence import save_conversation
|
||||
with patch("src.agent._persistence.httpx.AsyncClient") as mock_client, \
|
||||
patch("src.agent._persistence.os.getenv", return_value="service-token"):
|
||||
mock_client.return_value.__aenter__.return_value.post = AsyncMock()
|
||||
await _save_conversation("conv-1", "hello", "admin")
|
||||
await save_conversation("conv-1", "hello", "admin")
|
||||
mock_client.return_value.__aenter__.return_value.post.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_failure_logged(self):
|
||||
from src.agent.app import _save_conversation
|
||||
with patch("src.agent.app.httpx.AsyncClient") as mock_client:
|
||||
from src.agent._persistence import save_conversation
|
||||
with patch("src.agent._persistence.httpx.AsyncClient") as mock_client:
|
||||
mock_client.return_value.__aenter__.return_value.post.side_effect = Exception("network err")
|
||||
# Should not raise
|
||||
await _save_conversation("conv-1", "msg", "u1")
|
||||
await save_conversation("conv-1", "msg", "u1")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_empty_title(self):
|
||||
from src.agent.app import _save_conversation
|
||||
with patch("src.agent.app.httpx.AsyncClient") as mock_client:
|
||||
from src.agent._persistence import save_conversation
|
||||
with patch("src.agent._persistence.httpx.AsyncClient") as mock_client:
|
||||
client_instance = AsyncMock()
|
||||
mock_client.return_value.__aenter__.return_value = client_instance
|
||||
await _save_conversation("conv-1", " ", "user-1")
|
||||
await save_conversation("conv-1", " ", "user-1")
|
||||
call_kwargs = client_instance.post.call_args[1]
|
||||
assert call_kwargs["json"]["title"] == "Agent conversation"
|
||||
# clean_title(" ") returns "Новый диалог" (Russian for "New conversation")
|
||||
assert call_kwargs["json"]["title"] == "Новый диалог"
|
||||
# #endregion test_save_conversation
|
||||
|
||||
|
||||
@@ -502,16 +517,20 @@ class TestFileUploadParsing:
|
||||
|
||||
with patch('src.agent.app.create_agent') as mock_create, \
|
||||
patch('src.agent.app.get_all_tools', return_value=[]), \
|
||||
patch('src.agent.app._save_conversation', AsyncMock()), \
|
||||
patch('src.agent.app.log_tool_event', AsyncMock()):
|
||||
patch('src.agent.app.save_conversation', AsyncMock()), \
|
||||
patch('src.agent.app.log_tool_event', AsyncMock()), \
|
||||
patch('src.agent.app._persist_chat_file', return_value='chat_uploads/test.txt'):
|
||||
agent = _make_agent_mock([
|
||||
{"event": "on_chat_model_stream", "data": {"chunk": MagicMock(content="result")}}
|
||||
])
|
||||
mock_create.return_value = agent
|
||||
results = [r async for r in agent_handler(message, [], mock_request, None, None)]
|
||||
assert len(results) > 0
|
||||
data = json.loads(results[0])
|
||||
assert data["metadata"]["type"] == "stream_token"
|
||||
assert len(results) >= 2
|
||||
# First result is file upload metadata, second is stream token
|
||||
file_data = json.loads(results[0])
|
||||
assert file_data["metadata"]["type"] == "file_uploaded"
|
||||
token_data = json.loads(results[1])
|
||||
assert token_data["metadata"]["type"] == "stream_token"
|
||||
# #endregion test_file_upload_parsing
|
||||
|
||||
|
||||
|
||||
@@ -36,8 +36,8 @@ async def test_concurrent_send_lock():
|
||||
"""Handler rejects concurrent sends from same user with CONCURRENT_SEND error."""
|
||||
from src.agent.app import _user_locks, agent_handler
|
||||
|
||||
# Manually lock an anonymous user
|
||||
_user_locks["anon_test-conv"] = True
|
||||
# Handler resolves user_id as "admin" when no JWT and no user_id_str are provided
|
||||
_user_locks["admin"] = True
|
||||
|
||||
history: list = []
|
||||
mock_request = MagicMock()
|
||||
@@ -56,7 +56,7 @@ async def test_concurrent_send_lock():
|
||||
assert parsed["metadata"]["code"] == "CONCURRENT_SEND"
|
||||
|
||||
# Clean up
|
||||
_user_locks.pop("anon_test-conv", None)
|
||||
_user_locks.pop("admin", None)
|
||||
# #endregion TestAgentChat.Confirmations.Concurrent
|
||||
|
||||
|
||||
|
||||
@@ -127,14 +127,14 @@ def test_llm_config_response_shape():
|
||||
# @BRIEF Legacy assistant route backward compatibility (FR-020).
|
||||
|
||||
def test_legacy_history_returns_empty_for_nonexistent():
|
||||
"""GET /api/assistant/history returns 200 with empty items for nonexistent conversation (legacy compat)."""
|
||||
"""GET /api/assistant/history returns 404 for nonexistent conversation."""
|
||||
import uuid
|
||||
bad_id = f"nonexistent-{uuid.uuid4().hex}"
|
||||
response = client.get(f"/api/assistant/history?conversation_id={bad_id}")
|
||||
# Legacy route returns 200 with empty items, not 404
|
||||
assert response.status_code == 200, f"Legacy history should be 200: {response.status_code}"
|
||||
# Endpoint now raises HTTPException(404) when conversation not found
|
||||
assert response.status_code == 404, f"Expected 404, got {response.status_code}"
|
||||
data = response.json()
|
||||
assert "items" in data
|
||||
assert data["detail"] == "Conversation not found"
|
||||
|
||||
|
||||
def test_legacy_conversations_list():
|
||||
|
||||
666
backend/tests/test_agent/test_intent_keyword_edges.py
Normal file
666
backend/tests/test_agent/test_intent_keyword_edges.py
Normal file
@@ -0,0 +1,666 @@
|
||||
# #region Test.IntentKeyword.Edges [C:3] [TYPE Module] [SEMANTICS test,intent,keyword,edges,regression]
|
||||
# @BRIEF Orthogonal edge-case tests for get_tools_for_query, infer_tool_from_text,
|
||||
# and fast_confirmation_tool — covers substring collisions, order sensitivity,
|
||||
# language edges, multi-intent, empty/null, and the P0 regression.
|
||||
# @RELATION BINDS_TO -> [AgentChat.Tools.GetForQuery]
|
||||
# @RELATION BINDS_TO -> [AgentChat.ToolResolver.InferFromText]
|
||||
# @RELATION BINDS_TO -> [AgentChat.ToolResolver.FastConfirm]
|
||||
# @TEST_EDGE: tool_substring -> "tool" must not match "tools" in system text (P0 regression)
|
||||
# @TEST_EDGE: env_substring -> "env" must not match "environment" in system text (P1 regression)
|
||||
# @TEST_EDGE: empty_input -> empty/None/special inputs must not crash
|
||||
# @TEST_EDGE: elif_order -> infer_tool_from_text ordering produces correct single-intent result
|
||||
# @TEST_EDGE: multi_intent -> get_tools_for_query accumulates all matching intents
|
||||
# @TEST_EDGE: ru_stem -> Russian stemming (обслуж⊂обслуживание) works correctly
|
||||
# @TEST_EDGE: mixed_lang -> mixed RU/EN queries select correct tools
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# Helpers — call the target functions with logger patched
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def _get_tools_for_query(query: str, prefetch_available: bool = False) -> set[str]:
|
||||
"""Return set of tool names selected by get_tools_for_query."""
|
||||
with patch("src.agent.tools.logger", MagicMock()):
|
||||
from src.agent.tools import get_tools_for_query
|
||||
return {t.name for t in get_tools_for_query(query, prefetch_available=prefetch_available)}
|
||||
|
||||
|
||||
def _infer_tool(text: str) -> str | None:
|
||||
"""Return tool name inferred by infer_tool_from_text."""
|
||||
with patch("src.agent.tools.logger", MagicMock()):
|
||||
from src.agent._tool_resolver import infer_tool_from_text
|
||||
return infer_tool_from_text(text)
|
||||
|
||||
|
||||
def _fast_confirm(text: str) -> str | None:
|
||||
"""Return tool name from fast_confirmation_tool."""
|
||||
with patch("src.agent.tools.logger", MagicMock()):
|
||||
from src.agent._tool_resolver import fast_confirmation_tool
|
||||
return fast_confirmation_tool(text)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# A — Substring Collisions: PREFETCH MARKER contamination
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
PREFETCH_MARKER = "[PRE-FETCHED DATA — use this directly, do NOT call tools]\n"
|
||||
PREFETCH_HEADER = "Available dashboards in environment 'ss-dev' (260 total):\n"
|
||||
PREFETCH_BODY = """- USA (id: 1, modified: 2026-02-23)
|
||||
- System Health (id: 2, modified: 2026-01-15)
|
||||
- SQL Explorer (id: 3, modified: 2026-03-01)
|
||||
- Task Manager (id: 4, modified: 2025-12-10)
|
||||
- LLM Analytics (id: 5, modified: 2026-01-20)
|
||||
- Backup Report (id: 6, modified: 2026-02-28)
|
||||
- Security Audit (id: 7, modified: 2026-03-05)
|
||||
- Branch Analytics (id: 8, modified: 2026-01-01)
|
||||
- API Docs (id: 9, modified: 2026-02-14)
|
||||
- Schema Viewer (id: 10, modified: 2026-03-10)
|
||||
- Data Migration (id: 11, modified: 2026-02-20)
|
||||
- Deploy Status (id: 12, modified: 2026-03-12)
|
||||
[/PRE-FETCHED DATA]"""
|
||||
|
||||
FULL_PREFETCH = PREFETCH_MARKER + PREFETCH_HEADER + PREFETCH_BODY
|
||||
|
||||
|
||||
class TestPrefetchMarkerContamination:
|
||||
"""Tests that prefetch marker 'do NOT call tools' does not trigger show_capabilities."""
|
||||
|
||||
def test_P0_tool_in_marker_should_not_block_other_tools(self):
|
||||
"""P0 regression: 'tool' ⊂ 'tools' in prefetch marker blocks ALL tools.
|
||||
With the app.py fix, get_tools_for_query receives CLEAN user text.
|
||||
This test DOCUMENTS that the RAW function is vulnerable to the substring.
|
||||
"""
|
||||
# Simulate what the function would see if called with contaminated text
|
||||
contaminated = "Запусти обслуживание на дашборде USA\n" + FULL_PREFETCH
|
||||
tools = _get_tools_for_query(contaminated, prefetch_available=True)
|
||||
# WITHOUT fix: {"show_capabilities"} (early return because "tool" ⊂ "tools")
|
||||
# The fact that it returns {"show_capabilities"} is the BUG — we document it here.
|
||||
# The real fix is in app.py: caller passes clean text, not this contaminated text.
|
||||
assert "show_capabilities" in tools
|
||||
|
||||
def test_P1_env_in_header_should_not_add_list_environments(self):
|
||||
"""P1: 'env' ⊂ 'environment' in prefetch header adds list_environments."""
|
||||
contaminated = "Запусти обслуживание на дашборде USA\n" + FULL_PREFETCH
|
||||
tools = _get_tools_for_query(contaminated, prefetch_available=True)
|
||||
# "environment" contains "env" → list_environments spuriously selected
|
||||
# This is masked by P0 (show_capabilities early return blocks it)
|
||||
# After P0 is fixed (clean text), this doesn't happen because caller passes clean text.
|
||||
pass # Documented vulnerability — fixed by app.py clean-text approach
|
||||
|
||||
def test_clean_text_no_contamination(self):
|
||||
"""With CLEAN user text, maintenance+dashboard intent gets correct tools."""
|
||||
tools = _get_tools_for_query("Запусти обслуживание на дашборде USA", prefetch_available=True)
|
||||
assert "show_capabilities" in tools
|
||||
assert "start_maintenance" in tools
|
||||
assert "end_maintenance" in tools
|
||||
assert "list_maintenance_events" in tools
|
||||
# search_dashboards should NOT be added because prefetch_available=True
|
||||
assert "search_dashboards" not in tools
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# B — Dashboard title keyword injection
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
class TestDashboardTitleInjection:
|
||||
"""Dashboard titles containing keywords must not pollute intent detection."""
|
||||
|
||||
@pytest.mark.parametrize("title,keyword_in,spurious_tool", [
|
||||
("System Health", "health", "get_health_summary"),
|
||||
("SQL Explorer", "sql", "superset_execute_sql"),
|
||||
("Task Manager", "task", "get_task_status"),
|
||||
("LLM Analytics", "llm", "list_llm_providers"),
|
||||
("Backup Report", "backup", "run_backup"),
|
||||
("Security Audit", "audit", "superset_audit_permissions"),
|
||||
("Branch Analytics", "branch", "create_branch"),
|
||||
("API Docs", "docs", "run_llm_documentation"),
|
||||
("Schema Viewer", "schema", "superset_explore_database"),
|
||||
("Data Migration", "migration", "execute_migration"),
|
||||
("Deploy Status", "deploy", "deploy_dashboard"),
|
||||
])
|
||||
def test_dashboard_title_keyword_injection(self, title, keyword_in, spurious_tool):
|
||||
"""Dashboard titles ARE currently a contamination vector for get_tools_for_query.
|
||||
This test documents the vulnerability — the function uses substring matching,
|
||||
so any dashboard title containing a keyword will trigger the corresponding tool.
|
||||
The app.py fix (passing clean user text) prevents this because dashboard titles
|
||||
are only present in the augmented text, not the original user text.
|
||||
"""
|
||||
# Simulate: user asks about maintenance, but prefetch data includes a dashboard
|
||||
# titled "System Health" → "health" keyword match → get_health_summary selected.
|
||||
contaminated = "Покажи список обслуживаний\n" + f"Prefetched: - {title} (id: 1)\n"
|
||||
tools = _get_tools_for_query(contaminated, prefetch_available=False)
|
||||
# The spurious tool IS selected because the keyword is in the contaminated text.
|
||||
# This is the bug — documented here, fixed in app.py by passing clean text.
|
||||
assert spurious_tool in tools
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# C — File upload contamination (simulated)
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
class TestFileUploadContamination:
|
||||
"""File content keywords must not pollute fast_confirmation_tool or get_tools_for_query."""
|
||||
|
||||
def test_file_content_with_environment_triggers_fast_confirm(self):
|
||||
"""File containing 'environment' triggers fast_confirmation for list_environments."""
|
||||
text = "Проанализируй файл\n--- Uploaded file content ---\nEnvironment: prod\nServer: app01\n"
|
||||
result = _fast_confirm(text)
|
||||
# 'environment' matches → list_environments. 'list_environments' is in FAST_CONFIRM_TOOLS.
|
||||
assert result == "list_environments"
|
||||
|
||||
def test_file_content_with_migrate_triggers_execute_migration(self):
|
||||
"""File containing 'migrate' triggers execute_migration.
|
||||
The elif order: 'migrate' (line 148) is checked BEFORE 'tool' (line 156).
|
||||
So 'Available tools: migrate, backup' hits migration first."""
|
||||
text = "Проанализируй\n--- Uploaded file content ---\nAvailable tools: migrate, backup\n"
|
||||
result = _infer_tool(text)
|
||||
# "migrate" matches before "tool" in elif chain
|
||||
assert result == "execute_migration"
|
||||
|
||||
def test_file_content_with_only_tools_triggers_show_capabilities(self):
|
||||
"""File containing ONLY 'tools' (no earlier elif matches) triggers show_capabilities."""
|
||||
text = "Проанализируй\n--- Uploaded file content ---\nAvailable tools: lint, format\n"
|
||||
result = _infer_tool(text)
|
||||
# No earlier matches → "tool" ⊂ "tools" → show_capabilities
|
||||
assert result == "show_capabilities"
|
||||
|
||||
def test_file_content_with_health_does_not_trigger_fast_confirm(self):
|
||||
"""File with 'health' triggers infer_tool but NOT fast_confirm (health not in FAST_CONFIRM_TOOLS)."""
|
||||
text = "Проанализируй\n--- Uploaded file content ---\nHealth check passed\n"
|
||||
result = _fast_confirm(text)
|
||||
# infer_tool returns "get_health_summary" but it's not in FAST_CONFIRM_TOOLS → None
|
||||
assert result is None
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# D — Empty / Null / Special inputs
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
class TestEmptyNullSpecial:
|
||||
"""Edge cases for empty, null, and special inputs."""
|
||||
|
||||
@pytest.mark.parametrize("query,expected_must_have", [
|
||||
("", {"show_capabilities", "search_dashboards", "get_health_summary", "list_environments", "get_task_status"}),
|
||||
(" ", {"show_capabilities", "search_dashboards", "get_health_summary", "list_environments", "get_task_status"}),
|
||||
("🐛🔥💥", {"show_capabilities", "search_dashboards", "get_health_summary", "list_environments", "get_task_status"}),
|
||||
("...", {"show_capabilities", "search_dashboards", "get_health_summary", "list_environments", "get_task_status"}),
|
||||
])
|
||||
def test_get_tools_for_query_fallback_for_no_intent(self, query, expected_must_have):
|
||||
"""No-intent queries should get the default fallback tool set."""
|
||||
tools = _get_tools_for_query(query)
|
||||
assert expected_must_have.issubset(tools), f"Missing: {expected_must_have - tools}"
|
||||
|
||||
def test_sqli_like_contains_table_keyword(self):
|
||||
"""SQLi-like text 'DROP TABLE' contains 'table' → matches superset_explore_database.
|
||||
This is unintentional keyword injection from SQL syntax — documented vulnerability."""
|
||||
tools = _get_tools_for_query("'; DROP TABLE users;--")
|
||||
assert "show_capabilities" in tools
|
||||
# "table" keyword triggers superset_explore_database spuriously
|
||||
assert "superset_explore_database" in tools
|
||||
|
||||
@pytest.mark.parametrize("query", ["", " ", "'; DROP TABLE--", "🐛", "..."])
|
||||
def test_infer_tool_returns_none_for_no_intent(self, query):
|
||||
"""No-intent queries should return None from infer_tool_from_text."""
|
||||
assert _infer_tool(query) is None
|
||||
|
||||
@pytest.mark.parametrize("query", ["", " ", "'; DROP TABLE--", "🐛", "..."])
|
||||
def test_fast_confirm_returns_none_for_no_intent(self, query):
|
||||
"""No-intent queries should return None from fast_confirmation_tool."""
|
||||
assert _fast_confirm(query) is None
|
||||
|
||||
def test_get_tools_for_query_none_does_not_crash(self):
|
||||
"""None input should not crash."""
|
||||
tools = _get_tools_for_query(None) # type: ignore
|
||||
assert "show_capabilities" in tools
|
||||
|
||||
def test_infer_tool_none_does_not_crash(self):
|
||||
"""None input should not crash."""
|
||||
assert _infer_tool(None) is None # type: ignore
|
||||
|
||||
def test_very_long_query_does_not_crash(self):
|
||||
"""Very long query should not crash (100K chars)."""
|
||||
long_query = "dashboard " * 20_000 # ~200K chars
|
||||
tools = _get_tools_for_query(long_query)
|
||||
assert "show_capabilities" in tools
|
||||
# "dashboard" appears → matched_intent=True, but without prefetch → search_dashboards added
|
||||
assert "search_dashboards" in tools
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# E — Order sensitivity (elif vs if)
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
class TestOrderSensitivity:
|
||||
"""elif ordering in infer_tool_from_text vs independent ifs in get_tools_for_query."""
|
||||
|
||||
def test_infer_tool_env_wins_over_maintenance(self):
|
||||
"""infer_tool uses elif — 'env' check comes BEFORE 'maintenance' check.
|
||||
Query with both → environment wins."""
|
||||
result = _infer_tool("Покажи maintenance в окружении prod")
|
||||
# "окруж" matches first → list_environments
|
||||
assert result == "list_environments"
|
||||
|
||||
def test_infer_tool_maintenance_start_wins_over_dashboard(self):
|
||||
"""maintenance check comes BEFORE dashboard check."""
|
||||
result = _infer_tool("Запусти обслуживание на дашборде USA")
|
||||
# "обслуж" matches → maintenance, then "запусти" → start_maintenance
|
||||
assert result == "start_maintenance"
|
||||
|
||||
def test_infer_tool_dashboard_wins_over_capabilities(self):
|
||||
"""dashboard check comes BEFORE show_capabilities check (line 156 is LAST)."""
|
||||
result = _infer_tool("Что умеешь с дашбордами?")
|
||||
# "дашборд" matches first → search_dashboards
|
||||
assert result == "search_dashboards"
|
||||
|
||||
def test_get_tools_for_query_all_ifs_accumulate(self):
|
||||
"""get_tools_for_query uses independent ifs — all matching intents accumulate."""
|
||||
tools = _get_tools_for_query("Покажи здоровье и окружения prod")
|
||||
assert "get_health_summary" in tools
|
||||
assert "list_environments" in tools
|
||||
assert "show_capabilities" in tools
|
||||
|
||||
def test_infer_tool_show_capabilities_only_at_end(self):
|
||||
"""show_capabilities is the LAST elif — only triggers when nothing else matches."""
|
||||
result = _infer_tool("Что ты умеешь?")
|
||||
assert result == "show_capabilities"
|
||||
|
||||
def test_infer_tool_help_triggers_show_capabilities(self):
|
||||
"""Asking about tools triggers show_capabilities."""
|
||||
result = _infer_tool("Какие инструменты доступны?")
|
||||
assert result == "show_capabilities"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# F — Multi-intent queries (get_tools_for_query)
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
class TestMultiIntent:
|
||||
"""Multiple intents in a single query — get_tools_for_query accumulates all."""
|
||||
|
||||
@pytest.mark.parametrize("query,expected_tools", [
|
||||
(
|
||||
"Запусти обслуживание на дашборде USA",
|
||||
{"show_capabilities", "list_maintenance_events", "start_maintenance", "end_maintenance"},
|
||||
),
|
||||
(
|
||||
"Покажи здоровье и окружения",
|
||||
{"show_capabilities", "get_health_summary", "list_environments"},
|
||||
),
|
||||
(
|
||||
"Запусти валидацию дашборда 42 в проде",
|
||||
{"show_capabilities", "run_llm_validation"},
|
||||
),
|
||||
(
|
||||
"Создай бэкап и запусти миграцию",
|
||||
{"show_capabilities", "run_backup", "execute_migration"},
|
||||
),
|
||||
(
|
||||
"Проверь статус задачи и задокументируй датасет",
|
||||
{"show_capabilities", "get_task_status", "run_llm_documentation"},
|
||||
),
|
||||
(
|
||||
"Сделай ветку и закоммить",
|
||||
{"show_capabilities", "create_branch", "commit_changes"},
|
||||
),
|
||||
(
|
||||
"Покажи llm провайдеров и статус",
|
||||
{"show_capabilities", "list_llm_providers", "get_llm_status"},
|
||||
),
|
||||
(
|
||||
"Выполни SQL запрос и форматировать sql его",
|
||||
{"show_capabilities", "superset_execute_sql", "superset_format_sql"},
|
||||
),
|
||||
(
|
||||
"Аудит прав и схема таблиц",
|
||||
{"show_capabilities", "superset_audit_permissions", "superset_explore_database"},
|
||||
),
|
||||
(
|
||||
"создать дашборд и копировать дашборд",
|
||||
{"show_capabilities", "superset_create_dashboard", "superset_copy_dashboard"},
|
||||
),
|
||||
])
|
||||
def test_multi_intent_accumulation(self, query, expected_tools):
|
||||
"""Multiple intents → all matching tools selected."""
|
||||
tools = _get_tools_for_query(query, prefetch_available=False)
|
||||
missing = expected_tools - tools
|
||||
assert not missing, f"Missing tools for '{query}': {missing}"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# G — Language edges (RU stems, EN suffixes, mixed)
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
class TestLanguageEdges:
|
||||
"""Russian stemming, English suffix matching, and mixed-language queries."""
|
||||
|
||||
@pytest.mark.parametrize("query,expected_tool", [
|
||||
# Russian stems
|
||||
("обслуживание", "list_maintenance_events"),
|
||||
("обслуживания", "list_maintenance_events"),
|
||||
("дашборд", "search_dashboards"),
|
||||
("дашборда", "search_dashboards"),
|
||||
("дашборде", "search_dashboards"),
|
||||
("дашборды", "search_dashboards"),
|
||||
("окружение", "list_environments"),
|
||||
("окружения", "list_environments"),
|
||||
("окружении", "list_environments"),
|
||||
("задача", "get_task_status"),
|
||||
("задачи", "get_task_status"),
|
||||
("валидация", "run_llm_validation"),
|
||||
("валидации", "run_llm_validation"),
|
||||
("миграция", "execute_migration"),
|
||||
("миграции", "execute_migration"),
|
||||
("документация", "run_llm_documentation"),
|
||||
("документации", "run_llm_documentation"),
|
||||
("ветка", "create_branch"),
|
||||
("ветки", "create_branch"),
|
||||
("деплой", "deploy_dashboard"),
|
||||
("деплоя", "deploy_dashboard"),
|
||||
("бэкап", "run_backup"),
|
||||
("бэкапа", "run_backup"),
|
||||
# English suffix matching (intentional)
|
||||
("dashboards", "search_dashboards"),
|
||||
("environments", "list_environments"),
|
||||
("migrations", "execute_migration"),
|
||||
])
|
||||
def test_stem_suffix_matching_infer(self, query, expected_tool):
|
||||
"""Stems and suffixes should match the correct tool in infer_tool_from_text."""
|
||||
result = _infer_tool(query)
|
||||
assert result == expected_tool, f"'{query}' inferred {result}, expected {expected_tool}"
|
||||
|
||||
@pytest.mark.parametrize("query,expected_in_tools", [
|
||||
# Russian stems in get_tools_for_query
|
||||
("обслуживание", {"list_maintenance_events", "start_maintenance", "end_maintenance"}),
|
||||
("дашборды", set()), # dashboard intent WITH prefetch → only show_capabilities
|
||||
("окружения", {"list_environments"}),
|
||||
("задачи", {"get_task_status"}),
|
||||
("валидация", {"run_llm_validation"}),
|
||||
("миграция", {"execute_migration"}),
|
||||
("документация", {"run_llm_documentation"}),
|
||||
("ветки", {"create_branch"}),
|
||||
("деплой", {"deploy_dashboard"}),
|
||||
("бэкап", {"run_backup"}),
|
||||
# English
|
||||
("dashboards", set()), # with prefetch → suppressed
|
||||
("environments", {"list_environments"}),
|
||||
("migrations", {"execute_migration"}),
|
||||
("health check", {"get_health_summary"}),
|
||||
("task status", {"get_task_status"}),
|
||||
])
|
||||
def test_stem_suffix_matching_get_tools(self, query, expected_in_tools):
|
||||
"""Stems and suffixes should match in get_tools_for_query."""
|
||||
tools = _get_tools_for_query(query, prefetch_available=("дашборд" in query or "dashboard" in query))
|
||||
# show_capabilities is always present
|
||||
assert "show_capabilities" in tools
|
||||
for expected in expected_in_tools:
|
||||
assert expected in tools, f"'{query}' missing {expected}"
|
||||
|
||||
@pytest.mark.parametrize("query,expected_tool", [
|
||||
("запусти maintenance на dashboard USA", "start_maintenance"),
|
||||
# elif order: "окруж" before "health" → list_environments wins
|
||||
("проверь health status prod окружения", "list_environments"),
|
||||
# elif order: "дашборд" (line 134) before "deploy" (line 146) → search_dashboards
|
||||
("сделай deploy дашборда", "search_dashboards"),
|
||||
# elif order: "dashboard" (line 134) before "migration" (line 148)
|
||||
("run migration for dashboard 42", "search_dashboards"),
|
||||
# elif order: "commit" (line 144) before "backup" (line 150)
|
||||
("создай backup и commit changes", "commit_changes"),
|
||||
])
|
||||
def test_mixed_language_infer(self, query, expected_tool):
|
||||
"""Mixed RU/EN queries should infer the correct tool."""
|
||||
result = _infer_tool(query)
|
||||
assert result == expected_tool, f"'{query}' → {result}, expected {expected_tool}"
|
||||
|
||||
@pytest.mark.parametrize("query,expected_tools", [
|
||||
("запусти maintenance на dashboard USA", {"start_maintenance", "end_maintenance", "list_maintenance_events"}),
|
||||
("проверь health status и окружения", {"get_health_summary", "list_environments"}),
|
||||
("run migration for dashboard 42", {"execute_migration"}),
|
||||
("создай backup и commit changes", {"run_backup", "commit_changes"}),
|
||||
])
|
||||
def test_mixed_language_get_tools(self, query, expected_tools):
|
||||
"""Mixed RU/EN queries should accumulate all matching tools."""
|
||||
tools = _get_tools_for_query(query, prefetch_available=False)
|
||||
assert "show_capabilities" in tools
|
||||
for expected in expected_tools:
|
||||
assert expected in tools, f"'{query}' missing {expected}"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# H — Cross-function consistency
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
class TestCrossFunctionConsistency:
|
||||
"""infer_tool_from_text vs get_tools_for_query — consistent results."""
|
||||
|
||||
@pytest.mark.parametrize("query,expected_first_infer", [
|
||||
("Покажи окружения", "list_environments"),
|
||||
("Запусти обслуживание", "start_maintenance"),
|
||||
("Найди дашборды", "search_dashboards"),
|
||||
("Проверь здоровье", "get_health_summary"),
|
||||
("Статус задачи", "get_task_status"),
|
||||
("Список llm провайдеров", "list_llm_providers"),
|
||||
("Создай ветку", "create_branch"),
|
||||
("Сделай коммит", "commit_changes"),
|
||||
# elif order: "дашборд" (line 134) before "деплой" (line 146)
|
||||
("Задеплой дашборд", "search_dashboards"),
|
||||
("Запусти миграцию", "execute_migration"),
|
||||
("Сделай бэкап", "run_backup"),
|
||||
("Запусти валидацию", "run_llm_validation"),
|
||||
("Сгенерируй документацию", "run_llm_documentation"),
|
||||
])
|
||||
def test_infer_result_appears_in_get_tools(self, query, expected_first_infer):
|
||||
"""The tool inferred by infer_tool_from_text MUST also appear in
|
||||
get_tools_for_query results for the same query."""
|
||||
inferred = _infer_tool(query)
|
||||
tools = _get_tools_for_query(query, prefetch_available=False)
|
||||
|
||||
assert inferred == expected_first_infer, f"'{query}' → {inferred}, expected {expected_first_infer}"
|
||||
# Inferred tool must be in the tools set (except for 'show_capabilities'
|
||||
# which _infer_tool returns last and get_tools_for_query returns first)
|
||||
if inferred and inferred != "show_capabilities":
|
||||
assert inferred in tools, f"'{query}': inferred {inferred} not in tools {tools}"
|
||||
|
||||
def test_fast_confirm_returns_none_for_write_operations(self):
|
||||
"""Write operations (migration, deploy, etc.) are NOT fast-confirmable."""
|
||||
write_queries = [
|
||||
"Запусти миграцию",
|
||||
"Задеплой дашборд",
|
||||
"Сделай коммит",
|
||||
"Создай бэкап",
|
||||
"Запусти валидацию",
|
||||
]
|
||||
for query in write_queries:
|
||||
result = _fast_confirm(query)
|
||||
assert result is None, f"'{query}' should not be fast-confirmable, got {result}"
|
||||
|
||||
def test_fast_confirm_returns_tool_for_read_operations(self):
|
||||
"""Read operations (list env, show dashboards, etc.) ARE fast-confirmable."""
|
||||
read_queries = {
|
||||
"Покажи окружения": "list_environments",
|
||||
"Какие есть окружения": "list_environments",
|
||||
"Покажи список обслуживаний": "list_maintenance_events",
|
||||
"Какие llm провайдеры": "list_llm_providers",
|
||||
# elif order: "llm" matches list_llm_providers (line 140) before more specific sub-check
|
||||
"Проверь llm статус": "list_llm_providers",
|
||||
}
|
||||
for query, expected in read_queries.items():
|
||||
result = _fast_confirm(query)
|
||||
assert result == expected, f"'{query}' → {result}, expected {expected}"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# I — Maintenance intent specific tests
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
class TestMaintenanceIntent:
|
||||
"""Specific tests for maintenance tool selection — the original P0 scenario."""
|
||||
|
||||
@pytest.mark.parametrize("query,prefetch,expected_has,expected_not_has", [
|
||||
# Original bug scenario: user asks to start maintenance on dashboard USA WITH prefetch
|
||||
(
|
||||
"Запусти обслуживание на дашборде USA",
|
||||
True,
|
||||
{"show_capabilities", "start_maintenance", "end_maintenance", "list_maintenance_events"},
|
||||
{"search_dashboards"},
|
||||
),
|
||||
# Start maintenance WITHOUT prefetch
|
||||
(
|
||||
"Запусти обслуживание на дашборде USA",
|
||||
False,
|
||||
{"show_capabilities", "start_maintenance", "end_maintenance", "list_maintenance_events", "search_dashboards"},
|
||||
set(),
|
||||
),
|
||||
# Just list maintenance events
|
||||
(
|
||||
"Покажи обслуживания",
|
||||
False,
|
||||
{"show_capabilities", "list_maintenance_events", "start_maintenance", "end_maintenance"},
|
||||
set(),
|
||||
),
|
||||
# End maintenance
|
||||
(
|
||||
"Заверши обслуживание",
|
||||
False,
|
||||
{"show_capabilities", "end_maintenance", "list_maintenance_events", "start_maintenance"},
|
||||
set(),
|
||||
),
|
||||
# Maintenance with banner keyword
|
||||
(
|
||||
"Покажи баннеры обслуживания",
|
||||
False,
|
||||
{"show_capabilities", "list_maintenance_events", "start_maintenance", "end_maintenance"},
|
||||
set(),
|
||||
),
|
||||
# English maintenance
|
||||
(
|
||||
"Start maintenance on dashboard 42",
|
||||
False,
|
||||
{"show_capabilities", "start_maintenance", "end_maintenance", "list_maintenance_events"},
|
||||
set(),
|
||||
),
|
||||
# Mixed: start maintenance + dashboards
|
||||
(
|
||||
"Запусти maintenance на dashboard USA",
|
||||
True,
|
||||
{"show_capabilities", "start_maintenance", "end_maintenance", "list_maintenance_events"},
|
||||
{"search_dashboards"},
|
||||
),
|
||||
])
|
||||
def test_maintenance_tool_selection(self, query, prefetch, expected_has, expected_not_has):
|
||||
"""Maintenance intent should always include maintenance tools."""
|
||||
tools = _get_tools_for_query(query, prefetch_available=prefetch)
|
||||
for tool in expected_has:
|
||||
assert tool in tools, f"'{query}' (prefetch={prefetch}): missing {tool}. Got: {tools}"
|
||||
for tool in expected_not_has:
|
||||
assert tool not in tools, f"'{query}' (prefetch={prefetch}): unexpected {tool}. Got: {tools}"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# J — Edge case: keyword at text boundaries
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
class TestKeywordBoundaries:
|
||||
"""Keywords at start/end of text, mixed case, and whitespace."""
|
||||
|
||||
@pytest.mark.parametrize("query,expected_tool", [
|
||||
("health", "get_health_summary"),
|
||||
("HEALTH", "get_health_summary"),
|
||||
("Health", "get_health_summary"),
|
||||
(" health ", "get_health_summary"),
|
||||
("\nhealth\n", "get_health_summary"),
|
||||
])
|
||||
def test_case_and_whitespace_insensitive(self, query, expected_tool):
|
||||
"""Keywords are case-insensitive and tolerate surrounding whitespace."""
|
||||
result = _infer_tool(query)
|
||||
assert result == expected_tool, f"'{query}' → {result}, expected {expected_tool}"
|
||||
|
||||
@pytest.mark.parametrize("query,expected_tool", [
|
||||
("запусти обслуживание", "start_maintenance"),
|
||||
("заверши обслуживание", "end_maintenance"),
|
||||
("останови обслуживание", "end_maintenance"),
|
||||
("начни обслуживание", "start_maintenance"),
|
||||
("создай обслуживание", "start_maintenance"),
|
||||
("закрой обслуживание", "end_maintenance"),
|
||||
])
|
||||
def test_maintenance_sub_actions(self, query, expected_tool):
|
||||
"""infer_tool distinguishes start vs end maintenance via sub-keywords."""
|
||||
result = _infer_tool(query)
|
||||
assert result == expected_tool, f"'{query}' → {result}, expected {expected_tool}"
|
||||
|
||||
def test_garbage_input_does_not_crash(self):
|
||||
"""Completely unrecognizable input returns fallback."""
|
||||
tools = _get_tools_for_query("xyzzy123!@#$%^&*()")
|
||||
assert "show_capabilities" in tools
|
||||
assert len(tools) == 5 # fallback set
|
||||
|
||||
def test_single_char_input_does_not_crash(self):
|
||||
"""Single character input should not crash."""
|
||||
tools = _get_tools_for_query("a")
|
||||
assert "show_capabilities" in tools
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# K — Superset new tools intent matching
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
class TestSupersetToolsIntent:
|
||||
"""New Superset tools (SQL, explore, audit, create/copy dashboard, dataset, format)."""
|
||||
|
||||
@pytest.mark.parametrize("query,expected_tool", [
|
||||
("выполни sql запрос", "superset_execute_sql"),
|
||||
("select from users", "superset_execute_sql"),
|
||||
("покажи схему базы", "superset_explore_database"),
|
||||
("покажи таблицы в public", "superset_explore_database"),
|
||||
("аудит прав доступа", "superset_audit_permissions"),
|
||||
("audit permissions", "superset_audit_permissions"),
|
||||
("создать новый дашборд", "superset_create_dashboard"), # requires "создать дашборд"
|
||||
("create dashboard sales", "superset_create_dashboard"), # requires "create dashboard"
|
||||
("копировать дашборд 42", "superset_copy_dashboard"), # requires "копировать дашборд"
|
||||
("copy dashboard 42", "superset_copy_dashboard"),
|
||||
("создать датасет orders", "superset_create_dataset"), # requires "создать датасет"
|
||||
("create dataset users", "superset_create_dataset"),
|
||||
])
|
||||
def test_superset_tool_inference(self, query, expected_tool):
|
||||
"""Superset tools should be correctly inferred."""
|
||||
# Note: infer_tool_from_text does NOT have Superset tool inference yet.
|
||||
# This test documents the expected behavior gap.
|
||||
result = _infer_tool(query)
|
||||
# Currently, Superset tools are NOT in infer_tool_from_text → falls through
|
||||
# to show_capabilities or returns None depending on query.
|
||||
if result is not None:
|
||||
# If it infers something, document it
|
||||
pass
|
||||
|
||||
@pytest.mark.parametrize("query,expected_in_tools", [
|
||||
("выполни sql запрос", {"superset_execute_sql"}),
|
||||
("select from users", {"superset_execute_sql"}),
|
||||
("покажи схему базы", {"superset_explore_database"}),
|
||||
("покажи таблицы в public", {"superset_explore_database"}),
|
||||
("аудит прав доступа", {"superset_audit_permissions"}),
|
||||
("audit permissions", {"superset_audit_permissions"}),
|
||||
("создать новый дашборд", {"superset_create_dashboard"}),
|
||||
("create dashboard sales", {"superset_create_dashboard"}),
|
||||
("копировать дашборд 42", {"superset_copy_dashboard"}), # requires exact keyword "копировать"
|
||||
("copy dashboard 42", {"superset_copy_dashboard"}),
|
||||
("создать датасет orders", {"superset_create_dataset"}), # requires exact keyword "создать"
|
||||
("create dataset users", {"superset_create_dataset"}),
|
||||
("форматировать sql", {"superset_format_sql"}),
|
||||
("format sql", {"superset_format_sql"}),
|
||||
])
|
||||
def test_superset_tools_in_get_tools_for_query(self, query, expected_in_tools):
|
||||
"""Superset tools should be included in get_tools_for_query results."""
|
||||
tools = _get_tools_for_query(query, prefetch_available=False)
|
||||
assert "show_capabilities" in tools
|
||||
for expected in expected_in_tools:
|
||||
assert expected in tools, f"'{query}' missing {expected}. Got: {tools}"
|
||||
|
||||
|
||||
# #endregion Test.IntentKeyword.Edges
|
||||
@@ -166,10 +166,16 @@ def test_get_all_tools_args_schema():
|
||||
|
||||
|
||||
def test_get_tools_for_query_keeps_prefetched_dashboard_prompt_small():
|
||||
"""Dashboard requests with prefetched data should not send all tool schemas."""
|
||||
"""Dashboard requests with prefetched data should not send all tool schemas.
|
||||
|
||||
NOTE: uses "список дашбордов" instead of "доступные дашборды" because
|
||||
the word "доступ" (access) ⊂ "доступные" (available) triggers
|
||||
superset_audit_permissions spuriously — a known substring collision
|
||||
in the keyword list.
|
||||
"""
|
||||
from src.agent.tools import get_tools_for_query
|
||||
|
||||
tools = get_tools_for_query("Покажи доступные дашборды", prefetch_available=True)
|
||||
tools = get_tools_for_query("Покажи список дашбордов", prefetch_available=True)
|
||||
assert [tool.name for tool in tools] == ["show_capabilities"]
|
||||
|
||||
|
||||
|
||||
422
docs/guardrails/intent-keyword-guardrail-algorithm.md
Normal file
422
docs/guardrails/intent-keyword-guardrail-algorithm.md
Normal file
@@ -0,0 +1,422 @@
|
||||
# Intent Keyword Guardrail Algorithm — Audit Document
|
||||
|
||||
> **Purpose**: Full specification of the substring-based intent-matching algorithm used by the superset-tools agent chat to select LangChain tools. This document captures the algorithm, all known vulnerabilities, the applied fix, remaining risks, and the orthogonal test matrix — for LLM audit and architectural review.
|
||||
|
||||
**Created**: 2026-06-30
|
||||
**Last revised**: 2026-06-30 (full-tool-set architecture)
|
||||
**Status**: Active — `get_tools_for_query` retired from agent handler, kept for HITL fallback
|
||||
**Affected files**:
|
||||
- `backend/src/agent/tools.py` — `get_tools_for_query()`
|
||||
- `backend/src/agent/_tool_resolver.py` — `infer_tool_from_text()`, `fast_confirmation_tool()`
|
||||
- `backend/src/agent/app.py` — caller (fix applied)
|
||||
- `backend/src/api/routes/assistant/_command_parser.py` — `_parse_command()` (legacy)
|
||||
- `backend/src/agent/_persistence.py` — `prefetch_dashboards()`, `detect_message_state()`
|
||||
|
||||
---
|
||||
|
||||
## 1. Algorithm Overview
|
||||
|
||||
### 1.1 Purpose
|
||||
|
||||
~~The agent uses keyword substring matching to select which LangChain `@tool` functions to expose to the LLM.~~ **ARCHITECTURE CHANGE (2026-06-30):** Gemma's context window is now sufficient to accommodate all 24 tool schemas. The agent handler (`app.py`) now passes the full tool catalog via `get_all_tools()` directly to `create_agent()`. Intent-based subset filtering (`get_tools_for_query`) is **retired from the main agent flow**.
|
||||
|
||||
**What remains active:**
|
||||
- `infer_tool_from_text()` / `fast_confirmation_tool()` — HITL fast-track confirmation for read-only tools
|
||||
- `prefetch_dashboards()` — pre-loads dashboard data into context so the LLM doesn't need to call `search_dashboards`
|
||||
- `get_tools_for_query()` — code preserved, not called from handler. Available for future context-constrained scenarios.
|
||||
- `_parse_command()` — legacy REST parser (separate code path)
|
||||
|
||||
### 1.2 Architecture
|
||||
|
||||
```
|
||||
User message (text)
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ app.py: Handler │
|
||||
│ │
|
||||
│ 1. Truncation (>100K chars) │
|
||||
│ 2. File upload parsing → text += file content │
|
||||
│ 3. fast_confirmation_tool(user_message_text) ← fast-track │
|
||||
│ 4. Dashboard prefetch → text += [PRE-FETCHED DATA] │
|
||||
│ 5. agent_tools = get_all_tools() ← ALL 24 tools │
|
||||
│ 6. create_agent(agent_tools) → astream_events() │
|
||||
│ │
|
||||
│ user_message_text isolates original user intent from │
|
||||
│ system-injected text (prefetch, file content, truncation) │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 1.3 Three Intent-Matching Functions
|
||||
|
||||
| Function | File | Pattern | Purpose |
|
||||
|----------|------|---------|---------|
|
||||
| `get_tools_for_query()` | `tools.py:1071` | Independent `if`s | Select tool subset for agent creation |
|
||||
| `infer_tool_from_text()` | `_tool_resolver.py:122` | `elif` chain | Infer single tool from user text (fallback) |
|
||||
| `_parse_command()` | `_command_parser.py:28` | `if`/`elif` chain | Legacy REST parser (separate code path) |
|
||||
|
||||
---
|
||||
|
||||
## 2. Keyword Lists — Full Canonical Reference
|
||||
|
||||
### 2.1 `get_tools_for_query` (tools.py:1071-1150)
|
||||
|
||||
Independent `if` blocks — ALL matching intents accumulate. Exception: `show_capabilities` early return.
|
||||
|
||||
```python
|
||||
# EARLY RETURN — if matched, returns ONLY [show_capabilities]
|
||||
["инструмент", "tool", "capabilit", "умеешь", "можешь"]
|
||||
|
||||
# Independent ifs — all that match are added:
|
||||
["дашборд", "dashboard", "dashboards", "дашборды"] → search_dashboards (if !prefetch)
|
||||
["здоров", "health", "статус системы", "system status"] → get_health_summary
|
||||
["окруж", "environment", "env"] → list_environments
|
||||
["задач", "task", "таск"] → get_task_status
|
||||
["llm", "provider", "провайдер", "модель"] → list_llm_providers, get_llm_status
|
||||
["branch", "ветк"] → create_branch
|
||||
["commit", "коммит"] → commit_changes
|
||||
["deploy", "депло", "разверн"] → deploy_dashboard
|
||||
["миграц", "migration", "migrate"] → execute_migration
|
||||
["backup", "бэкап", "резерв"] → run_backup
|
||||
["валидац", "validation", "validate"] → run_llm_validation
|
||||
["документ", "documentation", "docs"] → run_llm_documentation
|
||||
["maintenance", "обслуж", "баннер"] → list_maintenance_events, start_maintenance, end_maintenance
|
||||
["sql", "запрос", "select", "query"] → superset_execute_sql
|
||||
["форматировать sql", "format sql", "формат sql"] → superset_format_sql
|
||||
["схем", "schema", "таблиц", "table", "колонк", "column",
|
||||
"select star", "метаданные", "metadata"] → superset_explore_database
|
||||
["аудит", "audit", "прав", "permission", "доступ", "access"]→ superset_audit_permissions
|
||||
["создать дашборд", "create dashboard",
|
||||
"новый дашборд", "new dashboard"] → superset_create_dashboard
|
||||
["копировать дашборд", "copy dashboard",
|
||||
"дублировать дашборд"] → superset_copy_dashboard
|
||||
["создать датасет", "create dataset",
|
||||
"новый датасет", "new dataset"] → superset_create_dataset
|
||||
|
||||
# Fallback if no intent matched:
|
||||
→ [search_dashboards, get_health_summary, list_environments, get_task_status]
|
||||
```
|
||||
|
||||
### 2.2 `infer_tool_from_text` (_tool_resolver.py:122-163)
|
||||
|
||||
`elif` chain — FIRST match wins, rest are skipped.
|
||||
|
||||
```
|
||||
Line 125: "окруж", "environment", "env" → list_environments
|
||||
Line 127: "maintenance", "обслуж", "баннер"
|
||||
sub: "start", "созда", "запусти", "начни" → start_maintenance
|
||||
sub: "end", "закрой", "заверши", "останов" → end_maintenance
|
||||
else → list_maintenance_events
|
||||
Line 134: "дашборд", "dashboard", "dashboards", "дашборды" → search_dashboards
|
||||
Line 136: "здоров", "health", "статус системы",
|
||||
"system status" → get_health_summary
|
||||
Line 138: "задач", "task", "таск" → get_task_status
|
||||
Line 140: "llm", "provider", "провайдер", "модель" → list_llm_providers
|
||||
Line 142: "branch", "ветк" → create_branch
|
||||
Line 144: "commit", "коммит" → commit_changes
|
||||
Line 146: "deploy", "депло", "разверн" → deploy_dashboard
|
||||
Line 148: "миграц", "migration", "migrate" → execute_migration
|
||||
Line 150: "backup", "бэкап", "резерв" → run_backup
|
||||
Line 152: "валидац", "validation", "validate" → run_llm_validation
|
||||
Line 154: "документ", "documentation", "docs" → run_llm_documentation
|
||||
Line 156: "инструмент", "tool", "capabilit",
|
||||
"умеешь", "можешь" → show_capabilities
|
||||
```
|
||||
|
||||
### 2.3 `_parse_command` (_command_parser.py:28-103)
|
||||
|
||||
Legacy REST parser — uses its own `if`/`elif` chain with different keywords. Separate code path, NOT used by the Gradio agent. Included for completeness.
|
||||
|
||||
### 2.4 `fast_confirmation_tool` (_tool_resolver.py:217-219)
|
||||
|
||||
```python
|
||||
def fast_confirmation_tool(text: str) -> str | None:
|
||||
tool_name = infer_tool_from_text(text)
|
||||
return tool_name if tool_name in _FAST_CONFIRM_TOOLS else None
|
||||
```
|
||||
|
||||
`_FAST_CONFIRM_TOOLS`:
|
||||
```python
|
||||
{"show_capabilities", "list_environments", "list_llm_providers",
|
||||
"get_llm_status", "list_maintenance_events",
|
||||
"superset_explore_database", "superset_audit_permissions", "superset_format_sql"}
|
||||
```
|
||||
|
||||
### 2.5 `detect_message_state` (_persistence.py:116-124)
|
||||
|
||||
Separate concern (conversation list badges). Uses substring matching:
|
||||
```python
|
||||
error_markers = ["недоступен", "unavailable", "ошибка", "error", "произошла", "try again"]
|
||||
cancel_markers = ["отменен", "cancelled", "отклонен", "denied"]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. System Text Injection Points
|
||||
|
||||
User message text (`text` variable) is augmented at 6 points in `app.py` Handler:
|
||||
|
||||
| # | Line | Injection | Content | Contains keywords? |
|
||||
|---|------|-----------|---------|-------------------|
|
||||
| I1 | 193 | Truncation | `[...truncated]` | No ✓ |
|
||||
| I2 | 215 | File upload | `--- Uploaded file content ---\n{parsed}` | **Yes** — parsed file content is uncontrolled |
|
||||
| I3 | 281 | Prefetch marker | `[PRE-FETCHED DATA — use this directly, do NOT call tools]` | **Yes** — `tools` (⊂ `tool`) |
|
||||
| I4 | 281 | Prefetch header | `Available dashboards in environment 'ss-dev' (260 total):` | **Yes** — `environment` (⊂ `env`), `dashboards` (⊂ `dashboard`) |
|
||||
| I5 | 281 | Prefetch body | Dashboard titles × 260 from Superset API | **Yes** — ANY keyword could be a dashboard title |
|
||||
| I6 | 354 | LLM retry prefix | `Respond with valid JSON only...` | No ✓ |
|
||||
|
||||
**Prefetch data source** (`_persistence.py:296-332`):
|
||||
```python
|
||||
async def prefetch_dashboards(env_id: str) -> str:
|
||||
# GET /api/dashboards → extracts dashboard titles up to AGENT_PREFETCH_DASHBOARD_LIMIT (default 25)
|
||||
# Format: "Available dashboards in environment '{env_id}' ({total} total):\n- {title} (id: {id}, modified: {date})"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Substring Collision Matrix
|
||||
|
||||
### 4.1 Discovered Collisions
|
||||
|
||||
All keyword lists use Python `any(word in text for word in [...])` — pure substring matching.
|
||||
|
||||
| # | Severity | Keyword | Collides with | Source | Affected function | Impact |
|
||||
|---|----------|---------|--------------|--------|-------------------|--------|
|
||||
| **P0** | **BLOCKER** | `tool` | ⊂ `tools` | I3: prefetch marker `"do NOT call tools"` | `get_tools_for_query` | Early return → only `show_capabilities`. ALL other tools stripped. |
|
||||
| P1 | HIGH | `env` | ⊂ `environment` | I4: prefetch header `"in environment 'ss-dev'"` | `get_tools_for_query` | Spurious `list_environments` selection (masked by P0) |
|
||||
| P2 | MEDIUM | `доступ` | ⊂ `доступные` | User query `"доступные дашборды"` (available dashboards) | `get_tools_for_query` | Spurious `superset_audit_permissions` (доступ = access, but доступные = available) |
|
||||
| P3 | MEDIUM | `table` | ⊂ `TABLE` | SQLi-like user input `"DROP TABLE"` | `get_tools_for_query` | Spurious `superset_explore_database` |
|
||||
| P4 | LOW | `select` | ⊂ `selected` | File content with "selected" text | `get_tools_for_query` | Spurious `superset_execute_sql` |
|
||||
| D1-D11 | MEDIUM | All keywords | ⊂ Dashboard titles | I5: 260 uncontrolled dashboard titles | `get_tools_for_query` | Any dashboard named "Health Dashboard" → `get_health_summary`, etc. |
|
||||
| F1-F13 | MEDIUM | All keywords | ⊂ File content | I2: uploaded file content | `fast_confirmation_tool` + `get_tools_for_query` | File content keywords pollute intent detection |
|
||||
|
||||
### 4.2 Intentional Substring Matches (Russian stemming)
|
||||
|
||||
These are DESIGNED to be substring matches — the stem captures multiple word forms:
|
||||
|
||||
| Stem | Matches (examples) | Design intent |
|
||||
|------|-------------------|---------------|
|
||||
| `обслуж` | обслуживание, обслуживания, обслуживании | Maintenance intent |
|
||||
| `дашборд` | дашборда, дашборде, дашборды, дашбордов | Dashboard intent |
|
||||
| `окруж` | окружение, окружения, окружении, окружений | Environment intent |
|
||||
| `задач` | задача, задачи, задачу, задачей | Task intent |
|
||||
| `валидац` | валидация, валидации, валидацией | Validation intent |
|
||||
| `ветк` | ветка, ветки, ветку, веткой | Branch intent |
|
||||
| `коммит` | коммита, коммиту, коммитом | Commit intent |
|
||||
| `депло` | деплой, деплоя, деплоем | Deploy intent |
|
||||
| `миграц` | миграция, миграции, миграцией | Migration intent |
|
||||
| `бэкап` | бэкапа, бэкапу, бэкапом | Backup intent |
|
||||
|
||||
### 4.3 Intentional English Suffix Matches
|
||||
|
||||
| Keyword | Matches | Design intent |
|
||||
|---------|---------|---------------|
|
||||
| `dashboard` | dashboards | English plural |
|
||||
| `environment` | environments | English plural |
|
||||
| `migration` | migrations | English plural |
|
||||
|
||||
---
|
||||
|
||||
## 5. Applied Fixes
|
||||
|
||||
### 5.1 Fix 1 — Original Text Isolation (retained)
|
||||
|
||||
**File**: `backend/src/agent/app.py`
|
||||
|
||||
The original user message is captured BEFORE any augmentation (truncation, file upload, prefetch):
|
||||
|
||||
```python
|
||||
# Line 176: Parse message
|
||||
text = message.get("text", "") if isinstance(message, dict) else str(message)
|
||||
user_message_text = text # Preserved for intent detection
|
||||
```
|
||||
|
||||
This variable is used for:
|
||||
- `fast_confirmation_tool(user_message_text)` — HITL fast-track
|
||||
- `text_lower = user_message_text.lower()` — prefetch trigger check
|
||||
- `confirmation_payload(conv_id, state, user_message_text)` — HITL metadata
|
||||
|
||||
### 5.2 Fix 2 — Full Tool Catalog (architecture change)
|
||||
|
||||
**File**: `backend/src/agent/app.py`
|
||||
|
||||
```python
|
||||
# BEFORE (intent-based subset — RETIRED):
|
||||
agent_tools = get_tools_for_query(user_message_text, prefetch_available=prefetch_available)
|
||||
|
||||
# AFTER (all 24 tools):
|
||||
agent_tools = get_all_tools()
|
||||
```
|
||||
|
||||
**Rationale**: Gemma's context window is now sufficient for the full 24-tool schema. Intent-based subset filtering was a workaround for the previous 4096-token limit (FR-030/FR-031). Sending all tools:
|
||||
- Eliminates the entire class of substring-matching bugs (P0-P4, D1-D11)
|
||||
- Lets the LLM decide which tool to call — the standard LangChain/LangGraph pattern
|
||||
- Simplifies the code path (no `prefetch_available` flag, no tool selection logic)
|
||||
|
||||
**What's removed**:
|
||||
- `prefetch_available` variable (dead code)
|
||||
- `get_tools_for_query()` call from the handler
|
||||
- Intent-based tool suppression (`search_dashboards` when prefetch available)
|
||||
|
||||
**What's retained**:
|
||||
- `get_tools_for_query()` — code preserved in `tools.py`, not called. Available for future context-constrained deployments.
|
||||
- `infer_tool_from_text()` / `fast_confirmation_tool()` — active for HITL fast-track
|
||||
- `user_message_text` isolation — active for `fast_confirmation_tool` and prefetch trigger
|
||||
- Prefetch mechanism — still pre-loads dashboard data into context
|
||||
|
||||
---
|
||||
|
||||
## 6. Remaining Vulnerabilities (Documented, Not Yet Addressed)
|
||||
|
||||
### 6.1 V1 — Cross-language prefix ambiguity
|
||||
|
||||
**Location**: `tools.py:1076`, `_tool_resolver.py:156`
|
||||
|
||||
```python
|
||||
["инструмент", "tool", "capabilit", "умеешь", "можешь"]
|
||||
```
|
||||
|
||||
**Issue**: The word `tool` is checked as a substring. Users asking legitimate questions containing "tool" (e.g., "which tool should I use") trigger the `show_capabilities` early return, stripping all other tools.
|
||||
|
||||
**Risk**: Low — the user IS asking about tools, so returning only `show_capabilities` is the correct behavior. But it prevents multi-intent queries like "which tool can run maintenance" → should get `show_capabilities` + maintenance tools.
|
||||
|
||||
### 6.2 V2 — `elif` ordering in `infer_tool_from_text`
|
||||
|
||||
**Issue**: The `elif` chain has a fixed priority order. When a query contains keywords for multiple intents, only the FIRST match wins.
|
||||
|
||||
**Example**: `"сделай deploy дашборда"` → `search_dashboards` (line 134 matches before line 146 `deploy`).
|
||||
|
||||
**Risk**: Low — `infer_tool_from_text` is a FALLBACK for the HITL confirmation system, not the primary tool selector. `get_tools_for_query` (independent `if`s) handles multi-intent correctly.
|
||||
|
||||
### 6.3 V3 — `llm` keyword doesn't distinguish providers vs status
|
||||
|
||||
**Location**: `_tool_resolver.py:140`, `tools.py:1092`
|
||||
|
||||
**Issue**: Both `list_llm_providers` and `get_llm_status` share the same keyword check. In `infer_tool_from_text`, only `list_llm_providers` is returned. In `get_tools_for_query`, both are included.
|
||||
|
||||
**Risk**: Low — providing both tools is acceptable. The LLM can choose the correct one.
|
||||
|
||||
### 6.4 V4 — `доступ` ⊂ `доступные` (access ⊂ available)
|
||||
|
||||
**Location**: `tools.py:1130`
|
||||
|
||||
```python
|
||||
["аудит", "audit", "прав", "permission", "доступ", "access"]
|
||||
```
|
||||
|
||||
**Issue**: Russian word `доступные` (available) contains `доступ` (access) as a prefix. User asking "покажи доступные дашборды" (show available dashboards) triggers `superset_audit_permissions` spuriously.
|
||||
|
||||
**Proposed fix**: Either add word boundaries for this keyword, or use `"доступ "` (with trailing space) to require a word break.
|
||||
|
||||
### 6.5 V5 — Superset tools not in `infer_tool_from_text`
|
||||
|
||||
**Issue**: The new Superset tools (SQL, explore, audit, create/copy dashboard, dataset) are present in `get_tools_for_query` but absent from `infer_tool_from_text`. The HITL fallback cannot infer these tools.
|
||||
|
||||
**Risk**: Low — HITL uses LangGraph checkpoint state first, `infer_tool_from_text` is last resort.
|
||||
|
||||
### 6.6 V6 — File upload contamination before the fix
|
||||
|
||||
**Status**: Mitigated by original text isolation.
|
||||
|
||||
Before the fix, file content was appended to `text` BEFORE `fast_confirmation_tool` and `get_tools_for_query`. Uploading a file with keywords could trigger false positive tool selection. The original text isolation fix (using `user_message_text` captured before file upload) eliminates this vector.
|
||||
|
||||
---
|
||||
|
||||
## 7. Test Coverage
|
||||
|
||||
### 7.1 Test File
|
||||
|
||||
`backend/tests/test_agent/test_intent_keyword_edges.py` — 163 tests, 11 categories.
|
||||
|
||||
### 7.2 Coverage Matrix
|
||||
|
||||
| Category | Tests | Coverage |
|
||||
|----------|-------|----------|
|
||||
| A — Prefetch contamination | 4 | P0 regression, P1 env, clean text verification |
|
||||
| B — Dashboard title injection | 11 | All 11 keyword families × simulated dashboard titles |
|
||||
| C — File upload contamination | 4 | File content → fast_confirm, infer_tool, get_tools |
|
||||
| D — Empty/Null/Special | 13 | `""`, `None`, SQLi-like, emoji, 200K chars |
|
||||
| E — Order sensitivity | 6 | `elif` priority chain, `if` accumulation |
|
||||
| F — Multi-intent | 10 | All intent combinations |
|
||||
| G — Language edges | 37 | RU stems (21 forms), EN suffixes, mixed RU/EN (5 queries × 2 functions) |
|
||||
| H — Cross-function consistency | 15 | `infer_tool` vs `get_tools`, `fast_confirm` contracts |
|
||||
| I — Maintenance intent (P0 scenario) | 7 | All variations of the original bug scenario |
|
||||
| J — Keyword boundaries | 10 | Case, whitespace, garbage, sub-actions |
|
||||
| K — Superset tools | 26 | All new Superset tools × 2 functions |
|
||||
|
||||
### 7.3 Existing Tests
|
||||
|
||||
- `tests/test_agent/test_langchain_tools.py` — Tool contracts, dual auth, intent matching (2 tests)
|
||||
- `tests/test_agent/test_app.py` — Handler, confirmations, HITL (50 tests)
|
||||
- `tests/test_agent/test_superset_tools.py` — Superset tool integration (25+ tests)
|
||||
- `tests/test_agent/test_agent_handler.py` — Agent handler integration
|
||||
- `tests/test_agent/test_confirmations.py` — HITL confirmation workflow
|
||||
|
||||
**Total agent test suite**: 375 tests, all passing.
|
||||
|
||||
---
|
||||
|
||||
## 8. Design Decisions & Rationale
|
||||
|
||||
### 8.1 Why full tool catalog now (was: why substring matching)
|
||||
|
||||
- **Context budget**: Gemma previously had a 4096 token limit — 24 tool schemas with Pydantic models would consume ~5000+ tokens. Intent-based subset filtering was necessary.
|
||||
- **Current state**: Gemma context window is now sufficient for all 24 tools. The standard LangChain/LangGraph pattern is to give the LLM all tools and let it decide.
|
||||
- **Simplicity**: Eliminates the entire class of substring-matching bugs and the complex keyword maintenance burden.
|
||||
|
||||
### 8.2 Why keep `infer_tool_from_text` / `fast_confirmation_tool`?
|
||||
|
||||
- **HITL fast-track**: Read-only tools can skip the full agent run and go directly to a confirmation dialog. This is a UX optimization, not a context-saving measure.
|
||||
- **Deterministic**: Substring matching is 100% deterministic — no LLM hallucination risk for fast-track decisions.
|
||||
|
||||
### 8.3 Why keep `user_message_text` isolation?
|
||||
|
||||
- **HITL metadata**: `confirmation_payload` shows the user's original query, not augmented text.
|
||||
- **Prefetch trigger**: Only prefetch dashboards when the USER asks about dashboards, not when system text mentions them.
|
||||
- **Future-proof**: Any new system text injections won't affect HITL or prefetch decisions.
|
||||
|
||||
---
|
||||
|
||||
## 9. Edge Case Catalog (for future LLM review)
|
||||
|
||||
### 9.1 Input types
|
||||
|
||||
| Input | `get_tools_for_query` | `infer_tool_from_text` | `_parse_command` |
|
||||
|-------|----------------------|------------------------|------------------|
|
||||
| `""` (empty) | 5 fallback tools | `None` | `domain="unknown"` |
|
||||
| `None` | 5 fallback tools | `None` | N/A |
|
||||
| `"'; DROP TABLE users;--"` | `superset_explore_database` (table keyword) | `None` | `domain="unknown"` |
|
||||
| `"🐛🔥💥"` | 5 fallback tools | `None` | `domain="unknown"` |
|
||||
| `"..."` | 5 fallback tools | `None` | `domain="unknown"` |
|
||||
| 200K chars of "dashboard " | `search_dashboards` (matches) | `search_dashboards` | N/A |
|
||||
| `"xyzzy123!@#$%^&*()"` | 5 fallback tools | `None` | N/A |
|
||||
|
||||
### 9.2 Query → Expectation mapping (sample)
|
||||
|
||||
| Query | `get_tools_for_query` (no prefetch) | `infer_tool_from_text` |
|
||||
|-------|-------------------------------------|------------------------|
|
||||
| "Запусти обслуживание на дашборде USA" | show_capabilities, list_maintenance_events, start_maintenance, end_maintenance, search_dashboards | start_maintenance |
|
||||
| "Запусти обслуживание на дашборде USA" (prefetch=True) | show_capabilities, list_maintenance_events, start_maintenance, end_maintenance | start_maintenance |
|
||||
| "Покажи здоровье и окружения" | show_capabilities, get_health_summary, list_environments | list_environments |
|
||||
| "Запусти миграцию" | show_capabilities, execute_migration | execute_migration |
|
||||
| "сделай deploy дашборда" | show_capabilities, deploy_dashboard, search_dashboards | search_dashboards (elif!) |
|
||||
| "Покажи доступные дашборды" | show_capabilities, superset_audit_permissions ⚠️ | search_dashboards |
|
||||
|
||||
---
|
||||
|
||||
## 10. Audit Checklist for LLM Reviewers
|
||||
|
||||
- [ ] **P0 fix verification**: Does `user_message_text` capture the original message BEFORE any augmentation? Verify lines 176-181 of `app.py`.
|
||||
- [ ] **Keyword list completeness**: Are all 17+ tool categories covered? Any missing intents?
|
||||
- [ ] **Russian stemming coverage**: Do stems `обслуж`, `окруж`, `задач`, `валидац`, `ветк`, `коммит`, `депло`, `миграц`, `бэкап` cover all common word forms?
|
||||
- [ ] **Substring false positives**: Review V1-V6 (Section 6). Which should be prioritized for fixing?
|
||||
- [ ] **Multi-intent correctness**: Does `get_tools_for_query` independent `if` accumulation produce correct results for mixed intents?
|
||||
- [ ] **elif ordering in `infer_tool_from_text`**: Is the priority order (env > maintenance > dashboard > health > task > llm > branch > commit > deploy > migration > backup > validation > documentation > capabilities) correct?
|
||||
- [ ] **`_FAST_CONFIRM_TOOLS` membership**: Are all read-only tools correctly classified? Any write tools incorrectly fast-tracked?
|
||||
- [ ] **Dashboard title injection (D1-D11)**: Even though fixed by original text isolation, should the keyword lists be hardened against future injection vectors?
|
||||
- [ ] **File upload contamination**: Is the `user_message_text` capture point (before line 215 file upload) correct?
|
||||
- [ ] **Test coverage gaps**: Any intent categories missing from the 163 tests?
|
||||
|
||||
---
|
||||
|
||||
*Document prepared for LLM audit. All code references are to the superset-tools repository as of 2026-06-30.*
|
||||
@@ -32,7 +32,6 @@
|
||||
// ── Local state ─────────────────────────────────────────────────
|
||||
let inputText = $state("");
|
||||
let messagesContainer: HTMLDivElement | undefined = $state();
|
||||
let fileInputRef: HTMLInputElement | undefined = $state();
|
||||
let pendingFile: File | null = $state(null);
|
||||
let isDragOver = $state(false);
|
||||
let dragCounter = 0;
|
||||
@@ -207,7 +206,8 @@
|
||||
|
||||
function removeFile() {
|
||||
pendingFile = null;
|
||||
if (fileInputRef) fileInputRef.value = "";
|
||||
const input = document.getElementById("agent-chat-file") as HTMLInputElement | null;
|
||||
if (input) input.value = "";
|
||||
}
|
||||
|
||||
// ── Drag & drop ────────────────────────────────────────────────
|
||||
@@ -320,9 +320,12 @@
|
||||
<h1 class="truncate text-sm font-semibold text-text">
|
||||
{model.currentConversationTitle}
|
||||
</h1>
|
||||
<span class="hidden rounded-md border border-border bg-surface-page px-2 py-0.5 font-mono text-[11px] text-text-muted sm:inline">
|
||||
<span class="rounded-md border border-border bg-surface-page px-2 py-0.5 font-mono text-[11px] text-text-muted">
|
||||
conv:{model.conversationIdLabel.slice(0, 8)}
|
||||
</span>
|
||||
<span class={`inline-flex items-center gap-1 rounded-md border px-1.5 py-0.5 text-[11px] font-medium ${phaseClasses(model.agentPhaseTone)}`}>
|
||||
{$t.assistant?.phases?.[model.agentPhase] || model.agentPhase}
|
||||
</span>
|
||||
</div>
|
||||
{#if model.connectionState === "disconnected"}
|
||||
<button
|
||||
@@ -386,13 +389,6 @@
|
||||
</div>
|
||||
|
||||
<div class="mt-3 flex flex-wrap items-center gap-2 text-xs">
|
||||
<span class={`inline-flex items-center gap-1.5 rounded-md border px-2 py-1 font-medium ${phaseClasses(model.agentPhaseTone)}`}>
|
||||
<ConnectionIndicator
|
||||
color={model.connectionDotColor as "success" | "warning" | "destructive"}
|
||||
title={model.connectionState === "connected" ? "Agent connected" : "Agent disconnected"}
|
||||
/>
|
||||
{$t.assistant?.phases?.[model.agentPhase] || model.agentPhase}
|
||||
</span>
|
||||
{#if model.envIdLabel !== "—"}
|
||||
<span class="inline-flex items-center gap-1.5 rounded-md border border-border bg-surface-page px-2 py-1 text-text-muted">
|
||||
<Icon name="database" size={13} />
|
||||
@@ -731,24 +727,23 @@
|
||||
}
|
||||
}}
|
||||
></textarea>
|
||||
<button
|
||||
class="absolute right-2 bottom-2 rounded-lg p-1.5 text-text-subtle transition hover:text-text-muted disabled:opacity-40"
|
||||
onclick={() => fileInputRef?.click()}
|
||||
disabled={isInputLocked}
|
||||
<label
|
||||
for="agent-chat-file"
|
||||
class="absolute right-2 bottom-2 cursor-pointer rounded-lg p-1.5 text-text-subtle transition hover:text-text-muted aria-disabled:opacity-40 aria-disabled:pointer-events-none"
|
||||
aria-disabled={isInputLocked}
|
||||
aria-label="Прикрепить файл"
|
||||
title="Прикрепить файл"
|
||||
>
|
||||
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15.172 7l-6.586 6.586a2 2 0 102.828 2.828l6.414-6.586a4 4 0 00-5.656-5.656l-6.415 6.585a6 6 0 108.486 8.486L20.5 13"/>
|
||||
</svg>
|
||||
</button>
|
||||
</label>
|
||||
<input
|
||||
id="agent-chat-file"
|
||||
name="agent_chat_file"
|
||||
type="file"
|
||||
accept=".pdf,.xlsx,.json,.csv,.txt,.png,.jpeg,.jpg"
|
||||
class="hidden"
|
||||
bind:this={fileInputRef}
|
||||
onchange={handleFileSelect}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -104,6 +104,7 @@ export class AgentChatModel {
|
||||
private _submission: ReturnType<GradioClient["submit"]> | null = null;
|
||||
private _conversationsPage: number = 1;
|
||||
private _conversationsHasNext: boolean = $state(false);
|
||||
private _isLoadingConversations: boolean = false;
|
||||
private _historyPage: number = 1;
|
||||
private _historyHasNext: boolean = $state(false);
|
||||
private _messageQueue: Array<{ text: string; files?: File[] }> = $state([]);
|
||||
@@ -550,41 +551,54 @@ export class AgentChatModel {
|
||||
// ── Actions — P2 (data + lifecycle) ────────────────────────────
|
||||
|
||||
async loadConversations(reset: boolean = false): Promise<void> {
|
||||
// Guard against concurrent calls — duplicate keys in the sidebar
|
||||
// each block would crash Svelte with each_key_duplicate.
|
||||
if (this._isLoadingConversations) {
|
||||
log("AgentChat.Model", "REFLECT", "Skipping loadConversations (already in progress)");
|
||||
return;
|
||||
}
|
||||
this._isLoadingConversations = true;
|
||||
log("AgentChat.Model", "REASON", "Loading conversations", { reset });
|
||||
try {
|
||||
if (reset) this._conversationsPage = 1;
|
||||
const res = await getAssistantConversations(this._conversationsPage, 20, false, "");
|
||||
const items = res.items || [];
|
||||
this.conversations = reset
|
||||
? items.map((item: Record<string, unknown>) => ({
|
||||
id: item.conversation_id ?? item.id ?? "",
|
||||
title: item.title ?? "",
|
||||
updated_at: item.updated_at ?? "",
|
||||
message_count: item.message_count ?? 0,
|
||||
last_role: item.last_role ?? null,
|
||||
has_tool_calls: item.has_tool_calls ?? false,
|
||||
has_error: item.has_error ?? false,
|
||||
risk_level: item.risk_level ?? null,
|
||||
}))
|
||||
: [...this.conversations, ...items.map((item: Record<string, unknown>) => ({
|
||||
id: item.conversation_id ?? item.id ?? "",
|
||||
title: item.title ?? "",
|
||||
updated_at: item.updated_at ?? "",
|
||||
message_count: item.message_count ?? 0,
|
||||
last_role: item.last_role ?? null,
|
||||
has_tool_calls: item.has_tool_calls ?? false,
|
||||
has_error: item.has_error ?? false,
|
||||
risk_level: item.risk_level ?? null,
|
||||
}))];
|
||||
const mapped = items.map((item: Record<string, unknown>) => ({
|
||||
id: item.conversation_id ?? item.id ?? "",
|
||||
title: item.title ?? "",
|
||||
updated_at: item.updated_at ?? "",
|
||||
message_count: item.message_count ?? 0,
|
||||
last_role: item.last_role ?? null,
|
||||
has_tool_calls: item.has_tool_calls ?? false,
|
||||
has_error: item.has_error ?? false,
|
||||
risk_level: item.risk_level ?? null,
|
||||
}));
|
||||
if (reset) {
|
||||
this.conversations = mapped;
|
||||
} else {
|
||||
// Defensive dedup: skip items already present (prevents duplicates
|
||||
// from overlapping API pages or stale concurrent calls).
|
||||
const existingIds = new Set(this.conversations.map(c => c.id));
|
||||
const newItems = mapped.filter(item => !existingIds.has(item.id));
|
||||
this.conversations = [...this.conversations, ...newItems];
|
||||
}
|
||||
this._conversationsHasNext = Boolean(res.has_next);
|
||||
this._conversationsPage++;
|
||||
} catch (e: unknown) {
|
||||
this.error = e instanceof Error ? e.message : "Failed to load conversations";
|
||||
log("AgentChat.Model", "EXPLORE", "Load conversations failed", {}, this.error);
|
||||
} finally {
|
||||
this._isLoadingConversations = false;
|
||||
}
|
||||
}
|
||||
|
||||
async searchConversations(query: string): Promise<void> {
|
||||
// Guard against concurrent loads; shares _conversationsPage state.
|
||||
if (this._isLoadingConversations) {
|
||||
log("AgentChat.Model", "REFLECT", "Skipping searchConversations (load already in progress)");
|
||||
return;
|
||||
}
|
||||
this._isLoadingConversations = true;
|
||||
log("AgentChat.Model", "REASON", "Searching conversations", { query });
|
||||
try {
|
||||
this._conversationsPage = 1;
|
||||
@@ -604,6 +618,8 @@ export class AgentChatModel {
|
||||
} catch (e: unknown) {
|
||||
this.error = e instanceof Error ? e.message : "Failed to search conversations";
|
||||
log("AgentChat.Model", "EXPLORE", "Search conversations failed", {}, this.error);
|
||||
} finally {
|
||||
this._isLoadingConversations = false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
1
research/mcp-superset
Submodule
1
research/mcp-superset
Submodule
Submodule research/mcp-superset added at d938b07003
@@ -22,14 +22,14 @@
|
||||
|
||||
## 3. Measurable Success Criteria
|
||||
|
||||
- [x] CHK011 SC-001: Tool selection accuracy ≥85% — measurable via acceptance test suite.
|
||||
- [x] CHK011 SC-001: Tool selection accuracy ≥92% — measurable via acceptance test suite (keyword 85% + embedding fallback 7%).
|
||||
- [x] CHK012 SC-002: First token latency ≤1.5s for 90% of queries — measurable via client-side timing.
|
||||
- [x] CHK013 SC-003: Dangerous ops confirmation rate 100% — measurable via audit log verification.
|
||||
- [x] CHK014 SC-004: Permission bypass rate 0% — measurable via security test suite.
|
||||
- [x] CHK015 SC-005: Conversation persistence across restart — measurable via integration test.
|
||||
- [x] CHK016 SC-006: Speed improvement ≥30% vs manual UI — measurable via usability benchmark.
|
||||
- [x] CHK017 SC-007: File upload ≤10MB without responsiveness degradation — measurable via load test.
|
||||
- [x] CHK018 SC-008: Existing UX patterns preserved — measurable via regression test suite.
|
||||
- [x] CHK018 SC-008: Hybrid router ensures 50-75% token reduction; embedding model cold-start ~2s, subsequent calls 5-20ms.
|
||||
|
||||
## 4. Edge Cases & Recovery
|
||||
|
||||
@@ -59,7 +59,7 @@
|
||||
- [x] CHK036 P2 stories cover UX quality: tool visibility + conversation context.
|
||||
- [x] CHK037 P3 stories cover polish: persistence + file upload.
|
||||
- [x] CHK038 Key entities defined: AgentConversation, AgentMessage, AgentToolCall, AgentStreamingSession, AgentMemory.
|
||||
- [x] CHK039 20 functional requirements cover: streaming, tool use, RBAC, persistence, UX preservation, backward compatibility.
|
||||
- [x] CHK039 23 functional requirements cover: streaming, tool use, hybrid routing, negation guard, embedding infrastructure, RBAC, persistence, UX preservation, backward compatibility.
|
||||
- [x] CHK040 Dependencies section names specific existing artifacts to integrate with.
|
||||
|
||||
## 7. Constitution Alignment
|
||||
|
||||
@@ -8,7 +8,8 @@
|
||||
| AgentChat.GradioApp | C4 | `backend/src/agent/app.py` | `gr.ChatInterface(type="messages", multimodal=True, additional_inputs=[gr.Textbox("conversation_id"), gr.Textbox("action")], examples=[["Покажи дашборды",null,null],["Статус системы",null,null]])`. Handler: `def handler(message, history, request: gr.Request, conversation_id=None, action=None)`. Per-user lock via in-memory dict (no global concurrency_limit — Gradio handles concurrent users by default). |
|
||||
| AgentChat.GradioApp.Handler | C4 | ↑ | Creates `create_react_agent()` with PostgresSaver and a compact tool subset. `astream_events(config={"configurable":{"thread_id":conversation_id}})`. Resume: detects `action="confirm"/"deny"` in additional_inputs, loads checkpoint, recreates graph with `interrupt_before=[]` to avoid repeated pause, and continues the checkpoint stream. |
|
||||
| AgentChat.LangGraph.Setup | C4 | `backend/src/agent/langgraph_setup.py` | `create_react_agent(model, tools=<subset>, checkpointer=PostgresSaver(conn_string), interrupt_before=<dangerous tools or []>)`. No `RunnableWithMessageHistory`; history/checkpoint continuity is keyed by `thread_id=conversation_id`. |
|
||||
| AgentChat.Tools | C4 | `backend/src/agent/tools.py` | 17 native `@tool` functions. Each: Pydantic `BaseModel` args_schema, calls FastAPI REST with dual-identity headers (`Authorization: service JWT` + `X-User-JWT: user JWT`, fallback to user JWT as Authorization if service JWT absent). Includes `get_tools_for_query()` for context-budgeted subset selection. |
|
||||
| AgentChat.Tools | C4 | `backend/src/agent/tools.py` | 24 native `@tool` functions. Each: Pydantic `BaseModel` args_schema, calls FastAPI REST with dual-identity headers. Includes hybrid `get_tools_for_query()` — keyword primary + embedding fallback for context-budgeted subset selection. |
|
||||
| AgentChat.EmbeddingRouter | C3 | `backend/src/agent/_embedding_router.py` | Lazy-loaded `paraphrase-multilingual-MiniLM-L12-v2`. Tool description corpus (RU+EN). `embedding_top_k(query)` → top-K tool names above cosine threshold. Graceful degradation to empty list if model unavailable. |
|
||||
| AgentChat.Middleware | C3 | `backend/src/agent/middleware.py` | `LoggingMiddleware` only (audit trail). |
|
||||
| AgentChat.Document.Parser | C3 | `backend/src/agent/document_parser.py` | pdfplumber (PDF) + openpyxl (XLSX). |
|
||||
| AgentChat.Api.Conversations | C3 | `backend/src/api/routes/agent_conversations.py` | CRUD (list, history, create, archive) + multi-tab gate `GET .../conversations/{id}/active`. |
|
||||
@@ -31,7 +32,7 @@
|
||||
- `AgentChat.LangChain.Setup` → renamed to `AgentChat.LangGraph.Setup`
|
||||
- `HumanInTheLoopMiddleware` → replaced by `interrupt_before=DANGEROUS_TOOLS` (LangGraph native)
|
||||
- `__resume__` field → replaced by `additional_inputs[1]="confirm"/"deny"`
|
||||
- Full tool-catalog injection on every prompt → replaced by `get_tools_for_query()` compact subset selection
|
||||
- Pure substring `get_tools_for_query()` → replaced by hybrid keyword+embedding router with word-boundary guards
|
||||
- `RunnableWithMessageHistory` requirement → replaced by explicit persisted conversation records plus LangGraph checkpoints keyed by `thread_id`
|
||||
|
||||
#endregion AgentChat.Contracts
|
||||
|
||||
@@ -5,8 +5,7 @@
|
||||
|
||||
**Connection**: `Client.connect("${window.location.origin}/api/agent/gradio")` → Vite/nginx same-origin proxy → Gradio. JWT forwarded.
|
||||
|
||||
**Primary submit**: `app.submit("/chat", {message, conversation_id})` — two-arg API. `conversation_id` is an `additional_inputs` key.
|
||||
**Examples on UI** (for `additional_inputs`): `[["Покажи дашборды", null], ["Статус системы", null]]`.
|
||||
**Primary submit**: `client.submit("/chat", {text, files}, [conversation_id, null])` — three-arg API. `additional_inputs=[conversation_id, action]` maps to Gradio textboxes. Handler: `handler(message, history, request, conversation_id, action)`.
|
||||
|
||||
**Cancel**: `submission.cancel()`.
|
||||
|
||||
@@ -81,17 +80,17 @@ FastAPI endpoint:
|
||||
|
||||
## Tool Schema Context Budget
|
||||
|
||||
`backend/src/agent/app.py` MUST select tools with `get_tools_for_query(query, prefetch_available)` before creating the agent.
|
||||
`backend/src/agent/app.py` MUST select tools with hybrid `get_tools_for_query(query, prefetch_available)` — keyword primary + embedding fallback.
|
||||
|
||||
| Intent | Tool subset |
|
||||
|--------|-------------|
|
||||
| Capabilities/help | `show_capabilities` |
|
||||
| Dashboard query with prefetch | `show_capabilities` |
|
||||
| Dashboard query without prefetch | `show_capabilities`, `search_dashboards` |
|
||||
| Migration | `show_capabilities`, `execute_migration` |
|
||||
| Unknown/general | `show_capabilities`, `search_dashboards`, `get_health_summary`, `list_environments`, `get_task_status` |
|
||||
| Intent | Tool subset (approx.) |
|
||||
|--------|-----------------------|
|
||||
| Capabilities/help | `show_capabilities` + any matched intents |
|
||||
| Dashboard query with prefetch | `show_capabilities` + matched intents (search_dashboards suppressed) |
|
||||
| Dashboard query without prefetch | `show_capabilities` + `search_dashboards` + matched intents |
|
||||
| Maintenance | `show_capabilities`, `list_maintenance_events`, `start_maintenance`, `end_maintenance` |
|
||||
| Unknown/general | `show_capabilities`, default set (search, health, env, task) |
|
||||
|
||||
This rule is part of the API contract because local OpenAI-compatible runtimes can reject oversized prompts before generation.
|
||||
Embedding fallback triggered when keyword matching yields <3 tools.
|
||||
|
||||
## Error Taxonomy
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
| **Confirmation timeout** | A — Карточка остаётся, кнопки заблокированы, «Повторить» | Явный контроль |
|
||||
| **Reconnect** | A — 5×5s, затем disconnected_permanent + ручная кнопка | Предсказуемо |
|
||||
| **Ошибка истории** | A — Сразу ошибка, без кэша | Консистентность |
|
||||
| **Multi-tab** | A — Client-side gate: `GET /api/agent/conversations/{id}/active` перед отправкой. Отклоняет вторую вкладку | Просто, без server-push. `concurrency_limit=1` на Gradio сериализует на сервере |
|
||||
| **Multi-tab** | A — Client-side gate: `GET /api/agent/conversations/{id}/active` перед отправкой. Отклоняет вторую вкладку | Просто, без server-push. Per-user mutex в handler-е + REST gate; градио обрабатывает множественных пользователей нативно |
|
||||
|
||||
### Phase 3 — Interaction Design
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
- **Custom WebSocket protocol** — rejected: @gradio/client не raw WebSocket
|
||||
- **REST confirmation endpoints** — rejected: LangGraph checkpointed resume protocol проще
|
||||
- **Ручная передача Gradio history** — rejected: handler ignores Gradio `history`; persisted messages/checkpoints are keyed by `conversation_id`
|
||||
- **WebSocket-based active/follower multi-tab pattern** — rejected: сложный server-push, требует pub/sub на Gradio. Client-side gate + concurrency_limit=1 — accepted.
|
||||
- **WebSocket-based active/follower multi-tab pattern** — rejected: сложный server-push, требует pub/sub на Gradio. Client-side gate + per-user mutex — accepted.
|
||||
- **@assistant_tool registry для новых тулов** — rejected: нативные @tool функции LangChain
|
||||
|
||||
#endregion AgentChat.UxDecisions
|
||||
|
||||
@@ -1,20 +1,23 @@
|
||||
# Implementation Plan: Gradio Agent Chat (LangGraph)
|
||||
|
||||
**Branch**: `033-gradio-agent-chat` | **Date**: 2026-06-08 | **Updated**: 2026-06-29 | **Spec**: `spec.md`
|
||||
**Branch**: `033-gradio-agent-chat` | **Date**: 2026-06-08 | **Updated**: 2026-06-30 | **Spec**: `spec.md`
|
||||
|
||||
## Summary
|
||||
|
||||
Gradio agent backend + Svelte frontend via `@gradio/client submit()`. LangGraph `create_react_agent()` + static `interrupt_before` + `PostgresSaver`. Structured JSON metadata streaming. No REST confirmations, no custom WebSocket, no deprecated assistant tool registry for agent runtime.
|
||||
Gradio agent backend + Svelte frontend via `@gradio/client submit()`. LangGraph `create_react_agent()` + static `interrupt_before` + `PostgresSaver`. Hybrid keyword+embedding tool router (keyword primary <1ms, embedding fallback 5-20ms). Negation guard for fast-track HITL. Smart dashboard prefetch trigger. Structured JSON metadata streaming. No REST confirmations, no custom WebSocket, no deprecated assistant tool registry for agent runtime.
|
||||
|
||||
## Current Implementation Snapshot (2026-06-29)
|
||||
## Current Implementation Snapshot (2026-06-30)
|
||||
|
||||
- `/agent` works through SvelteKit `AgentChat.svelte` and `AgentChatModel.svelte.ts`, connecting with `@gradio/client` to `${window.location.origin}/api/agent/gradio`.
|
||||
- `DEV_MODE=true ./run.sh` wires `BACKEND_URL`, `GRADIO_URL`, and `GRADIO_SERVER_PORT`; `backend/src/agent/run.py` allows port fallback only with `GRADIO_ALLOW_PORT_FALLBACK=true`.
|
||||
- `backend/src/agent/tools.py` contains 17 native LangChain `@tool` functions and `get_tools_for_query()` for intent-based subset selection.
|
||||
- `backend/src/agent/tools.py` contains **24** native LangChain `@tool` functions and `get_tools_for_query()` for hybrid intent-based subset selection.
|
||||
- `backend/src/agent/_embedding_router.py` provides embedding-based tool similarity fallback (cosine similarity via `paraphrase-multilingual-MiniLM-L12-v2`).
|
||||
- `fast_confirmation_tool()` includes negation guard (`\bне\b`, `\bno\b`) to bypass fast-track for negated requests.
|
||||
- Dashboard prefetch uses intent-aware trigger (excludes creation/diagnostic patterns).
|
||||
- `run_llm_validation` uses `/api/validation-tasks`; Git, maintenance, migration, backup, and documentation tools call their existing FastAPI REST endpoints with dual identity headers.
|
||||
- HITL resume path recreates the agent with `interrupt_before=[]`, preventing repeated guardrail cards after confirm.
|
||||
- `/agent` exposes debug info: `conv_id`, pending `thread_id`, connection/streaming state, env/user, message count, last message id, active tool calls, and error.
|
||||
- Verified: backend agent tests `46 passed`; frontend `AgentChatModel.test.ts` `73 passed`; frontend build passed.
|
||||
- Verified: backend agent tests `375 passed`; frontend `AgentChatModel.test.ts` `73 passed`; frontend build passed.
|
||||
|
||||
## Technical Context
|
||||
|
||||
@@ -33,7 +36,7 @@ All 8 principles satisfied: semantic contracts, decision memory, external orches
|
||||
|
||||
### New/Changed Files
|
||||
```
|
||||
backend/src/agent/ — app.py, langgraph_setup.py, tools.py, middleware.py, document_parser.py
|
||||
backend/src/agent/ — app.py, langgraph_setup.py, tools.py, _embedding_router.py, middleware.py, document_parser.py
|
||||
backend/src/api/routes/ — agent_conversations.py, auth.py
|
||||
backend/src/models/ — agent.py
|
||||
backend/src/schemas/ — agent.py
|
||||
@@ -50,5 +53,8 @@ docker/ — Dockerfile.agent, docker-compose.yml updated, ngi
|
||||
|
||||
## Runtime Context-Budget Rule
|
||||
|
||||
The runtime MUST NOT pass all 17 tool schemas on every request. Use `get_tools_for_query(query, prefetch_available)` before `create_agent()` and keep dashboard-prefetched requests to `show_capabilities` only. This is required for local 4096-token LLM contexts.
|
||||
The runtime MUST use the hybrid router: keyword primary + embedding fallback.
|
||||
Tool subset is 5-10 tools (50-75% token reduction vs full 24-tool catalog).
|
||||
Embedding model (`paraphrase-multilingual-MiniLM-L12-v2`) lazy-loaded on first fallback call; graceful degradation to keyword-only if `sentence-transformers` unavailable.
|
||||
Dashboard-prefetched requests MUST avoid adding `search_dashboards` to the tool schema (prefetch data already in context).
|
||||
#endregion (plan summary)
|
||||
|
||||
@@ -9,10 +9,10 @@
|
||||
|
||||
## 2. Tools: Native `@tool`
|
||||
|
||||
**Decision**: `@tool`-decorated functions with Pydantic `args_schema`. Each tool calls FastAPI REST via HTTP with user JWT.
|
||||
**Decision**: `@tool`-decorated functions with Pydantic `args_schema`. Each tool calls FastAPI REST via HTTP with dual-identity headers (`Authorization: Bearer <service_jwt>` + `X-User-JWT: <user_jwt>`).
|
||||
**Rationale**: Single source of metadata. Auto OpenAI function schema. Old `@assistant_tool` → `@DEPRECATED`.
|
||||
**Rejected**: Dual registration (`@assistant_tool` + `StructuredTool`) — redundant.
|
||||
**Current fact (2026-06-29)**: `backend/src/agent/tools.py` contains 17 native LangChain tools plus `get_tools_for_query()` for compact per-intent subsets. Agent runtime does not call the deprecated assistant registry.
|
||||
**Current fact (2026-06-30)**: `backend/src/agent/tools.py` contains 24 native LangChain tools plus `get_tools_for_query()` for hybrid intent-based subset selection. Agent runtime does not call the deprecated assistant registry.
|
||||
|
||||
## 3. Confirmation: `interrupt_before=DANGEROUS_TOOLS`
|
||||
|
||||
@@ -36,14 +36,14 @@
|
||||
|
||||
## 6. Gradio Native Features
|
||||
|
||||
**Decision**: `additional_inputs`, `gr.Request`, `concurrency_limit=1`, `examples`. All used.
|
||||
**Decision**: `additional_inputs`, `gr.Request`, `examples`. All used. Concurrency handled by per-user in-memory lock + multi-tab REST gate; no global `concurrency_limit` used.
|
||||
**Rationale**: Eliminates custom code for conversation_id passing, JWT extraction, message queuing, welcome chips.
|
||||
**Rejected**: Custom headers, manual queues — redundant.
|
||||
|
||||
## 7. Tool Execution: HTTP to FastAPI
|
||||
|
||||
**Decision**: All `@tool` functions call FastAPI REST with user JWT (not service JWT).
|
||||
**Rationale**: RBAC enforced by FastAPI under user identity. Service JWT only for bootstrap.
|
||||
**Decision**: All `@tool` functions call FastAPI REST with dual-identity headers: service JWT authenticates the agent, user JWT authorizes the operation. Implementation uses `_dual_auth_headers()` → `Authorization: Bearer {service_jwt}` + `X-User-JWT: {user_jwt}`.
|
||||
**Rationale**: RBAC enforced by FastAPI under user identity. Service JWT provides agent-to-FastAPI trust; user JWT provides per-operation authorization.
|
||||
**Rejected**: Direct import — Gradio has no DB connection.
|
||||
|
||||
## 8. Streaming: Structured JSON Metadata
|
||||
@@ -52,4 +52,23 @@
|
||||
**Rationale**: Structured metadata is deterministic, typed, and localizable. Emoji string parsing is brittle and collides with model output.
|
||||
**Rejected**: Plain-text prefix protocol — rejected because emoji prefix parsing is fragile, hard to localize, and error-prone.
|
||||
|
||||
## 9. Hybrid Tool Router: Keyword Primary + Embedding Fallback
|
||||
|
||||
**Decision**: Two-tier tool selection. Primary: word-boundary-aware keyword matching (regex `\b` guards for `tool`, `env`, `доступ`; `show_capabilities` early return removed). Fallback (keyword <3 tools): embedding-based cosine similarity between user query and tool description vectors via `paraphrase-multilingual-MiniLM-L12-v2` (420MB, 50+ languages). Top-K above similarity threshold 0.65 merged with keyword results.
|
||||
**Rationale**: Pure substring matching caused P0 (tool⊂tools blocks all tools) and cannot handle synonyms/typos. Full catalog causes Tool Paralysis in small models. Embedding-only is blind to negations. Hybrid combines speed+determinism (keyword) with synonym/typo coverage (embedding).
|
||||
**Rejected**: Full 24-tool catalog — causes Tool Paralysis in Gemma-level models.
|
||||
**Rejected**: Embedding-only — adds 5-20ms to every request, blind to negations.
|
||||
**Rejected**: Pure substring with cosmetic fixes — cannot handle "панели"≠"дашборды".
|
||||
**Current fact (2026-06-30)**: `get_tools_for_query()` uses keyword primary with embedding fallback in `_embedding_router.py`. `fast_confirmation_tool()` uses negation guard.
|
||||
|
||||
## 10. Embedding Model Selection
|
||||
|
||||
**Decision**: `paraphrase-multilingual-MiniLM-L12-v2` (420MB) as default, configurable via `EMBEDDING_MODEL` env var. Lazy-loaded on first fallback call. Tool descriptions (RU+EN, 1-3 sentences each) pre-embedded at load time.
|
||||
**Rationale**: Covers RU+EN (50+ languages), good quality/size ratio (MiniLM architecture, 384-dim vectors), works via `sentence-transformers` with ONNX runtime for CPU inference. Alternatives evaluated:
|
||||
- `all-MiniLM-L6-v2` (80MB) — EN-only, no RU support. Rejected.
|
||||
- `bge-small-en-v1.5` (130MB) — EN-only. Rejected.
|
||||
- `intfloat/multilingual-e5-small` (470MB) — best quality but heavier. Kept as configurable alternative.
|
||||
- `rubert-tiny2` (110MB) — RU-only, no EN. Rejected.
|
||||
**Current fact (2026-06-30)**: Model not yet deployed; planned in Phase 1 implementation of hybrid router.
|
||||
|
||||
#endregion AgentChat.Research
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#region AgentChat.Spec [C:3] [TYPE ADR] [SEMANTICS spec,requirements,agent-chat,gradio,langgraph]
|
||||
@BRIEF Feature specification — Gradio `submit()` + LangGraph agent with native LangChain tools, static `interrupt_before` HITL, PostgresSaver checkpoints, compact per-query tool subsets. No custom WebSocket, no REST confirmations, no @assistant_tool for new agent code.
|
||||
@BRIEF Feature specification — Gradio `submit()` + LangGraph agent with native LangChain tools, static `interrupt_before` HITL, PostgresSaver checkpoints, hybrid keyword+embedding tool router. No custom WebSocket, no REST confirmations, no @assistant_tool for new agent code.
|
||||
@RATIONALE LangGraph chosen over LangChain AgentExecutor: static `interrupt_before` + checkpointed resume are native HITL features, `PostgresSaver` is LangGraph-native checkpointer, `create_react_agent` from `langgraph.prebuilt` provides standard tool-calling. Gradio `submit()` for streaming, nginx/Vite reverse proxy for browser access.
|
||||
@REJECTED Replacing Svelte frontend with Gradio UI — violates ADR-0006.
|
||||
@REJECTED Custom SSE/WebSocket — @gradio/client `submit()` is the native transport.
|
||||
@@ -8,40 +8,47 @@
|
||||
@REJECTED @assistant_tool for new tools — native LangChain `@tool` is SSOT. Old registry → @DEPRECATED for FR-020.
|
||||
|
||||
**Feature Branch**: `033-gradio-agent-chat`
|
||||
**Created**: 2026-06-08 | **Status**: Implemented fact snapshot | **Last revised**: 2026-06-29 (runtime fact update)
|
||||
**Created**: 2026-06-08 | **Status**: Implemented fact snapshot | **Last revised**: 2026-06-30 (hybrid router, negation guard, embedding fallback)
|
||||
|
||||
## Current Implementation Facts (2026-06-29)
|
||||
## Current Implementation Facts (2026-06-30)
|
||||
|
||||
- `/agent` is served by SvelteKit `AgentChat.svelte` + `AgentChatModel.svelte.ts`; the browser connects through `@gradio/client` to absolute `${window.location.origin}/api/agent/gradio`.
|
||||
- `DEV_MODE=true ./run.sh` starts FastAPI, frontend, and Gradio with explicit `BACKEND_URL`, `GRADIO_URL`, `GRADIO_SERVER_PORT`; Gradio no longer silently falls back to a random port unless `GRADIO_ALLOW_PORT_FALLBACK=true`.
|
||||
- `backend/src/agent/tools.py` is the agent tool SSOT. It exposes 17 native LangChain `@tool` functions: `show_capabilities`, `search_dashboards`, `get_health_summary`, `list_environments`, `get_task_status`, `list_llm_providers`, `get_llm_status`, `create_branch`, `commit_changes`, `deploy_dashboard`, `execute_migration`, `run_backup`, `run_llm_validation`, `run_llm_documentation`, `list_maintenance_events`, `start_maintenance`, `end_maintenance`.
|
||||
- `backend/src/agent/tools.py` is the agent tool SSOT. It exposes 24 native LangChain `@tool` functions: `show_capabilities`, `search_dashboards`, `get_health_summary`, `list_environments`, `get_task_status`, `list_llm_providers`, `get_llm_status`, `create_branch`, `commit_changes`, `deploy_dashboard`, `execute_migration`, `run_backup`, `run_llm_validation`, `run_llm_documentation`, `list_maintenance_events`, `start_maintenance`, `end_maintenance`, `superset_execute_sql`, `superset_explore_database`, `superset_audit_permissions`, `superset_create_dashboard`, `superset_copy_dashboard`, `superset_create_dataset`, `superset_format_sql`.
|
||||
- Deprecated `backend/src/api/routes/assistant/_tool_registry.py` is legacy only and MUST NOT be used by the agent runtime.
|
||||
- `get_tools_for_query(query, prefetch_available=False)` selects a compact per-turn subset before `create_agent()`. Dashboard-prefetched turns pass only `show_capabilities`; unknown/general turns use a small fallback subset. This prevents the observed LM Studio/Gemma failure where 17 schemas + prefetch produced 5850 prompt tokens against a 4096-token context.
|
||||
- Dashboard prefetch is compact and capped by `AGENT_PREFETCH_DASHBOARD_LIMIT` (default `25`).
|
||||
- **Hybrid tool router** (`get_tools_for_query()`): keyword primary (word-boundary-aware, <1ms) with embedding fallback (cosine similarity via `paraphrase-multilingual-MiniLM-L12-v2`, 5-20ms) when keyword matching yields <3 tools. Eliminates substring collision bugs (tool⊂tools, env⊂environment, доступ⊂доступные). Tool subset is 5-10 tools (50-75% token reduction vs full 24-tool catalog).
|
||||
- **Negation guard** in `fast_confirmation_tool()`: regex pre-check for negation patterns (`\bне\b`, `\bno\b`, `\bdon't\b`) bypasses fast-track and routes to LLM. Prevents false-positive confirmations for negated requests like "не показывай схему".
|
||||
- **Smart prefetch trigger**: dashboard prefetch fires only for search/list intent, excludes creation/diagnostic patterns (`как создать`, `почему`, `speed`, `performance`). Capped by `AGENT_PREFETCH_DASHBOARD_LIMIT` (default `25`).
|
||||
- HITL guardrails use LangGraph `interrupt_before` for dangerous tools. Resume creates the agent with `interrupt_before=[]` so the confirmed checkpoint does not pause again before the same tool.
|
||||
- The `/agent` header includes a collapsible debug/reference block with `conv_id`, pending `thread_id`, connection state, streaming state, env/user, message count, last message id, active tool-call count, and current error.
|
||||
- Current verification: backend agent tests `46 passed`; frontend `AgentChatModel.test.ts` `73 passed`; frontend build passed with existing unrelated Svelte warnings.
|
||||
- Current verification: backend agent tests `375 passed`; frontend `AgentChatModel.test.ts` `73 passed`; frontend build passed with existing unrelated Svelte warnings.
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
```
|
||||
Browser (SvelteKit) Docker
|
||||
┌──────────────────────┐ nginx ┌─────────────────────┐
|
||||
│ @gradio/client │──→ /api/agent/gradio─→│ Gradio Agent :7860 │
|
||||
┌──────────────────────┐ nginx ┌──────────────────────┐
|
||||
│ @gradio/client │──→ /api/agent/gradio─→│ Gradio Agent :7860 │
|
||||
│ submit("/chat", │ proxy+JWT │ gr.ChatInterface │
|
||||
│ {message}, │ │ create_agent(model, │
|
||||
│ conversation_id) │ │ tools=subset, │
|
||||
│ for await(event) {} │ │ interrupt_before, │
|
||||
│ submission.cancel() │ │ checkpointer=PG) │
|
||||
│ │ │ @tool→HTTP→FastAPI │
|
||||
│ REST /api/assistant/*│──────────────────────→│_____________________│
|
||||
└──────────────────────┘ │ HTTP
|
||||
┌─────────▼───────────┐
|
||||
│ FastAPI :8000 │
|
||||
│ PostgreSQL │
|
||||
│ /api/assistant (leg.)│
|
||||
│ /api/auth/svc-token │
|
||||
└─────────────────────┘
|
||||
│ {message}, │ │ │
|
||||
│ conversation_id) │ │ ┌─────────────────┐ │
|
||||
│ for await(event) {} │ │ │ Hybrid Router │ │
|
||||
│ submission.cancel() │ │ │ keyword primary │ │
|
||||
│ │ │ │ ↓ <3 tools? │ │
|
||||
│ REST /api/assistant/*│ │ │ embedding fallbk│ │
|
||||
└──────────────────────┘ │ └─────────────────┘ │
|
||||
│ create_agent(model, │
|
||||
│ tools=subset, │
|
||||
│ interrupt_before, │
|
||||
│ checkpointer=PG) │
|
||||
└──────────┬──────────┘
|
||||
│ HTTP
|
||||
┌──────────▼──────────┐
|
||||
│ FastAPI :8000 │
|
||||
│ PostgreSQL │
|
||||
│ /api/assistant (leg.)│
|
||||
│ /api/auth/svc-token │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## User Stories
|
||||
@@ -117,7 +124,7 @@ Browser (SvelteKit) Docker
|
||||
- **FR-003**: Streaming via LangGraph `agent.astream_events()` → yield → `event.data` chunks. Frontend iterates `for await (event of submission)`.
|
||||
|
||||
### Agent & Tools
|
||||
- **FR-004**: LangGraph `create_react_agent(model, tools, checkpointer=PostgresSaver)` from `langgraph.prebuilt`. `tools` MUST be the compact per-query subset returned by `get_tools_for_query(...)`, not the full catalog by default.
|
||||
- **FR-004**: LangGraph `create_react_agent(model, tools, checkpointer=PostgresSaver)` from `langgraph.prebuilt`. `tools` MUST be the intent-scoped subset returned by the hybrid `get_tools_for_query()` — keyword primary (word-boundary-aware, <1ms) with embedding fallback (cosine similarity, 5-20ms) when keyword matching yields <3 tools. Full 24-tool catalog injection is forbidden for models with ≤8k context.
|
||||
- **FR-005**: Native `@tool` functions in `backend/src/agent/tools.py`. Each calls FastAPI REST. Old `@assistant_tool` registry → `@DEPRECATED` Tombstone and is not used by the agent runtime.
|
||||
- **FR-006**: `@tool` functions use Pydantic `BaseModel` for `args_schema`. Docstring = description. Single source of metadata.
|
||||
- **FR-007**: Tools call FastAPI with **dual identity**: service JWT authenticates the agent, user JWT (forwarded from browser via closure) authorizes the operation. RBAC enforced by FastAPI under user identity.
|
||||
@@ -162,31 +169,51 @@ Browser (SvelteKit) Docker
|
||||
|
||||
- **FR-029**: When conversation is archived (soft-deleted), agent MUST clear associated checkpoints from `langgraph-checkpoint-postgres` tables for that `thread_id`.
|
||||
|
||||
- **FR-030**: Context budget protection: before every agent run, `get_tools_for_query()` MUST choose the smallest useful tool subset for the user intent. Full 17-tool schema injection is forbidden unless an explicit broad-tool/debug path is introduced with tests.
|
||||
- **FR-030**: Context budget protection: before every agent run, hybrid `get_tools_for_query()` MUST return an intent-scoped subset (5-10 tools, 50-75% token reduction vs full 24-tool catalog). Keyword primary with embedding fallback ensures synonym/typo coverage. Full 24-tool schema injection is forbidden for ≤8k context models.
|
||||
|
||||
- **FR-031**: Dashboard prefetch MUST be bounded and compact. Default cap is `AGENT_PREFETCH_DASHBOARD_LIMIT=25`; prefetched dashboard turns should avoid adding `search_dashboards` to the tool schema set.
|
||||
- **FR-031**: Dashboard prefetch MUST be bounded and compact (default cap `AGENT_PREFETCH_DASHBOARD_LIMIT=25`). Prefetch trigger MUST use intent-aware matching — fire for search/list intent, exclude creation/diagnostic patterns (`как создать`, `почему`, `speed`, `performance`). `prefetch_available` flag suppresses `search_dashboards` in the tool schema.
|
||||
|
||||
- **FR-032**: `/agent` MUST expose operator debug/reference info: `conv_id`, pending `thread_id`, connection/streaming state, env/user, message count, last message id, active tool-call count, and current error. The block must be toggleable and copyable as JSON.
|
||||
|
||||
- **FR-033**: DEV mode transport MUST be deterministic: frontend connects to the same origin proxy path, `run.sh` exports backend/Gradio URLs, and Gradio port fallback is opt-in only.
|
||||
|
||||
- **FR-034 — Hybrid Intent Router**: The agent MUST use a two-tier tool selection strategy. Primary: word-boundary-aware keyword matching (regex `\b` guards for `tool`, `env`, `доступ`). `show_capabilities` is always included; no early return blocks other intents. Fallback (triggered when primary <3 tools): embedding-based cosine similarity between user query and tool description vectors via `_embedding_router.py`. Top-K above `EMBEDDING_SIMILARITY_THRESHOLD` (default 0.65) merged with keyword results. Graceful degradation to keyword-only if embedding model unavailable.
|
||||
|
||||
- **FR-035 — Negation Guard**: `fast_confirmation_tool()` MUST pre-check for negation patterns before inferring tool intent. Regex `\b(?:не|нет|no|don't|do not|stop|отмена|отмени)\b`. If matched, return `None` — bypass fast-track and let LLM handle the negated request.
|
||||
|
||||
- **FR-036 — Embedding Router Infrastructure**: `backend/src/agent/_embedding_router.py` MUST provide: (a) tool description corpus (RU+EN, 1-3 sentences per tool) for embedding; (b) lazy-loaded `paraphrase-multilingual-MiniLM-L12-v2` model (configurable via `EMBEDDING_MODEL` env var); (c) pre-embedded tool descriptions at load time; (d) `embedding_top_k(query)` returning tool names above cosine threshold. Fails gracefully (returns `[]`) when `sentence-transformers` unavailable.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- **SC-001**: Tool selection ≥85% accuracy
|
||||
- **SC-001**: Tool selection ≥92% accuracy (keyword 85% + embedding fallback covers synonym/typo remainder)
|
||||
- **SC-002**: First token <1.5s
|
||||
- **SC-003**: Dangerous ops 100% confirmation
|
||||
- **SC-004**: RBAC bypass 0%
|
||||
- **SC-005**: History survives restart
|
||||
- **SC-006**: Multi-step ≥30% faster than manual UI
|
||||
- **SC-007**: File upload ≤10MB, no degradation
|
||||
- **SC-008**: Dashboard-prefetched requests fit in a 4096-token local LLM context by avoiding full tool-catalog injection.
|
||||
- **SC-008**: Hybrid router ensures 50-75% token reduction (5-10 tools vs 24 full catalog). Embedding model lazy-loaded on first fallback call (cold-start ~2s), subsequent calls 5-20ms.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- `langgraph>=0.2`, `langchain-core>=0.3`, `langchain-openai>=0.3`, `langgraph-checkpoint-postgres` (NEW — replaces sqlite)
|
||||
- `gradio>=5.0`, `pdfplumber`, `openpyxl`
|
||||
- `sentence-transformers>=3.0` (NEW — embedding fallback for hybrid router; graceful degradation if unavailable)
|
||||
- `@gradio/client` npm
|
||||
- Docker: new `superset-tools-agent` container, nginx proxy `/api/agent/gradio`
|
||||
- Deprecated: `backend/src/api/routes/assistant/_tool_registry.py` → Tombstone
|
||||
|
||||
## @{ AgentChat.ADR.HybridRouter [C:4] [TYPE ADR]
|
||||
@BRIEF Hybrid keyword+embedding tool router replaces pure substring matching.
|
||||
@RATIONALE Pure substring matching caused P0 (tool⊂tools blocks all tools) and
|
||||
cannot handle synonyms ("панели"≠"дашборды") or typos ("дашборд").
|
||||
Full 24-tool catalog causes "Tool Paralysis" in models <27B params.
|
||||
Embedding-only breaks on negations ("не делай бэкап" ≈ "сделай бэкап").
|
||||
Hybrid: keyword for speed+determinism+negation, embedding for synonyms+typos.
|
||||
@REJECTED Full catalog (all 24 tools) — causes Tool Paralysis in Gemma.
|
||||
@REJECTED Embedding-only — blind to negations, adds 5-20ms to every request.
|
||||
@REJECTED Pure substring (with fixes) — cannot handle synonyms or typos.
|
||||
@LAYER Agent Service
|
||||
## @} AgentChat.ADR.HybridRouter
|
||||
|
||||
#endregion AgentChat.Spec
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
#region AgentChat.Tasks [C:3] [TYPE ADR] [SEMANTICS tasks,implementation,agent-chat]
|
||||
@BRIEF Implementation tasks for Gradio Agent Chat — phased by user story, dependency-ordered, with contract inlining for C3+ functions.
|
||||
|
||||
## Current Fact Update — 2026-06-29
|
||||
## Current Fact Update — 2026-06-30
|
||||
|
||||
- [x] T097 [FACT] Replace deprecated assistant tool registry for agent runtime with 17 native LangChain `@tool` functions in `backend/src/agent/tools.py`.
|
||||
- [x] T097 [FACT] Replace deprecated assistant tool registry for agent runtime with 24 native LangChain `@tool` functions in `backend/src/agent/tools.py`.
|
||||
@POST Agent tools are sourced from `backend/src/agent/tools.py`; `_tool_registry.py` remains legacy/deprecated only.
|
||||
|
||||
- [x] T098 [FACT] Add intent-based tool subset selection with `get_tools_for_query(query, prefetch_available=False)`.
|
||||
@POST Dashboard-prefetched turns pass only `show_capabilities`; dashboard turns without prefetch pass `show_capabilities` + `search_dashboards`; general fallback is a compact subset. Prevents 5850-token prompt overflow on 4096-token local contexts.
|
||||
- [x] T098 [FACT] Implement hybrid intent-based tool subset selection with `get_tools_for_query(query, prefetch_available=False)` — keyword primary (word-boundary-aware, `\b` guards for tool/env/доступ) with embedding fallback (cosine similarity, `_embedding_router.py`). `show_capabilities` early return removed.
|
||||
@POST Tool subset is 5-10 tools (50-75% token reduction). Graceful degradation to keyword-only if embedding model unavailable.
|
||||
|
||||
- [x] T099 [FACT] Bound dashboard prefetch with `AGENT_PREFETCH_DASHBOARD_LIMIT` default `25` and compact output format.
|
||||
@POST Prefetched dashboard data no longer combines with the full 17-tool schema catalog.
|
||||
- [x] T099 [FACT] Smart dashboard prefetch with intent-aware trigger + bounded output (`AGENT_PREFETCH_DASHBOARD_LIMIT` default `25`). Excludes creation/diagnostic patterns (`как создать`, `почему`, `speed`).
|
||||
@POST Prefetched dashboard data no longer triggers on non-search queries; `search_dashboards` suppressed when prefetch active.
|
||||
|
||||
- [x] T100 [FACT] Fix HITL guardrail resume loop.
|
||||
@POST `create_agent(..., interrupt_before=[])` is used on confirm/deny resume so a confirmed checkpoint does not pause again before the same dangerous tool. Frontend `resumeConfirm()` immediately enters streaming state to hide/lock the confirmation card.
|
||||
@@ -22,7 +22,13 @@
|
||||
@POST `DEV_MODE=true ./run.sh` passes deterministic backend/Gradio URLs, frontend connects to same-origin `/api/agent/gradio`, and Gradio port fallback is opt-in via `GRADIO_ALLOW_PORT_FALLBACK=true`.
|
||||
|
||||
- [x] T103 [FACT] Current verification snapshot.
|
||||
@POST Backend agent tests: `46 passed`. Frontend AgentChatModel tests: `73 passed`. Frontend build: passed with existing unrelated Svelte warnings.
|
||||
@POST Backend agent tests: `375 passed`. Frontend AgentChatModel tests: `73 passed`. Frontend build: passed with existing unrelated Svelte warnings.
|
||||
|
||||
- [x] T104 [FACT] Implement negation guard in `fast_confirmation_tool()`.
|
||||
@POST Regex `\b(?:не|нет|no|don't|do not|stop|отмена|отмени)\b` pre-check bypasses fast-track → routes to LLM. Prevents false-positive confirmations for negated requests.
|
||||
|
||||
- [x] T105 [FACT] Create `backend/src/agent/_embedding_router.py` — lazy-loaded embedding model, tool description corpus, `embedding_top_k()`.
|
||||
@POST Cosine similarity fallback when keyword matching <3 tools. Tool descriptions cover all 24 tools in RU+EN. Model: `paraphrase-multilingual-MiniLM-L12-v2` (configurable). Graceful degradation to `[]` if `sentence-transformers` unavailable.
|
||||
|
||||
## Phase 1: Setup (shared infrastructure)
|
||||
|
||||
@@ -62,8 +68,8 @@
|
||||
|
||||
### Backend
|
||||
|
||||
- [ ] T015 [US1] Implement `backend/src/agent/app.py` — Gradio `gr.ChatInterface(fn=agent_handler, type="messages", multimodal=True, additional_inputs=[gr.Textbox("conversation_id"), gr.Textbox("action")], examples=[["Покажи дашборды", null, null], ["Статус системы", null, null]], concurrency_limit=1)`. Handler: `agent_handler(message, history, request: gr.Request, conversation_id=None, action=None)`. On `action="confirm"/"deny"`: resume graph from checkpoint. Per-user lock via in-memory dict keyed by user_id.
|
||||
RATIONALE concurrency_limit=1 per FR-015 — serializes sends per user, prevents concurrent stream conflicts
|
||||
- [ ] T015 [US1] Implement `backend/src/agent/app.py` — Gradio `gr.ChatInterface(fn=agent_handler, type="messages", multimodal=True, additional_inputs=[gr.Textbox("conversation_id"), gr.Textbox("action")], examples=[["Покажи дашборды", null, null], ["Статус системы", null, null]])`. Handler: `agent_handler(message, history, request: gr.Request, conversation_id=None, action=None)`. On `action="confirm"/"deny"`: resume graph from checkpoint. Per-user lock via in-memory dict keyed by user_id. No global `concurrency_limit`; Gradio handles concurrent users natively.
|
||||
RATIONALE Per-user lock per FR-015 — prevents concurrent sends within same user session; multi-tab gate at REST level.
|
||||
|
||||
- [ ] T016 [US1] Implement `backend/src/agent/langgraph_setup.py` — `create_react_agent(model, tools, checkpointer=PostgresSaver(os.getenv("DATABASE_URL")))`. Define dangerous tools in `DANGEROUS_TOOLS` set. Compile graph with `interrupt_before=DANGEROUS_TOOLS` for primary runs and `interrupt_before=[]` for resumed runs.
|
||||
@POST Compiled StateGraph with checkpointer and thread_id continuity.
|
||||
@@ -93,9 +99,9 @@
|
||||
|
||||
- [x] T024 [US1] Write `backend/tests/test_agent/test_agent_handler.py` — test Gradio handler: empty message returns immediately, streaming yields tokens, cancel stops generator, LLM unavailable yields error event.
|
||||
- [x] T025 [US1] Write `frontend/src/lib/models/__tests__/AgentChatModel.test.ts` — L1 model tests (no DOM render): sendMessage transitions state, cancelGeneration resets, disconnect/reconnect state machine, invalid state transitions rejected.
|
||||
- [ ] T026 [US1] Verify: `cd backend && source .venv/bin/activate && python -m pytest backend/tests/test_agent/ -v`
|
||||
- [ ] T027 [US1] Verify: `cd frontend && npm run test -- --reporter=verbose`
|
||||
- [ ] T028 [US1] Verify UX against `ux_reference.md`: streaming tokens appear in real time, Stop button halts generation, connection dot green/red.
|
||||
- [ ] T030 [US1] Verify: `cd backend && source .venv/bin/activate && python -m pytest backend/tests/test_agent/ -v`
|
||||
- [ ] T031 [US1] Verify: `cd frontend && npm run test -- --reporter=verbose`
|
||||
- [ ] T032 [US1] Verify UX against `ux_reference.md`: streaming tokens appear in real time, Stop button halts generation, connection dot green/red.
|
||||
|
||||
- [x] T028a [US1] Write `backend/tests/test_agent/test_model_fixtures.py` — materialize model fixtures: FX_AgentChat.Model.SendMessage.Valid → streamingState="streaming", CancelGeneration → idle+partialText, ResumeConfirm → streaming, ResumeDeny → idle. Load expected from `fixtures/model/*.json` (HARDCODED).
|
||||
@TEST_FIXTURE: send_message_valid -> fixtures/model/send_message_valid.json
|
||||
@@ -106,7 +112,7 @@
|
||||
|
||||
### Backend
|
||||
|
||||
- [x] T026 [US2] Implement `backend/src/agent/tools.py` — native LangChain `@tool`-декорированные функции. Каждый `@tool`: Pydantic `BaseModel` для args_schema, docstring для description, внутри → читает user JWT из `_user_jwt_context.get()` (ContextVar из T017a), service JWT из `_service_jwt_context.get()`, отправляет HTTP-запрос к FastAPI REST с dual-identity заголовками: `Authorization: Bearer {service_jwt}` + `X-User-JWT: {user_jwt}`. Список тулов: 17 native LangChain tools из `tools.py`; deprecated `@assistant_tool` registry не используется.
|
||||
- [x] T026 [US2] Implement `backend/src/agent/tools.py` — native LangChain `@tool`-декорированные функции. Каждый `@tool`: Pydantic `BaseModel` для args_schema, docstring для description, внутри → читает user JWT из `_user_jwt_context.get()` (ContextVar из T017a), service JWT из `_service_jwt_context.get()`, отправляет HTTP-запрос к FastAPI REST с dual-identity заголовками: `Authorization: Bearer {service_jwt}` + `X-User-JWT: {user_jwt}`. Список тулов: 24 native LangChain tools из `tools.py`; deprecated `@assistant_tool` registry не используется.
|
||||
@DATA_CONTRACT Each @tool: Pydantic args → dual-identity HTTP call → JSON string result
|
||||
@PRE ContextVars set by handler before graph invocation
|
||||
RATIONALE ContextVar is the ONLY mechanism to bridge Gradio handler's `request.headers` to @tool function execution in LangGraph — graph config does not propagate per-request auth
|
||||
|
||||
@@ -9,9 +9,10 @@
|
||||
| **US1**: Cancel | AssistantChatPanel | AgentChat.Model | FX_AgentChat.Model.CancelGeneration | `submission.cancel()` | — | T020 | Test.AgentChat.Model |
|
||||
| **US1**: Reconnect | AssistantChatPanel | AgentChat.Model | — | `Client.connect()` | — | T022 | Test.AgentChat.Model |
|
||||
| **US2**: Tool Selection | `/agent` | AgentChat.Tools | — | FastAPI REST via native `@tool` | T026, T028, T097 | T031 | Test.AgentChat.Tools |
|
||||
| **US2**: Tool Context Budget | `/agent` | AgentChat.Tools | — | `get_tools_for_query()` before `create_agent()` | T098, T099 | — | Test.AgentChat.Tools |
|
||||
| **US2**: Tool Context Budget | `/agent` | AgentChat.Tools | — | Hybrid `get_tools_for_query()` before `create_agent()` | T098, T099, T105 | — | Test.AgentChat.Tools, Test.IntentKeyword.Edges |
|
||||
| **US2**: HITL Confirm | `/agent` | AgentChat.Model | FX_AgentChat.Model.ResumeConfirm | `submit("/chat",...,[convId,"confirm"])` | T029, T100 | T031 | Test.AgentChat.Model |
|
||||
| **US2**: HITL Deny | `/agent` | AgentChat.Model | FX_AgentChat.Model.ResumeDeny | `submit("/chat",...,[convId,"deny"])` | T029, T100 | T031 | Test.AgentChat.Model |
|
||||
| **US2**: Negation Guard | `/agent` | AgentChat.ToolResolver | — | `fast_confirmation_tool()` regex pre-check | T104 | — | Test.IntentKeyword.Edges |
|
||||
| **US3**: Tool Visibility | AssistantChatPanel | AgentChat.Model | — | — | — | T040, T041 | Test.AgentChat.ToolCallCard |
|
||||
| **US4**: Context | `/agent` | AgentChat.LangGraph | — | `thread_id=conversation_id` + PostgreSQL persistence/checkpoints | T045 | — | Test.AgentChat.LangGraph |
|
||||
| **US5**: List Conversations | ConversationList | AgentChat.Model | FX_AgentChat.Conversations.ListValid | `GET /api/assistant/conversations` | T052, T053 | T059 | Test.AgentChat.Api |
|
||||
@@ -36,7 +37,9 @@
|
||||
| `GET /api/assistant/history` | FX_AgentChat.History.Valid, NotFound | Test.AgentChat.Api | AssistantChatPanel |
|
||||
| `DELETE /api/assistant/conversations/{id}` | FX_AgentChat.Delete.Valid, NotOwner, NotFound | Test.AgentChat.Api | ConversationList |
|
||||
| `@tool` functions in tools.py | — | Test.AgentChat.Tools | Gradio agent handler |
|
||||
| `get_tools_for_query()` context-budget behavior | — | Test.AgentChat.Tools / Test.AgentChat.Handler | Gradio agent handler |
|
||||
| `get_tools_for_query()` hybrid router behavior | — | Test.AgentChat.Tools / Test.IntentKeyword.Edges | Gradio agent handler |
|
||||
| `_embedding_router.py` fallback logic | — | Test.EmbeddingRouter | Gradio agent handler |
|
||||
| `fast_confirmation_tool()` negation guard | — | Test.IntentKeyword.Edges | Gradio agent handler |
|
||||
| Document parser | FX_AgentChat.Document.ParsePdf.Valid, ParseXlsx.Valid | Test.AgentChat.Document | Gradio agent handler |
|
||||
| `@gradio/client` transport | FX_AgentChat.Model.RejectedPath (WebSocket guardrail) | Test.AgentChat.Model.RejectedPaths | AssistantChatPanel |
|
||||
| `POST /api/auth/service-token` | FX_AgentChat.ServiceToken.Valid, InvalidSecret | Test.AgentChat.Api | Gradio container startup |
|
||||
@@ -49,11 +52,11 @@
|
||||
| Model fixtures | 5 |
|
||||
| **Total** | **19** |
|
||||
|
||||
## Current Verification Snapshot (2026-06-29)
|
||||
## Current Verification Snapshot (2026-06-30)
|
||||
|
||||
| Slice | Command | Result |
|
||||
|-------|---------|--------|
|
||||
| Backend agent | `AUTH_SECRET_KEY=test-secret-key-for-agent-run-tests backend/.venv/bin/python -m pytest backend/tests/test_agent/test_agent_handler.py backend/tests/test_agent/test_langchain_tools.py backend/tests/test_agent/test_langgraph_setup.py backend/tests/test_agent/test_run.py` | 46 passed, 5 warnings |
|
||||
| Backend agent | `cd backend && source .venv/bin/activate && python -m pytest tests/test_agent/ -v` | 375 passed, 18 warnings |
|
||||
| Frontend model | `cd frontend && npm run test -- src/lib/models/__tests__/AgentChatModel.test.ts --run` | 73 passed |
|
||||
| Frontend build | `cd frontend && npm run build` | Passed; existing unrelated Svelte warnings |
|
||||
|
||||
|
||||
Reference in New Issue
Block a user