491 lines
22 KiB
Python
491 lines
22 KiB
Python
# backend/tests/agent/test_confirmation.py
|
|
# #region Test.AgentChat.Confirmation [C:3] [TYPE Module] [SEMANTICS test,agent,hitl,confirmation,integration]
|
|
# @BRIEF Integration tests for _confirmation.py — handle_resume fast-path, LLM formatting,
|
|
# build_confirmation_contract, metadata, and title race-condition coverage.
|
|
# @RELATION BINDS_TO -> [AgentChat.Confirmation]
|
|
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
|
|
|
|
import json
|
|
import asyncio
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
import pytest
|
|
|
|
|
|
# ── Helpers ─────────────────────────────────────────────────────────
|
|
|
|
def _json_chunks(chunks: list[str]) -> list[dict]:
|
|
"""Parse all JSON chunk strings into dicts."""
|
|
return [json.loads(c) for c in chunks]
|
|
|
|
|
|
async def _collect(generator):
|
|
"""Collect all items from an async generator into a list."""
|
|
return [item async for item in generator]
|
|
|
|
|
|
def _make_fake_llm_chunk(content: str):
|
|
"""Create a fake ChatOpenAI chunk with .content."""
|
|
chunk = MagicMock()
|
|
chunk.content = content
|
|
return chunk
|
|
|
|
|
|
# ── Fixtures ────────────────────────────────────────────────────────
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def clear_pending():
|
|
"""Clear _pending_confirmations before each test."""
|
|
from src.agent._confirmation import _pending_confirmations
|
|
_pending_confirmations.clear()
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
# build_confirmation_contract
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
|
|
# #region test_build_confirmation_contract [C:2] [TYPE Class]
|
|
# @BRIEF Test build_confirmation_contract for all risk levels.
|
|
class TestBuildConfirmationContract:
|
|
def test_safe_tool_returns_read_contract(self):
|
|
from src.agent._confirmation import build_confirmation_contract
|
|
c = build_confirmation_contract("list_environments")
|
|
assert c["risk"] == "read"
|
|
assert c["risk_level"] == "safe"
|
|
assert c["prompt"] == "Разрешить чтение данных?"
|
|
assert c["requires_confirmation"] is True
|
|
|
|
def test_dangerous_tool_returns_write_contract(self):
|
|
from src.agent._confirmation import build_confirmation_contract
|
|
# deploy_dashboard starts with "deploy" → guarded risk level in v1 contract
|
|
c = build_confirmation_contract("deploy_dashboard")
|
|
assert c["risk"] == "write"
|
|
assert c["risk_level"] == "guarded"
|
|
assert c["prompt"] == "Подтвердить изменение данных?"
|
|
|
|
def test_guarded_tool_returns_write_contract(self):
|
|
from src.agent._confirmation import build_confirmation_contract
|
|
c = build_confirmation_contract("start_maintenance")
|
|
assert c["risk"] == "write"
|
|
assert c["risk_level"] == "guarded"
|
|
assert c["prompt"] == "Подтвердить изменение данных?"
|
|
|
|
def test_unknown_tool_returns_unknown_contract(self):
|
|
from src.agent._confirmation import build_confirmation_contract
|
|
c = build_confirmation_contract("some_unknown_tool")
|
|
assert c["risk"] == "read"
|
|
assert c["risk_level"] == "safe"
|
|
assert c["prompt"] == "Разрешить чтение данных?"
|
|
|
|
def test_none_tool_name_returns_unknown(self):
|
|
from src.agent._confirmation import build_confirmation_contract
|
|
c = build_confirmation_contract(None)
|
|
assert c["operation"] == "unknown_action"
|
|
assert c["risk"] == "read"
|
|
assert c["risk_level"] == "safe"
|
|
# #endregion test_build_confirmation_contract
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
# confirmation_metadata_for_tool
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
|
|
# #region test_confirmation_metadata_for_tool [C:2] [TYPE Class]
|
|
# @BRIEF Test confirmation_metadata_for_tool output shape.
|
|
class TestConfirmationMetadataForTool:
|
|
def test_includes_all_required_fields(self):
|
|
from src.agent._confirmation import confirmation_metadata_for_tool
|
|
meta = confirmation_metadata_for_tool("conv-42", "list_environments", {"env_id": "prod"})
|
|
assert meta["type"] == "confirm_required"
|
|
assert meta["thread_id"] == "conv-42"
|
|
assert meta["tool_name"] == "list_environments"
|
|
assert meta["tool_args"] == {"env_id": "prod"}
|
|
assert meta["risk"] == "read"
|
|
assert meta["risk_level"] == "safe"
|
|
assert "intent" in meta
|
|
assert meta["intent"]["operation"] == "list_environments"
|
|
|
|
def test_empty_args_defaults_to_empty_dict(self):
|
|
from src.agent._confirmation import confirmation_metadata_for_tool
|
|
meta = confirmation_metadata_for_tool("conv-1", "list_environments", None)
|
|
assert meta["tool_args"] == {}
|
|
|
|
def test_no_args_passed_defaults_to_empty_dict(self):
|
|
from src.agent._confirmation import confirmation_metadata_for_tool
|
|
meta = confirmation_metadata_for_tool("conv-1", "list_environments")
|
|
assert meta["tool_args"] == {}
|
|
# #endregion test_confirmation_metadata_for_tool
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
# _format_tool_output_via_llm
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
|
|
# #region test_format_tool_output [C:3] [TYPE Class]
|
|
# @BRIEF Integration tests for _format_tool_output_via_llm — LLM path and fallbacks.
|
|
class TestFormatToolOutput:
|
|
@pytest.mark.asyncio
|
|
async def test_empty_output_yields_placeholder(self):
|
|
from src.agent._confirmation import _format_tool_output_via_llm
|
|
chunks = await _collect(_format_tool_output_via_llm("test_tool", ""))
|
|
data = _json_chunks(chunks)
|
|
assert len(data) == 1
|
|
assert "нет данных" in data[0]["content"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_whitespace_only_yields_placeholder(self):
|
|
from src.agent._confirmation import _format_tool_output_via_llm
|
|
chunks = await _collect(_format_tool_output_via_llm("test_tool", " "))
|
|
data = _json_chunks(chunks)
|
|
assert len(data) == 1
|
|
assert "нет данных" in data[0]["content"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_formatting_streams_tokens(self):
|
|
"""When LLM config exists, output is streamed through ChatOpenAI."""
|
|
from src.agent._confirmation import _format_tool_output_via_llm
|
|
|
|
fake_llm = MagicMock()
|
|
fake_llm.astream = MagicMock(return_value=_make_async_iter([
|
|
_make_fake_llm_chunk("Доступно"),
|
|
_make_fake_llm_chunk(" три"),
|
|
_make_fake_llm_chunk(" окружения."),
|
|
]))
|
|
|
|
with patch("src.agent.langgraph_setup._fetch_llm_config") as mock_cfg, \
|
|
patch("src.agent._confirmation.ChatOpenAI", return_value=fake_llm):
|
|
mock_cfg.return_value = {
|
|
"configured": True,
|
|
"api_key": "sk-test",
|
|
"default_model": "gpt-4o-mini",
|
|
"base_url": "https://api.test.com/v1",
|
|
}
|
|
chunks = await _collect(
|
|
_format_tool_output_via_llm("list_environments", '[{"id":"ss-dev"}]')
|
|
)
|
|
data = _json_chunks(chunks)
|
|
assert len(data) == 3
|
|
assert data[0]["content"] == "Доступно"
|
|
assert data[1]["content"] == " три"
|
|
assert data[2]["content"] == " окружения."
|
|
for d in data:
|
|
assert d["metadata"]["type"] == "stream_token"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fallback_when_llm_unavailable(self):
|
|
"""When fetch_llm_config returns None, fall back to prettified JSON."""
|
|
from src.agent._confirmation import _format_tool_output_via_llm
|
|
|
|
raw = '[{"id":"ss-dev","name":"Dev"}]'
|
|
with patch("src.agent.langgraph_setup._fetch_llm_config", return_value=None):
|
|
chunks = await _collect(_format_tool_output_via_llm("list_environments", raw))
|
|
data = _json_chunks(chunks)
|
|
assert len(data) == 1
|
|
# Should be prettified JSON (indent=2)
|
|
assert ' "' in data[0]["content"]
|
|
assert "ss-dev" in data[0]["content"]
|
|
assert data[0]["metadata"]["type"] == "stream_token"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fallback_when_config_not_configured(self):
|
|
"""When config exists but configured=False, fall back to prettified JSON."""
|
|
from src.agent._confirmation import _format_tool_output_via_llm
|
|
|
|
raw = '{"key": "value"}'
|
|
with patch("src.agent.langgraph_setup._fetch_llm_config",
|
|
return_value={"configured": False}):
|
|
chunks = await _collect(_format_tool_output_via_llm("some_tool", raw))
|
|
data = _json_chunks(chunks)
|
|
assert len(data) == 1
|
|
assert '"key"' in data[0]["content"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fallback_on_llm_exception(self):
|
|
"""When ChatOpenAI raises, fall back to prettified JSON."""
|
|
from src.agent._confirmation import _format_tool_output_via_llm
|
|
|
|
raw = '[{"a": 1}]'
|
|
with patch("src.agent.langgraph_setup._fetch_llm_config") as mock_cfg, \
|
|
patch("src.agent._confirmation.ChatOpenAI") as mock_llm_cls:
|
|
mock_cfg.return_value = {
|
|
"configured": True,
|
|
"api_key": "sk-test",
|
|
"default_model": "gpt-4o-mini",
|
|
}
|
|
mock_llm_cls.side_effect = RuntimeError("LLM connection refused")
|
|
chunks = await _collect(_format_tool_output_via_llm("test_tool", raw))
|
|
data = _json_chunks(chunks)
|
|
assert len(data) == 1
|
|
assert "a" in data[0]["content"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_non_json_output_passed_through_raw(self):
|
|
"""Non-JSON output (plain text) is yielded as-is in fallback."""
|
|
from src.agent._confirmation import _format_tool_output_via_llm
|
|
|
|
raw = "Operation completed successfully"
|
|
with patch("src.agent.langgraph_setup._fetch_llm_config", return_value=None):
|
|
chunks = await _collect(_format_tool_output_via_llm("test_tool", raw))
|
|
data = _json_chunks(chunks)
|
|
assert len(data) == 1
|
|
assert data[0]["content"] == raw
|
|
# #endregion test_format_tool_output
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
|
|
# #region test_handle_resume_integration [C:3] [TYPE Class]
|
|
# @BRIEF Integration tests for handle_resume — fast-path confirm/deny, error paths,
|
|
# LLM formatting integration, and title race-condition coverage.
|
|
class TestHandleResumeIntegration:
|
|
@pytest.mark.asyncio
|
|
async def test_deny_yields_cancelled(self):
|
|
from src.agent._confirmation import handle_resume, _pending_confirmations
|
|
_pending_confirmations["conv-deny"] = {
|
|
"tool_name": "list_environments",
|
|
"tool_args": {},
|
|
}
|
|
chunks = await _collect(handle_resume("conv-deny", "deny"))
|
|
data = _json_chunks(chunks)
|
|
assert len(data) == 1
|
|
assert data[0]["metadata"]["type"] == "confirm_resolved"
|
|
assert data[0]["metadata"]["result"] == "denied"
|
|
assert "отменена" in data[0]["content"].lower()
|
|
# Pending is popped
|
|
assert "conv-deny" not in _pending_confirmations
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_confirm_executes_tool_and_formats_via_llm(self):
|
|
from src.agent._confirmation import handle_resume, _pending_confirmations
|
|
|
|
tool = MagicMock()
|
|
tool.ainvoke = AsyncMock(return_value='[{"id":"ss-dev","name":"Dev"}]')
|
|
_pending_confirmations["conv-confirm"] = {
|
|
"tool_name": "list_environments",
|
|
"tool_args": {},
|
|
}
|
|
|
|
async def collect():
|
|
with patch("src.agent._confirmation.find_tool", return_value=tool), \
|
|
patch("src.agent._confirmation._format_tool_output_via_llm") as mock_fmt:
|
|
async def _fake_fmt(tool_name, output):
|
|
yield json.dumps({
|
|
"content": f"SUMMARY: {tool_name} -> {output[:20]}",
|
|
"metadata": {"type": "stream_token", "token": "X"},
|
|
})
|
|
mock_fmt.side_effect = _fake_fmt
|
|
chunks = [c async for c in handle_resume("conv-confirm", "confirm")]
|
|
return chunks
|
|
|
|
chunks = await collect()
|
|
data = _json_chunks(chunks)
|
|
|
|
types = [d["metadata"]["type"] for d in data]
|
|
assert "confirm_resolved" in types
|
|
assert "tool_start" in types
|
|
assert "tool_end" in types
|
|
assert "stream_token" in types
|
|
|
|
# Check tool_end has the output summary
|
|
tool_end = [d for d in data if d["metadata"]["type"] == "tool_end"]
|
|
assert len(tool_end) == 1
|
|
assert tool_end[0]["metadata"]["output"]["result"].startswith('[{"id":"ss-dev"')
|
|
|
|
# Check LLM formatting chunk
|
|
stream_tokens = [d for d in data if d["metadata"]["type"] == "stream_token"]
|
|
assert len(stream_tokens) >= 1
|
|
assert "SUMMARY" in stream_tokens[0]["content"]
|
|
|
|
# Pending is popped
|
|
assert "conv-confirm" not in _pending_confirmations
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_confirm_unknown_tool_yields_error(self):
|
|
from src.agent._confirmation import handle_resume, _pending_confirmations
|
|
|
|
_pending_confirmations["conv-unknown"] = {
|
|
"tool_name": "nonexistent_tool_xyz",
|
|
"tool_args": {},
|
|
}
|
|
|
|
with patch("src.agent._confirmation.find_tool", return_value=None):
|
|
chunks = await _collect(handle_resume("conv-unknown", "confirm"))
|
|
data = _json_chunks(chunks)
|
|
|
|
types = [d["metadata"]["type"] for d in data]
|
|
assert "confirm_resolved" in types
|
|
assert "tool_start" in types
|
|
assert "tool_error" in types
|
|
assert "stream_token" not in types, "No LLM output on tool error"
|
|
|
|
error_chunk = [d for d in data if d["metadata"]["type"] == "tool_error"]
|
|
assert len(error_chunk) == 1
|
|
assert "Unknown tool" in error_chunk[0]["metadata"]["error"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_confirm_tool_invocation_failure_yields_error(self):
|
|
from src.agent._confirmation import handle_resume, _pending_confirmations
|
|
|
|
tool = MagicMock()
|
|
tool.ainvoke = AsyncMock(side_effect=RuntimeError("API timeout"))
|
|
_pending_confirmations["conv-fail"] = {
|
|
"tool_name": "list_environments",
|
|
"tool_args": {},
|
|
}
|
|
|
|
with patch("src.agent._confirmation.find_tool", return_value=tool):
|
|
chunks = await _collect(handle_resume("conv-fail", "confirm"))
|
|
data = _json_chunks(chunks)
|
|
|
|
types = [d["metadata"]["type"] for d in data]
|
|
assert "tool_error" in types
|
|
assert "tool_end" not in types, "No tool_end on failure"
|
|
assert "stream_token" not in types, "No LLM output on failure"
|
|
|
|
error_chunk = [d for d in data if d["metadata"]["type"] == "tool_error"]
|
|
assert len(error_chunk) == 1
|
|
assert "API timeout" in error_chunk[0]["metadata"]["error"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_confirm_with_no_pending_falls_to_langgraph(self):
|
|
"""When no pending confirmation exists, handle_resume should fall through
|
|
to the LangGraph checkpoint resume path."""
|
|
from src.agent._confirmation import handle_resume
|
|
|
|
mock_agent = MagicMock()
|
|
with patch("src.agent._confirmation.create_agent", return_value=mock_agent), \
|
|
patch("src.agent._confirmation.get_all_tools", return_value=[]):
|
|
chunks = await _collect(handle_resume("conv-no-pending", "confirm"))
|
|
data = _json_chunks(chunks)
|
|
assert len(data) == 1
|
|
assert data[0]["metadata"]["type"] == "confirm_resolved"
|
|
assert data[0]["metadata"]["result"] == "confirmed"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_deny_with_no_pending_returns_denied(self):
|
|
"""When no pending confirmation exists, deny also goes through
|
|
the LangGraph checkpoint path."""
|
|
from src.agent._confirmation import handle_resume
|
|
|
|
mock_agent = MagicMock()
|
|
with patch("src.agent._confirmation.create_agent", return_value=mock_agent), \
|
|
patch("src.agent._confirmation.get_all_tools", return_value=[]):
|
|
chunks = await _collect(handle_resume("conv-no-pending", "deny"))
|
|
data = _json_chunks(chunks)
|
|
assert len(data) == 1
|
|
assert data[0]["metadata"]["type"] == "confirm_resolved"
|
|
assert data[0]["metadata"]["result"] == "denied"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_confirm_streams_langgraph_events_when_no_pending(self):
|
|
"""When no pending and LangGraph agent streams events, they are forwarded."""
|
|
from src.agent._confirmation import handle_resume
|
|
|
|
mock_chunk = MagicMock()
|
|
mock_chunk.content = "Hello from checkpoint"
|
|
mock_agent = MagicMock()
|
|
mock_agent.astream_events = MagicMock(return_value=_make_async_iter([
|
|
{"event": "on_chat_model_stream", "data": {"chunk": mock_chunk}},
|
|
{"event": "on_tool_start", "name": "some_tool", "data": {"input": {}}},
|
|
{"event": "on_tool_end", "name": "some_tool", "data": {"output": "ok"}},
|
|
]))
|
|
|
|
with patch("src.agent._confirmation.create_agent", return_value=mock_agent), \
|
|
patch("src.agent._confirmation.get_all_tools", return_value=[]):
|
|
chunks = await _collect(handle_resume("conv-stream", "confirm"))
|
|
data = _json_chunks(chunks)
|
|
|
|
types = [d["metadata"]["type"] for d in data]
|
|
assert "confirm_resolved" in types
|
|
assert "stream_token" in types
|
|
assert "tool_start" in types
|
|
assert "tool_end" in types
|
|
# #endregion test_handle_resume_integration
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
# Title race-condition coverage
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
|
|
# #region test_title_race_condition [C:2] [TYPE Class]
|
|
# @BRIEF Verify that agent_handler captures tool_name BEFORE handle_resume pops it,
|
|
# so the conversation title is descriptive (e.g. "✅ list_environments"), not
|
|
# the fallback "HITL: confirm".
|
|
class TestTitleRaceCondition:
|
|
@pytest.mark.asyncio
|
|
async def test_tool_name_read_before_pop(self):
|
|
"""Simulate the exact flow from agent_handler: capture tool_name from
|
|
_pending_confirmations BEFORE calling handle_resume (which pops it)."""
|
|
from src.agent._confirmation import _pending_confirmations
|
|
|
|
conv_id = "title-race-test"
|
|
_pending_confirmations[conv_id] = {
|
|
"tool_name": "list_environments",
|
|
"tool_args": {},
|
|
}
|
|
|
|
# Step 1: Capture BEFORE pop (this is what the fix does)
|
|
pending_pre = _pending_confirmations.get(conv_id, {})
|
|
tool_name = pending_pre.get("tool_name", "") if pending_pre else ""
|
|
|
|
assert tool_name == "list_environments", (
|
|
"tool_name must be readable BEFORE handle_resume pops it"
|
|
)
|
|
|
|
# Step 2: Pop (simulating handle_resume's internal pop)
|
|
_pending_confirmations.pop(conv_id, None)
|
|
|
|
# Step 3: After pop, accessing _pending_confirmations would give empty
|
|
pending_post = _pending_confirmations.get(conv_id, {})
|
|
assert pending_post == {}, "After pop, pending should be gone"
|
|
|
|
# But tool_name was already captured — title should be "✅ list_environments"
|
|
title = f"✅ {tool_name}" if tool_name else "HITL: confirm"
|
|
assert title == "✅ list_environments", (
|
|
f"Title should use captured tool_name, got: {title}"
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_race_condition_without_fix_would_fallback(self):
|
|
"""Demonstrate the bug: if tool_name is read AFTER pop, it's lost."""
|
|
from src.agent._confirmation import _pending_confirmations
|
|
|
|
conv_id = "race-bug-test"
|
|
_pending_confirmations[conv_id] = {
|
|
"tool_name": "get_health_summary",
|
|
"tool_args": {},
|
|
}
|
|
|
|
# Pop first (bad order — this is what the old code did)
|
|
_pending_confirmations.pop(conv_id, None)
|
|
|
|
# THEN try to read tool_name (too late!)
|
|
pending_post = _pending_confirmations.get(conv_id, {})
|
|
tool_name = pending_post.get("tool_name", "") if pending_post else ""
|
|
|
|
assert tool_name == "", "tool_name should be empty after pop — this IS the bug"
|
|
title = f"✅ {tool_name}" if tool_name else "HITL: confirm"
|
|
assert title == "HITL: confirm", "Without the fix, title falls back to generic"
|
|
# #endregion test_title_race_condition
|
|
|
|
|
|
# ── Helper for async iter ─────────────────────────────────────────
|
|
|
|
def _make_async_iter(items):
|
|
"""Create an async iterator from a list."""
|
|
class AsyncIter:
|
|
def __init__(self, items):
|
|
self._iter = iter(items)
|
|
def __aiter__(self):
|
|
return self
|
|
async def __anext__(self):
|
|
try:
|
|
return next(self._iter)
|
|
except StopIteration:
|
|
raise StopAsyncIteration
|
|
return AsyncIter(items)
|
|
|
|
|
|
# #endregion Test.AgentChat.Confirmation
|