# backend/tests/agent/test_confirmation.py # #region Test.AgentChat.Confirmation [C:3] [TYPE Module] [SEMANTICS test,agent,hitl,confirmation,integration] # @BRIEF Integration tests for _confirmation.py — handle_resume fast-path, LLM formatting, # build_confirmation_contract, metadata, and title race-condition coverage. # @RELATION BINDS_TO -> [AgentChat.Confirmation] from pathlib import Path import sys sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) import json import asyncio from unittest.mock import AsyncMock, MagicMock, patch import pytest # ── Helpers ───────────────────────────────────────────────────────── def _json_chunks(chunks: list[str]) -> list[dict]: """Parse all JSON chunk strings into dicts.""" return [json.loads(c) for c in chunks] async def _collect(generator): """Collect all items from an async generator into a list.""" return [item async for item in generator] def _make_fake_llm_chunk(content: str): """Create a fake ChatOpenAI chunk with .content.""" chunk = MagicMock() chunk.content = content return chunk # ── Fixtures ──────────────────────────────────────────────────────── @pytest.fixture(autouse=True) def clear_pending(): """Clear _pending_confirmations before each test.""" from src.agent._confirmation import _pending_confirmations _pending_confirmations.clear() # ═══════════════════════════════════════════════════════════════════ # build_confirmation_contract # ═══════════════════════════════════════════════════════════════════ # #region test_build_confirmation_contract [C:2] [TYPE Class] # @BRIEF Test build_confirmation_contract for all risk levels. class TestBuildConfirmationContract: def test_safe_tool_returns_read_contract(self): from src.agent._confirmation import build_confirmation_contract c = build_confirmation_contract("list_environments") assert c["risk"] == "read" assert c["risk_level"] == "safe" assert c["prompt"] == "Разрешить чтение данных?" assert c["requires_confirmation"] is True def test_dangerous_tool_returns_write_contract(self): from src.agent._confirmation import build_confirmation_contract # deploy_dashboard starts with "deploy" → guarded risk level in v1 contract c = build_confirmation_contract("deploy_dashboard") assert c["risk"] == "write" assert c["risk_level"] == "guarded" assert c["prompt"] == "Подтвердить изменение данных?" def test_guarded_tool_returns_write_contract(self): from src.agent._confirmation import build_confirmation_contract c = build_confirmation_contract("start_maintenance") assert c["risk"] == "write" assert c["risk_level"] == "guarded" assert c["prompt"] == "Подтвердить изменение данных?" def test_unknown_tool_returns_unknown_contract(self): from src.agent._confirmation import build_confirmation_contract c = build_confirmation_contract("some_unknown_tool") assert c["risk"] == "read" assert c["risk_level"] == "safe" assert c["prompt"] == "Разрешить чтение данных?" def test_none_tool_name_returns_unknown(self): from src.agent._confirmation import build_confirmation_contract c = build_confirmation_contract(None) assert c["operation"] == "unknown_action" assert c["risk"] == "read" assert c["risk_level"] == "safe" # #endregion test_build_confirmation_contract # ═══════════════════════════════════════════════════════════════════ # confirmation_metadata_for_tool # ═══════════════════════════════════════════════════════════════════ # #region test_confirmation_metadata_for_tool [C:2] [TYPE Class] # @BRIEF Test confirmation_metadata_for_tool output shape. class TestConfirmationMetadataForTool: def test_includes_all_required_fields(self): from src.agent._confirmation import confirmation_metadata_for_tool meta = confirmation_metadata_for_tool("conv-42", "list_environments", {"env_id": "prod"}) assert meta["type"] == "confirm_required" assert meta["thread_id"] == "conv-42" assert meta["tool_name"] == "list_environments" assert meta["tool_args"] == {"env_id": "prod"} assert meta["risk"] == "read" assert meta["risk_level"] == "safe" assert "intent" in meta assert meta["intent"]["operation"] == "list_environments" def test_empty_args_defaults_to_empty_dict(self): from src.agent._confirmation import confirmation_metadata_for_tool meta = confirmation_metadata_for_tool("conv-1", "list_environments", None) assert meta["tool_args"] == {} def test_no_args_passed_defaults_to_empty_dict(self): from src.agent._confirmation import confirmation_metadata_for_tool meta = confirmation_metadata_for_tool("conv-1", "list_environments") assert meta["tool_args"] == {} # #endregion test_confirmation_metadata_for_tool # ═══════════════════════════════════════════════════════════════════ # _format_tool_output_via_llm # ═══════════════════════════════════════════════════════════════════ # #region test_format_tool_output [C:3] [TYPE Class] # @BRIEF Integration tests for _format_tool_output_via_llm — LLM path and fallbacks. class TestFormatToolOutput: @pytest.mark.asyncio async def test_empty_output_yields_placeholder(self): from src.agent._confirmation import _format_tool_output_via_llm chunks = await _collect(_format_tool_output_via_llm("test_tool", "")) data = _json_chunks(chunks) assert len(data) == 1 assert "нет данных" in data[0]["content"] @pytest.mark.asyncio async def test_whitespace_only_yields_placeholder(self): from src.agent._confirmation import _format_tool_output_via_llm chunks = await _collect(_format_tool_output_via_llm("test_tool", " ")) data = _json_chunks(chunks) assert len(data) == 1 assert "нет данных" in data[0]["content"] @pytest.mark.asyncio async def test_llm_formatting_streams_tokens(self): """When LLM config exists, output is streamed through ChatOpenAI.""" from src.agent._confirmation import _format_tool_output_via_llm fake_llm = MagicMock() fake_llm.astream = MagicMock(return_value=_make_async_iter([ _make_fake_llm_chunk("Доступно"), _make_fake_llm_chunk(" три"), _make_fake_llm_chunk(" окружения."), ])) with patch("src.agent.langgraph_setup._fetch_llm_config") as mock_cfg, \ patch("src.agent._confirmation.ChatOpenAI", return_value=fake_llm): mock_cfg.return_value = { "configured": True, "api_key": "sk-test", "default_model": "gpt-4o-mini", "base_url": "https://api.test.com/v1", } chunks = await _collect( _format_tool_output_via_llm("list_environments", '[{"id":"ss-dev"}]') ) data = _json_chunks(chunks) assert len(data) == 3 assert data[0]["content"] == "Доступно" assert data[1]["content"] == " три" assert data[2]["content"] == " окружения." for d in data: assert d["metadata"]["type"] == "stream_token" @pytest.mark.asyncio async def test_fallback_when_llm_unavailable(self): """When fetch_llm_config returns None, fall back to prettified JSON.""" from src.agent._confirmation import _format_tool_output_via_llm raw = '[{"id":"ss-dev","name":"Dev"}]' with patch("src.agent.langgraph_setup._fetch_llm_config", return_value=None): chunks = await _collect(_format_tool_output_via_llm("list_environments", raw)) data = _json_chunks(chunks) assert len(data) == 1 # Should be prettified JSON (indent=2) assert ' "' in data[0]["content"] assert "ss-dev" in data[0]["content"] assert data[0]["metadata"]["type"] == "stream_token" @pytest.mark.asyncio async def test_fallback_when_config_not_configured(self): """When config exists but configured=False, fall back to prettified JSON.""" from src.agent._confirmation import _format_tool_output_via_llm raw = '{"key": "value"}' with patch("src.agent.langgraph_setup._fetch_llm_config", return_value={"configured": False}): chunks = await _collect(_format_tool_output_via_llm("some_tool", raw)) data = _json_chunks(chunks) assert len(data) == 1 assert '"key"' in data[0]["content"] @pytest.mark.asyncio async def test_fallback_on_llm_exception(self): """When ChatOpenAI raises, fall back to prettified JSON.""" from src.agent._confirmation import _format_tool_output_via_llm raw = '[{"a": 1}]' with patch("src.agent.langgraph_setup._fetch_llm_config") as mock_cfg, \ patch("src.agent._confirmation.ChatOpenAI") as mock_llm_cls: mock_cfg.return_value = { "configured": True, "api_key": "sk-test", "default_model": "gpt-4o-mini", } mock_llm_cls.side_effect = RuntimeError("LLM connection refused") chunks = await _collect(_format_tool_output_via_llm("test_tool", raw)) data = _json_chunks(chunks) assert len(data) == 1 assert "a" in data[0]["content"] @pytest.mark.asyncio async def test_non_json_output_passed_through_raw(self): """Non-JSON output (plain text) is yielded as-is in fallback.""" from src.agent._confirmation import _format_tool_output_via_llm raw = "Operation completed successfully" with patch("src.agent.langgraph_setup._fetch_llm_config", return_value=None): chunks = await _collect(_format_tool_output_via_llm("test_tool", raw)) data = _json_chunks(chunks) assert len(data) == 1 assert data[0]["content"] == raw # #endregion test_format_tool_output # ═══════════════════════════════════════════════════════════════════ # #region test_handle_resume_integration [C:3] [TYPE Class] # @BRIEF Integration tests for handle_resume — fast-path confirm/deny, error paths, # LLM formatting integration, and title race-condition coverage. class TestHandleResumeIntegration: @pytest.mark.asyncio async def test_deny_yields_cancelled(self): from src.agent._confirmation import handle_resume, _pending_confirmations _pending_confirmations["conv-deny"] = { "tool_name": "list_environments", "tool_args": {}, } chunks = await _collect(handle_resume("conv-deny", "deny")) data = _json_chunks(chunks) assert len(data) == 1 assert data[0]["metadata"]["type"] == "confirm_resolved" assert data[0]["metadata"]["result"] == "denied" assert "отменена" in data[0]["content"].lower() # Pending is popped assert "conv-deny" not in _pending_confirmations @pytest.mark.asyncio async def test_confirm_executes_tool_and_formats_via_llm(self): from src.agent._confirmation import handle_resume, _pending_confirmations tool = MagicMock() tool.ainvoke = AsyncMock(return_value='[{"id":"ss-dev","name":"Dev"}]') _pending_confirmations["conv-confirm"] = { "tool_name": "list_environments", "tool_args": {}, } async def collect(): with patch("src.agent._confirmation.find_tool", return_value=tool), \ patch("src.agent._confirmation._format_tool_output_via_llm") as mock_fmt: async def _fake_fmt(tool_name, output): yield json.dumps({ "content": f"SUMMARY: {tool_name} -> {output[:20]}", "metadata": {"type": "stream_token", "token": "X"}, }) mock_fmt.side_effect = _fake_fmt chunks = [c async for c in handle_resume("conv-confirm", "confirm")] return chunks chunks = await collect() data = _json_chunks(chunks) types = [d["metadata"]["type"] for d in data] assert "confirm_resolved" in types assert "tool_start" in types assert "tool_end" in types assert "stream_token" in types # Check tool_end has the output summary tool_end = [d for d in data if d["metadata"]["type"] == "tool_end"] assert len(tool_end) == 1 assert tool_end[0]["metadata"]["output"]["result"].startswith('[{"id":"ss-dev"') # Check LLM formatting chunk stream_tokens = [d for d in data if d["metadata"]["type"] == "stream_token"] assert len(stream_tokens) >= 1 assert "SUMMARY" in stream_tokens[0]["content"] # Pending is popped assert "conv-confirm" not in _pending_confirmations @pytest.mark.asyncio async def test_confirm_unknown_tool_yields_error(self): from src.agent._confirmation import handle_resume, _pending_confirmations _pending_confirmations["conv-unknown"] = { "tool_name": "nonexistent_tool_xyz", "tool_args": {}, } with patch("src.agent._confirmation.find_tool", return_value=None): chunks = await _collect(handle_resume("conv-unknown", "confirm")) data = _json_chunks(chunks) types = [d["metadata"]["type"] for d in data] assert "confirm_resolved" in types assert "tool_start" in types assert "tool_error" in types assert "stream_token" not in types, "No LLM output on tool error" error_chunk = [d for d in data if d["metadata"]["type"] == "tool_error"] assert len(error_chunk) == 1 assert "Unknown tool" in error_chunk[0]["metadata"]["error"] @pytest.mark.asyncio async def test_confirm_tool_invocation_failure_yields_error(self): from src.agent._confirmation import handle_resume, _pending_confirmations tool = MagicMock() tool.ainvoke = AsyncMock(side_effect=RuntimeError("API timeout")) _pending_confirmations["conv-fail"] = { "tool_name": "list_environments", "tool_args": {}, } with patch("src.agent._confirmation.find_tool", return_value=tool): chunks = await _collect(handle_resume("conv-fail", "confirm")) data = _json_chunks(chunks) types = [d["metadata"]["type"] for d in data] assert "tool_error" in types assert "tool_end" not in types, "No tool_end on failure" assert "stream_token" not in types, "No LLM output on failure" error_chunk = [d for d in data if d["metadata"]["type"] == "tool_error"] assert len(error_chunk) == 1 assert "API timeout" in error_chunk[0]["metadata"]["error"] @pytest.mark.asyncio async def test_confirm_with_no_pending_falls_to_langgraph(self): """When no pending confirmation exists, handle_resume should fall through to the LangGraph checkpoint resume path.""" from src.agent._confirmation import handle_resume mock_agent = MagicMock() with patch("src.agent._confirmation.create_agent", return_value=mock_agent), \ patch("src.agent._confirmation.get_all_tools", return_value=[]): chunks = await _collect(handle_resume("conv-no-pending", "confirm")) data = _json_chunks(chunks) assert len(data) == 1 assert data[0]["metadata"]["type"] == "confirm_resolved" assert data[0]["metadata"]["result"] == "confirmed" @pytest.mark.asyncio async def test_deny_with_no_pending_returns_denied(self): """When no pending confirmation exists, deny also goes through the LangGraph checkpoint path.""" from src.agent._confirmation import handle_resume mock_agent = MagicMock() with patch("src.agent._confirmation.create_agent", return_value=mock_agent), \ patch("src.agent._confirmation.get_all_tools", return_value=[]): chunks = await _collect(handle_resume("conv-no-pending", "deny")) data = _json_chunks(chunks) assert len(data) == 1 assert data[0]["metadata"]["type"] == "confirm_resolved" assert data[0]["metadata"]["result"] == "denied" @pytest.mark.asyncio async def test_confirm_streams_langgraph_events_when_no_pending(self): """When no pending and LangGraph agent streams events, they are forwarded.""" from src.agent._confirmation import handle_resume mock_chunk = MagicMock() mock_chunk.content = "Hello from checkpoint" mock_agent = MagicMock() mock_agent.astream_events = MagicMock(return_value=_make_async_iter([ {"event": "on_chat_model_stream", "data": {"chunk": mock_chunk}}, {"event": "on_tool_start", "name": "some_tool", "data": {"input": {}}}, {"event": "on_tool_end", "name": "some_tool", "data": {"output": "ok"}}, ])) with patch("src.agent._confirmation.create_agent", return_value=mock_agent), \ patch("src.agent._confirmation.get_all_tools", return_value=[]): chunks = await _collect(handle_resume("conv-stream", "confirm")) data = _json_chunks(chunks) types = [d["metadata"]["type"] for d in data] assert "confirm_resolved" in types assert "stream_token" in types assert "tool_start" in types assert "tool_end" in types # #endregion test_handle_resume_integration # ═══════════════════════════════════════════════════════════════════ # Title race-condition coverage # ═══════════════════════════════════════════════════════════════════ # #region test_title_race_condition [C:2] [TYPE Class] # @BRIEF Verify that agent_handler captures tool_name BEFORE handle_resume pops it, # so the conversation title is descriptive (e.g. "✅ list_environments"), not # the fallback "HITL: confirm". class TestTitleRaceCondition: @pytest.mark.asyncio async def test_tool_name_read_before_pop(self): """Simulate the exact flow from agent_handler: capture tool_name from _pending_confirmations BEFORE calling handle_resume (which pops it).""" from src.agent._confirmation import _pending_confirmations conv_id = "title-race-test" _pending_confirmations[conv_id] = { "tool_name": "list_environments", "tool_args": {}, } # Step 1: Capture BEFORE pop (this is what the fix does) pending_pre = _pending_confirmations.get(conv_id, {}) tool_name = pending_pre.get("tool_name", "") if pending_pre else "" assert tool_name == "list_environments", ( "tool_name must be readable BEFORE handle_resume pops it" ) # Step 2: Pop (simulating handle_resume's internal pop) _pending_confirmations.pop(conv_id, None) # Step 3: After pop, accessing _pending_confirmations would give empty pending_post = _pending_confirmations.get(conv_id, {}) assert pending_post == {}, "After pop, pending should be gone" # But tool_name was already captured — title should be "✅ list_environments" title = f"✅ {tool_name}" if tool_name else "HITL: confirm" assert title == "✅ list_environments", ( f"Title should use captured tool_name, got: {title}" ) @pytest.mark.asyncio async def test_race_condition_without_fix_would_fallback(self): """Demonstrate the bug: if tool_name is read AFTER pop, it's lost.""" from src.agent._confirmation import _pending_confirmations conv_id = "race-bug-test" _pending_confirmations[conv_id] = { "tool_name": "get_health_summary", "tool_args": {}, } # Pop first (bad order — this is what the old code did) _pending_confirmations.pop(conv_id, None) # THEN try to read tool_name (too late!) pending_post = _pending_confirmations.get(conv_id, {}) tool_name = pending_post.get("tool_name", "") if pending_post else "" assert tool_name == "", "tool_name should be empty after pop — this IS the bug" title = f"✅ {tool_name}" if tool_name else "HITL: confirm" assert title == "HITL: confirm", "Without the fix, title falls back to generic" # #endregion test_title_race_condition # ── Helper for async iter ───────────────────────────────────────── def _make_async_iter(items): """Create an async iterator from a list.""" class AsyncIter: def __init__(self, items): self._iter = iter(items) def __aiter__(self): return self async def __anext__(self): try: return next(self._iter) except StopIteration: raise StopAsyncIteration return AsyncIter(items) # #endregion Test.AgentChat.Confirmation