179 lines
7.6 KiB
Python
179 lines
7.6 KiB
Python
# #region Test.AgentChat.ToolRetry [C:3] [TYPE Module] [SEMANTICS test,agent,tools,retry]
|
|
# @BRIEF Contract tests for _retry_read_tool — fixed-delay retry on transient errors.
|
|
# @RELATION BINDS_TO -> [AgentChat.Tools.Retry]
|
|
# @TEST_EDGE: first_attempt_502 -> Auto-retries once, succeeds.
|
|
# @TEST_EDGE: both_attempts_502 -> Raises original error.
|
|
# @TEST_EDGE: write_tool_502 -> No retry, raises immediately.
|
|
# @TEST_EDGE: connect_error -> Retries on ConnectError too.
|
|
# @TEST_EDGE: read_timeout -> Retries on ReadTimeout too.
|
|
|
|
import pytest
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
import httpx
|
|
|
|
from src.agent.tools import (
|
|
_execute_with_timeout,
|
|
_retry_read_tool,
|
|
drain_tool_retry_events,
|
|
start_tool_retry_event_buffer,
|
|
)
|
|
|
|
# ── Shared fixtures ──────────────────────────────────────────────────
|
|
|
|
def _make_502_error() -> httpx.HTTPStatusError:
|
|
"""Build a synthetic 502 HTTPStatusError for consistent test usage."""
|
|
request = httpx.Request("GET", "http://test.local/api/test")
|
|
response = httpx.Response(502, request=request)
|
|
return httpx.HTTPStatusError("Bad Gateway", request=request, response=response)
|
|
|
|
|
|
def _make_connect_error() -> httpx.ConnectError:
|
|
"""Build a synthetic ConnectError for transient-connectivity tests."""
|
|
return httpx.ConnectError("Connection refused")
|
|
|
|
|
|
def _make_read_timeout() -> httpx.ReadTimeout:
|
|
"""Build a synthetic ReadTimeout for transient-timeout tests."""
|
|
return httpx.ReadTimeout("Read timed out")
|
|
|
|
|
|
# ── Tests ────────────────────────────────────────────────────────────
|
|
|
|
|
|
class TestRetryReadTool:
|
|
"""Contract tests for _retry_read_tool — the fixed-delay retry wrapper."""
|
|
|
|
# #region test_first_attempt_502_retries_once [C:2] [TYPE Function]
|
|
# @BRIEF First attempt raises 502 → retries once → second attempt succeeds.
|
|
async def test_first_attempt_502_retries_once(self):
|
|
"""Prove @TEST_EDGE first_attempt_502: one retry + 1s delay → success."""
|
|
expected = {"status": "ok", "data": [1, 2, 3]}
|
|
error_502 = _make_502_error()
|
|
mock_fn = AsyncMock(side_effect=[error_502, expected])
|
|
start_tool_retry_event_buffer()
|
|
|
|
with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep:
|
|
result = await _retry_read_tool("read_dashboards", mock_fn)
|
|
|
|
assert result == expected
|
|
assert mock_fn.call_count == 2
|
|
mock_sleep.assert_awaited_once_with(1)
|
|
assert drain_tool_retry_events() == [{
|
|
"content": "🔁 read_dashboards retry 1/2",
|
|
"metadata": {
|
|
"type": "tool_retry",
|
|
"tool": "read_dashboards",
|
|
"attempt": 1,
|
|
"max_attempts": 2,
|
|
},
|
|
}]
|
|
# #endregion test_first_attempt_502_retries_once
|
|
|
|
# #region test_both_attempts_502_raises [C:2] [TYPE Function]
|
|
# @BRIEF Both attempts raise 502 → exhaust retries → raises original error.
|
|
async def test_both_attempts_502_raises(self):
|
|
"""Prove @TEST_EDGE both_attempts_502: max 2 attempts, then raise."""
|
|
error_502 = _make_502_error()
|
|
mock_fn = AsyncMock(side_effect=[error_502, error_502])
|
|
|
|
with patch("asyncio.sleep", new_callable=AsyncMock), pytest.raises(httpx.HTTPStatusError) as exc_info:
|
|
await _retry_read_tool("read_dashboards", mock_fn)
|
|
|
|
assert exc_info.value is error_502
|
|
assert mock_fn.call_count == 2
|
|
# #endregion test_both_attempts_502_raises
|
|
|
|
# #region test_connect_error_retried [C:2] [TYPE Function]
|
|
# @BRIEF ConnectError is also retried — not just HTTP status errors.
|
|
async def test_connect_error_retried(self):
|
|
"""Prove ConnectError triggers the retry path."""
|
|
conn_err = _make_connect_error()
|
|
expected = "recovered_after_connect_error"
|
|
mock_fn = AsyncMock(side_effect=[conn_err, expected])
|
|
|
|
with patch("asyncio.sleep", new_callable=AsyncMock):
|
|
result = await _retry_read_tool("read_some_tool", mock_fn)
|
|
|
|
assert result == expected
|
|
assert mock_fn.call_count == 2
|
|
# #endregion test_connect_error_retried
|
|
|
|
# #region test_read_timeout_retried [C:2] [TYPE Function]
|
|
# @BRIEF ReadTimeout is also retried — transient I/O timeouts are recoverable.
|
|
async def test_read_timeout_retried(self):
|
|
"""Prove ReadTimeout triggers the retry path."""
|
|
timeout_err = _make_read_timeout()
|
|
expected = "recovered_after_timeout"
|
|
mock_fn = AsyncMock(side_effect=[timeout_err, expected])
|
|
|
|
with patch("asyncio.sleep", new_callable=AsyncMock):
|
|
result = await _retry_read_tool("read_big_dataset", mock_fn)
|
|
|
|
assert result == expected
|
|
assert mock_fn.call_count == 2
|
|
# #endregion test_read_timeout_retried
|
|
|
|
# #region test_retry_skips_delay_on_success [C:2] [TYPE Function]
|
|
# @BRIEF When first attempt succeeds, no sleep occurs at all.
|
|
async def test_retry_skips_delay_on_success(self):
|
|
"""Prove that the happy path never sleeps — sleep is only for retries."""
|
|
expected = {"result": "immediate"}
|
|
mock_fn = AsyncMock(return_value=expected)
|
|
|
|
with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep:
|
|
result = await _retry_read_tool("fast_tool", mock_fn)
|
|
|
|
assert result == expected
|
|
assert mock_fn.call_count == 1
|
|
mock_sleep.assert_not_awaited()
|
|
# #endregion test_retry_skips_delay_on_success
|
|
|
|
# #region test_non_http_error_not_retried [C:2] [TYPE Function]
|
|
# @BRIEF Non-HTTP errors (e.g. ValueError) propagate immediately — no retry.
|
|
async def test_non_http_error_not_retried(self):
|
|
"""Prove that only the three specific httpx exception types are retried."""
|
|
non_http_err = ValueError("something broken in business logic")
|
|
mock_fn = AsyncMock(side_effect=non_http_err)
|
|
|
|
with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep, pytest.raises(ValueError) as exc_info:
|
|
await _retry_read_tool("broken_tool", mock_fn)
|
|
|
|
assert exc_info.value is non_http_err
|
|
assert mock_fn.call_count == 1
|
|
mock_sleep.assert_not_awaited()
|
|
# #endregion test_non_http_error_not_retried
|
|
|
|
|
|
class TestWriteToolNoRetry:
|
|
"""Prove that write tools bypass _retry_read_tool entirely."""
|
|
|
|
# #region test_write_tool_502_no_retry [C:2] [TYPE Function]
|
|
# @BRIEF Write tool (is_write=True) gets 502 → no retry, raises immediately.
|
|
async def test_write_tool_502_raises_immediately(self):
|
|
"""Prove @TEST_EDGE write_tool_502: _execute_with_timeout does NOT retry writes.
|
|
|
|
_post() calls _execute_with_timeout with is_write=True and the raw _request
|
|
function — never _retry_read_tool. This test ensures that layer propagates
|
|
errors immediately without any retry loop.
|
|
"""
|
|
error_502 = _make_502_error()
|
|
write_op = AsyncMock(side_effect=error_502)
|
|
|
|
with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep, pytest.raises(httpx.HTTPStatusError) as exc_info:
|
|
await _execute_with_timeout(
|
|
"create_dashboard",
|
|
write_op,
|
|
is_write=True,
|
|
timeout_s=5,
|
|
)
|
|
|
|
assert exc_info.value is error_502
|
|
assert write_op.call_count == 1
|
|
# Critical invariant: no sleep = no retry loop entered
|
|
mock_sleep.assert_not_awaited()
|
|
# #endregion test_write_tool_502_no_retry
|
|
|
|
|
|
# #endregion Test.AgentChat.ToolRetry
|