refactor(agent): extract agent+shared into standalone packages with full GRACE semantic markup

- Move agent code from backend/src/agent/ to agent/src/ss_tools/agent/
- Extract shared stdlib-only utilities to shared/src/ss_tools/shared/
- Add #region/#endregion contracts to all ~140 functions (INV_1 compliance)
- Update docker files, entrypoint, build scripts for new package layout
- Backend now imports ss_tools.shared._llm_health (no gradio/langchain deps)
- Add specs for 036-039 feature plans

agent/tests/test_agent_retry.py

@@ -0,0 +1,178 @@
+# #region Test.AgentChat.ToolRetry [C:3] [TYPE Module] [SEMANTICS test,agent,tools,retry]
+# @BRIEF Contract tests for _retry_read_tool — fixed-delay retry on transient errors.
+# @RELATION BINDS_TO -> [AgentChat.Tools.Retry]
+# @TEST_EDGE: first_attempt_502 -> Auto-retries once, succeeds.
+# @TEST_EDGE: both_attempts_502 -> Raises original error.
+# @TEST_EDGE: write_tool_502 -> No retry, raises immediately.
+# @TEST_EDGE: connect_error -> Retries on ConnectError too.
+# @TEST_EDGE: read_timeout -> Retries on ReadTimeout too.
+
+import pytest
+from unittest.mock import AsyncMock, patch
+
+import httpx
+
+from ss_tools.agent.tools import (
+    _execute_with_timeout,
+    _retry_read_tool,
+    drain_tool_retry_events,
+    start_tool_retry_event_buffer,
+)
+
+# ── Shared fixtures ──────────────────────────────────────────────────
+
+def _make_502_error() -> httpx.HTTPStatusError:
+    """Build a synthetic 502 HTTPStatusError for consistent test usage."""
+    request = httpx.Request("GET", "http://test.local/api/test")
+    response = httpx.Response(502, request=request)
+    return httpx.HTTPStatusError("Bad Gateway", request=request, response=response)
+
+
+def _make_connect_error() -> httpx.ConnectError:
+    """Build a synthetic ConnectError for transient-connectivity tests."""
+    return httpx.ConnectError("Connection refused")
+
+
+def _make_read_timeout() -> httpx.ReadTimeout:
+    """Build a synthetic ReadTimeout for transient-timeout tests."""
+    return httpx.ReadTimeout("Read timed out")
+
+
+# ── Tests ────────────────────────────────────────────────────────────
+
+
+class TestRetryReadTool:
+    """Contract tests for _retry_read_tool — the fixed-delay retry wrapper."""
+
+    # #region test_first_attempt_502_retries_once [C:2] [TYPE Function]
+    # @BRIEF First attempt raises 502 → retries once → second attempt succeeds.
+    async def test_first_attempt_502_retries_once(self):
+        """Prove @TEST_EDGE first_attempt_502: one retry + 1s delay → success."""
+        expected = {"status": "ok", "data": [1, 2, 3]}
+        error_502 = _make_502_error()
+        mock_fn = AsyncMock(side_effect=[error_502, expected])
+        start_tool_retry_event_buffer()
+
+        with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep:
+            result = await _retry_read_tool("read_dashboards", mock_fn)
+
+        assert result == expected
+        assert mock_fn.call_count == 2
+        mock_sleep.assert_awaited_once_with(1)
+        assert drain_tool_retry_events() == [{
+            "content": "🔁 read_dashboards retry 1/2",
+            "metadata": {
+                "type": "tool_retry",
+                "tool": "read_dashboards",
+                "attempt": 1,
+                "max_attempts": 2,
+            },
+        }]
+    # #endregion test_first_attempt_502_retries_once
+
+    # #region test_both_attempts_502_raises [C:2] [TYPE Function]
+    # @BRIEF Both attempts raise 502 → exhaust retries → raises original error.
+    async def test_both_attempts_502_raises(self):
+        """Prove @TEST_EDGE both_attempts_502: max 2 attempts, then raise."""
+        error_502 = _make_502_error()
+        mock_fn = AsyncMock(side_effect=[error_502, error_502])
+
+        with patch("asyncio.sleep", new_callable=AsyncMock), pytest.raises(httpx.HTTPStatusError) as exc_info:
+            await _retry_read_tool("read_dashboards", mock_fn)
+
+        assert exc_info.value is error_502
+        assert mock_fn.call_count == 2
+    # #endregion test_both_attempts_502_raises
+
+    # #region test_connect_error_retried [C:2] [TYPE Function]
+    # @BRIEF ConnectError is also retried — not just HTTP status errors.
+    async def test_connect_error_retried(self):
+        """Prove ConnectError triggers the retry path."""
+        conn_err = _make_connect_error()
+        expected = "recovered_after_connect_error"
+        mock_fn = AsyncMock(side_effect=[conn_err, expected])
+
+        with patch("asyncio.sleep", new_callable=AsyncMock):
+            result = await _retry_read_tool("read_some_tool", mock_fn)
+
+        assert result == expected
+        assert mock_fn.call_count == 2
+    # #endregion test_connect_error_retried
+
+    # #region test_read_timeout_retried [C:2] [TYPE Function]
+    # @BRIEF ReadTimeout is also retried — transient I/O timeouts are recoverable.
+    async def test_read_timeout_retried(self):
+        """Prove ReadTimeout triggers the retry path."""
+        timeout_err = _make_read_timeout()
+        expected = "recovered_after_timeout"
+        mock_fn = AsyncMock(side_effect=[timeout_err, expected])
+
+        with patch("asyncio.sleep", new_callable=AsyncMock):
+            result = await _retry_read_tool("read_big_dataset", mock_fn)
+
+        assert result == expected
+        assert mock_fn.call_count == 2
+    # #endregion test_read_timeout_retried
+
+    # #region test_retry_skips_delay_on_success [C:2] [TYPE Function]
+    # @BRIEF When first attempt succeeds, no sleep occurs at all.
+    async def test_retry_skips_delay_on_success(self):
+        """Prove that the happy path never sleeps — sleep is only for retries."""
+        expected = {"result": "immediate"}
+        mock_fn = AsyncMock(return_value=expected)
+
+        with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep:
+            result = await _retry_read_tool("fast_tool", mock_fn)
+
+        assert result == expected
+        assert mock_fn.call_count == 1
+        mock_sleep.assert_not_awaited()
+    # #endregion test_retry_skips_delay_on_success
+
+    # #region test_non_http_error_not_retried [C:2] [TYPE Function]
+    # @BRIEF Non-HTTP errors (e.g. ValueError) propagate immediately — no retry.
+    async def test_non_http_error_not_retried(self):
+        """Prove that only the three specific httpx exception types are retried."""
+        non_http_err = ValueError("something broken in business logic")
+        mock_fn = AsyncMock(side_effect=non_http_err)
+
+        with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep, pytest.raises(ValueError) as exc_info:
+            await _retry_read_tool("broken_tool", mock_fn)
+
+        assert exc_info.value is non_http_err
+        assert mock_fn.call_count == 1
+        mock_sleep.assert_not_awaited()
+    # #endregion test_non_http_error_not_retried
+
+
+class TestWriteToolNoRetry:
+    """Prove that write tools bypass _retry_read_tool entirely."""
+
+    # #region test_write_tool_502_no_retry [C:2] [TYPE Function]
+    # @BRIEF Write tool (is_write=True) gets 502 → no retry, raises immediately.
+    async def test_write_tool_502_raises_immediately(self):
+        """Prove @TEST_EDGE write_tool_502: _execute_with_timeout does NOT retry writes.
+
+        _post() calls _execute_with_timeout with is_write=True and the raw _request
+        function — never _retry_read_tool. This test ensures that layer propagates
+        errors immediately without any retry loop.
+        """
+        error_502 = _make_502_error()
+        write_op = AsyncMock(side_effect=error_502)
+
+        with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep, pytest.raises(httpx.HTTPStatusError) as exc_info:
+            await _execute_with_timeout(
+                "create_dashboard",
+                write_op,
+                is_write=True,
+                timeout_s=5,
+            )
+
+        assert exc_info.value is error_502
+        assert write_op.call_count == 1
+        # Critical invariant: no sleep = no retry loop entered
+        mock_sleep.assert_not_awaited()
+    # #endregion test_write_tool_502_no_retry
+
+
+# #endregion Test.AgentChat.ToolRetry