busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ssl+agent): capath for all HTTP clients + isolate gradio import

Browse Source 
SSL fix (ADR-0009 Finding 7):
- Replace ssl.create_default_context() with system_ssl_context(capath)
  in client_registry.py, async_network.py, notifications/providers.py,
  git/_base.py. Previous fix (0.5.1) only covered LLM clients; the
  Superset API client still used ssl.create_default_context() which
  loads cafile (flat bundle) where OpenSSL 3.x ignores intermediate CA
  certificates. system_ssl_context() uses capath only (hash symlinks).

Agent fix:
- Extract _check_llm_provider_health + _llm_status from agent/app.py
  into agent/_llm_health.py. The /api/agent/llm-status endpoint was
  importing from agent/app.py which triggers 'import gradio' at module
  level. Backend container does not have gradio installed, causing
  ModuleNotFoundError (500 error) every 30s on health check polling.

Config:
- Add ALLOWED_ORIGINS to docker-compose.yml + docker-compose.enterprise-clean.yml

ADR-0009 updated: Layer 2 table expanded with 4 missing clients,
Finding 5 reconciled (LLM_CA_CERT_URLS restored in 0.5.1), version 0.5.2.

Verified: 1110 unit tests passed, gradio import isolation confirmed.
This commit is contained in:
busya
2026-07-07 12:18:43 +03:00
parent 18b9f2e94e
commit 58d06fb287
10 changed files with 161 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
backend/src/agent/_llm_health.py Normal file
View File

@@ -0,0 +1,96 @@
# backend/src/agent/_llm_health.py
# #region AgentChat.LlmHealth [C:3] [TYPE Module] [SEMANTICS agent-chat,llm,health,status]
# @ingroup AgentChat
# @BRIEF LLM provider health check with in-memory cache (30s TTL).
# Extracted from agent/app.py to avoid importing gradio in backend container.
# @LAYER Infrastructure
# @RELATION CALLED_BY -> [api/routes/agent_status.py]
# @RELATION CALLED_BY -> [agent/app.py]
# @RATIONALE agent/app.py imports gradio at module level (line 28). The backend
# container does not have gradio installed (only the agent container does).
# The /api/agent/llm-status endpoint needs _check_llm_provider_health but must
# not trigger gradio import. This module isolates the health-check logic.
# @POST Returns status string: 'ok' | 'unavailable' | 'timeout' | 'auth_error'.
# @SIDE_EFFECT Makes a probe request to the LLM provider; caches result in module memory.
import time
from typing import Any
import httpx
from langchain_core.messages import HumanMessage
from langchain_openai import ChatOpenAI
from openai import APIConnectionError, APITimeoutError, AuthenticationError, RateLimitError
from src.agent._llm_params import chat_openai_kwargs
from src.core.logger import logger
# ── LLM provider health cache ─────────────────────────────────────────
_llm_status: dict[str, Any] = {
"status": "ok",
"last_error": "",
"last_check_ts": 0.0,
}
_LLM_CHECK_CACHE_TTL = 30 # seconds between health checks
_LLM_LAST_ERROR_KEY = "last_llm_error"
_LLM_LAST_ERROR_TS_KEY = "last_llm_error_ts"
# #region AgentChat.LlmHealth.Check [C:2] [TYPE Function] [SEMANTICS agent-chat,llm,health,check]
# @ingroup AgentChat
# @BRIEF Check LLM provider connectivity with in-memory cache (30s TTL).
# @POST Returns status string: 'ok' | 'unavailable' | 'timeout' | 'auth_error'.
# @SIDE_EFFECT Makes a probe request to the LLM provider; caches result in module memory.
# @RATIONALE Prevents sending every user request into a dead LLM backend.
async def _check_llm_provider_health() -> str:
"""Check LLM provider connectivity. Cached for _LLM_CHECK_CACHE_TTL seconds."""
now = time.time()
if now - _llm_status["last_check_ts"] < _LLM_CHECK_CACHE_TTL:
return _llm_status["status"]
try:
from src.agent.langgraph_setup import _fetch_llm_config as _get_config
config = await _get_config()
if not config or not config.get("configured"):
return _llm_status["status"]
llm = ChatOpenAI(
**chat_openai_kwargs(
model=config.get("default_model", "gpt-4o-mini"),
base_url=config.get("base_url", "https://api.openai.com/v1"),
api_key=config.get("api_key", ""),
max_tokens=1,
)
)
await llm.ainvoke([HumanMessage(content="ping")])
_llm_status["status"] = "ok"
_llm_status["last_error"] = ""
_llm_status["last_check_ts"] = time.time()
return "ok"
except (APIConnectionError, httpx.ConnectError):
_llm_status["status"] = "unavailable"
_llm_status["last_error"] = "Connection refused"
_llm_status["last_check_ts"] = time.time()
return "unavailable"
except (APITimeoutError, httpx.ReadTimeout):
_llm_status["status"] = "timeout"
_llm_status["last_error"] = "Request timed out"
_llm_status["last_check_ts"] = time.time()
return "timeout"
except AuthenticationError:
_llm_status["status"] = "auth_error"
_llm_status["last_error"] = "Invalid API key"
_llm_status["last_check_ts"] = time.time()
return "auth_error"
except Exception as exc:
if "usage_metadata.total_tokens" in str(exc):
_llm_status["status"] = "ok"
_llm_status["last_error"] = ""
_llm_status["last_check_ts"] = time.time()
return "ok"
logger.explore("LLM health check failed", error=str(exc), extra={"src": "AgentChat.LlmHealth.Check"})
return _llm_status["status"] # return cached status on unexpected errors
# #endregion AgentChat.LlmHealth.Check
# #endregion AgentChat.LlmHealth

79
backend/src/agent/app.py
View File

@@ -40,6 +40,14 @@ from src.agent._confirmation import (
handle_resume,
permission_denied_payload,
)
from src.agent._jwt_decoder import decode_token
from src.agent._llm_health import (
_LLM_CHECK_CACHE_TTL,
_LLM_LAST_ERROR_KEY,
_LLM_LAST_ERROR_TS_KEY,
_check_llm_provider_health,
_llm_status,
)
from src.agent._llm_params import chat_openai_kwargs
from src.agent._persistence import (
extract_user_id,
@@ -58,7 +66,6 @@ from src.agent.tools import (
get_all_tools,
start_tool_retry_event_buffer,
)
from src.agent._jwt_decoder import decode_token
from src.core.cot_logger import seed_trace_id
from src.core.logger import logger
@@ -140,14 +147,9 @@ async def _generate_title_best_effort(conv_id: str, visible_user_text: str) -> N
_user_locks: dict[str, bool] = {}
# ── LLM provider health cache ─────────────────────────────────---
_llm_status: dict[str, Any] = {
"status": "ok",
"last_error": "",
"last_check_ts": 0.0,
}
_LLM_CHECK_CACHE_TTL = 30 # seconds between health checks
_LLM_LAST_ERROR_KEY = "last_llm_error"
_LLM_LAST_ERROR_TS_KEY = "last_llm_error_ts"
# _llm_status, _LLM_CHECK_CACHE_TTL, _LLM_LAST_ERROR_KEY, _LLM_LAST_ERROR_TS_KEY,
# and _check_llm_provider_health() are imported from src.agent._llm_health
# to avoid triggering gradio import in backend container.
# ── File persistence ────────────────────────────────────────────
@@ -214,64 +216,9 @@ _service_jwt_cache: dict[str, str] = {}
# @REJECTED Returning a single response (non-streaming) was rejected — violates FR-003 (streaming mandate).
# #region AgentChat.GradioApp.LlmHealthCheck [C:2] [TYPE Function] [SEMANTICS agent-chat,llm,health]
# @ingroup AgentChat
# @BRIEF Check LLM provider connectivity with in-memory cache (30s TTL).
# @POST Returns status string: 'ok' | 'unavailable' | 'timeout' | 'auth_error'.
# @SIDE_EFFECT Makes a probe request to the LLM provider; caches result in module memory.
# @RATIONALE Prevents sending every user request into an dead LLM backend.
async def _check_llm_provider_health() -> str:
"""Check LLM provider connectivity. Cached for _LLM_CHECK_CACHE_TTL seconds."""
now = time.time()
if now - _llm_status["last_check_ts"] < _LLM_CHECK_CACHE_TTL:
return _llm_status["status"]
# _check_llm_provider_health() moved to src.agent._llm_health
# (avoids gradio import in backend container for /api/agent/llm-status endpoint)
try:
from src.agent.langgraph_setup import _fetch_llm_config as _get_config
config = await _get_config()
if not config or not config.get("configured"):
return _llm_status["status"]
llm = ChatOpenAI(
**chat_openai_kwargs(
model=config.get("default_model", "gpt-4o-mini"),
base_url=config.get("base_url", "https://api.openai.com/v1"),
api_key=config.get("api_key", ""),
max_tokens=1,
)
)
await llm.ainvoke([HumanMessage(content="ping")])
_llm_status["status"] = "ok"
_llm_status["last_error"] = ""
_llm_status["last_check_ts"] = time.time()
return "ok"
except (APIConnectionError, httpx.ConnectError):
_llm_status["status"] = "unavailable"
_llm_status["last_error"] = "Connection refused"
_llm_status["last_check_ts"] = time.time()
return "unavailable"
except (APITimeoutError, httpx.ReadTimeout):
_llm_status["status"] = "timeout"
_llm_status["last_error"] = "Request timed out"
_llm_status["last_check_ts"] = time.time()
return "timeout"
except AuthenticationError:
_llm_status["status"] = "auth_error"
_llm_status["last_error"] = "Invalid API key"
_llm_status["last_check_ts"] = time.time()
return "auth_error"
except Exception as exc:
if "usage_metadata.total_tokens" in str(exc):
_llm_status["status"] = "ok"
_llm_status["last_error"] = ""
_llm_status["last_check_ts"] = time.time()
return "ok"
logger.explore("LLM health check failed", error=str(exc), extra={"src": "AgentChat.GradioApp.LlmHealthCheck"})
return _llm_status["status"] # return cached status on unexpected errors
# #endregion AgentChat.GradioApp.LlmHealthCheck
# #region AgentChat.GradioApp.InjectUIContext [C:2] [TYPE Function] [SEMANTICS agent-chat,context,uicontext]

2
backend/src/api/routes/agent_status.py
View File

@@ -16,7 +16,7 @@ router = APIRouter(prefix="/api/agent", tags=["agent-status"])
@router.get("/llm-status")
async def get_llm_status():
"""Get cached LLM provider health status. Probes provider if cache expired."""
from src.agent.app import _check_llm_provider_health, _llm_status
from src.agent._llm_health import _check_llm_provider_health, _llm_status
status = await _check_llm_provider_health()
return {
"status": status,

8
backend/src/core/utils/async_network.py
View File

@@ -24,6 +24,7 @@ from typing import Any
import httpx
from ..logger import belief_scope, logger as app_logger
from ..ssl import system_ssl_context
from .network import (
AuthenticationError,
DashboardNotFoundError,
@@ -66,10 +67,11 @@ class AsyncAPIClient:
self.api_base_url: str = f"{self.base_url}/api/v1"
self.auth = config.get("auth")
self.request_settings = {"verify_ssl": verify_ssl, "timeout": timeout}
# Use system CA store via ssl.create_default_context() instead of bool True.
# httpx defaults to certifi, ignoring SSL_CERT_FILE and corporate CA certs.
# Use system CA store via capath (not ssl.create_default_context() which
# loads cafile + capath; OpenSSL 3.x ignores intermediate CAs in cafile,
# ADR-0009 Finding 7). system_ssl_context() uses capath only.
if verify_ssl is True:
ssl_context: bool | ssl.SSLContext = ssl.create_default_context()
ssl_context: bool | ssl.SSLContext = system_ssl_context()
elif verify_ssl is False:
ssl_context = False
else:

11
backend/src/core/utils/client_registry.py
View File

@@ -23,6 +23,7 @@ import ssl
from typing import Any
from ..logger import logger
from ..ssl import system_ssl_context
from .async_network import AsyncAPIClient
@@ -99,10 +100,14 @@ async def get_client(
config = {}
verify_ssl = True
timeout = request_timeout
# Convert bool to SSLContext with system CA store — httpx defaults to certifi,
# ignoring corporate CA certificates installed via update-ca-certificates.
# Convert bool to SSLContext with system CA store via capath.
# ssl.create_default_context() without capath loads BOTH cafile (flat bundle)
# and capath (hash symlinks). OpenSSL 3.x ignores intermediate CA certs in
# cafile (ADR-0009 Finding 7), causing CERTIFICATE_VERIFY_FAILED for chains
# with intermediate CAs (e.g. Root → Policy CA → Issuing CA → Server).
# system_ssl_context() uses capath only, correctly building intermediate chains.
if verify_ssl is True:
ssl_context: bool | ssl.SSLContext = ssl.create_default_context()
ssl_context: bool | ssl.SSLContext = system_ssl_context()
else:
ssl_context = verify_ssl
async_client = AsyncAPIClient(

8
backend/src/services/git/_base.py
View File

@@ -25,6 +25,7 @@ import httpx
from src.core.database import SessionLocal
from src.core.logger import belief_scope, logger
from src.core.ssl import system_ssl_context
from src.core.utils.executors import run_blocking
from src.models.config import AppConfigRecord
from src.models.git import GitRepository
@@ -58,10 +59,11 @@ class GitServiceBase:
self._close_lock = threading.Lock()
# Fix 5: Shared httpx.AsyncClient with connection pooling.
# Uses system CA store (ssl.create_default_context) instead of certifi
# to trust corporate CA certificates installed via update-ca-certificates.
# Uses system CA store via capath (ADR-0009 Finding 7) instead of
# ssl.create_default_context() which loads cafile where OpenSSL 3.x
# ignores intermediate CA certificates.
self._http_client = httpx.AsyncClient(
verify=ssl.create_default_context(),
verify=system_ssl_context(),
timeout=30.0,
limits=httpx.Limits(max_keepalive_connections=20, max_connections=100),
)

8
backend/src/services/notifications/providers.py
View File

@@ -29,6 +29,7 @@ import aiosmtplib
import httpx
from src.core.logger import logger
from src.core.ssl import system_ssl_context
# Module-level httpx client, lazily initialized for connection reuse
_http_client: httpx.AsyncClient | None = None
@@ -36,13 +37,14 @@ _http_client: httpx.AsyncClient | None = None
async def _get_http_client() -> httpx.AsyncClient:
"""Get or create the module-level httpx.AsyncClient singleton.
Uses system CA store (ssl.create_default_context) instead of certifi
to trust corporate CA certificates installed via update-ca-certificates.
Uses system CA store via capath (ADR-0009 Finding 7) instead of
ssl.create_default_context() which loads cafile where OpenSSL 3.x
ignores intermediate CA certificates.
"""
global _http_client
if _http_client is None:
_http_client = httpx.AsyncClient(
verify=ssl.create_default_context(),
verify=system_ssl_context(),
timeout=httpx.Timeout(30.0),
)
return _http_client