feat(translate): language detection, async HTTP LLM, history model, agent improvements
- Add async HTTP-based LLM transport (_llm_async_http.py) - Add orthogonal LLM call tests - Improve language detection (_lang_detect.py) and batch insert - Update translate schemas, service utils, preview constants/prompts - Add TranslateHistoryModel with pagination and filtering - Update agent confirmation, persistence, langgraph setup, run, tools - Improve LLM health checking in shared module - Update translate runs API, history route
This commit is contained in:
@@ -3,6 +3,8 @@
|
||||
# Contains: cot_logger (stdlib-only trace propagation),
|
||||
# ssl (stdlib-only SSL context),
|
||||
# logger (lightweight JSON logger, no pydantic),
|
||||
# _llm_http (shared httpx.AsyncClient with system SSL),
|
||||
# _llm_health (LLM health probe, openai+httpx).
|
||||
# @INVARIANT No dependency on FastAPI, SQLAlchemy, Gradio, LangChain.
|
||||
# @INVARIANT All HTTP clients use system_ssl_context() from ssl module.
|
||||
# #endregion ss_tools.shared
|
||||
|
||||
@@ -27,6 +27,7 @@ from openai import (
|
||||
RateLimitError,
|
||||
)
|
||||
|
||||
from ._llm_http import get_shared_http_client
|
||||
from .logger import logger
|
||||
|
||||
# ── LLM provider health cache ─────────────────────────────────────────
|
||||
@@ -50,19 +51,19 @@ async def _check_llm_provider_health() -> str:
|
||||
# Fetch LLM config from backend's own API (same as agent container does)
|
||||
try:
|
||||
fastapi_url = os.getenv("FASTAPI_URL", "http://localhost:8000")
|
||||
async with httpx.AsyncClient(timeout=5) as client:
|
||||
resp = await client.get(f"{fastapi_url}/api/agent/llm-config")
|
||||
if resp.status_code != 200:
|
||||
_llm_status["status"] = "unavailable"
|
||||
_llm_status["last_error"] = f"LLM config endpoint returned {resp.status_code}"
|
||||
_llm_status["last_check_ts"] = time.time()
|
||||
return "unavailable"
|
||||
config = resp.json()
|
||||
if not config or not config.get("configured"):
|
||||
_llm_status["status"] = "unavailable"
|
||||
_llm_status["last_error"] = "No LLM provider configured"
|
||||
_llm_status["last_check_ts"] = time.time()
|
||||
return "unavailable"
|
||||
client = get_shared_http_client(timeout=10)
|
||||
resp = await client.get(f"{fastapi_url}/api/agent/llm-config")
|
||||
if resp.status_code != 200:
|
||||
_llm_status["status"] = "unavailable"
|
||||
_llm_status["last_error"] = f"LLM config endpoint returned {resp.status_code}"
|
||||
_llm_status["last_check_ts"] = time.time()
|
||||
return "unavailable"
|
||||
config = resp.json()
|
||||
if not config or not config.get("configured"):
|
||||
_llm_status["status"] = "unavailable"
|
||||
_llm_status["last_error"] = "No LLM provider configured"
|
||||
_llm_status["last_check_ts"] = time.time()
|
||||
return "unavailable"
|
||||
except Exception as exc:
|
||||
_llm_status["status"] = "unavailable"
|
||||
_llm_status["last_error"] = f"Failed to fetch LLM config: {exc}"
|
||||
@@ -72,15 +73,11 @@ async def _check_llm_provider_health() -> str:
|
||||
# Probe LLM API using AsyncOpenAI (available in both backend and agent containers)
|
||||
try:
|
||||
from openai import AsyncOpenAI
|
||||
from .ssl import system_ssl_context
|
||||
|
||||
api = AsyncOpenAI(
|
||||
api_key=config.get("api_key", ""),
|
||||
base_url=config.get("base_url", "https://api.openai.com/v1"),
|
||||
http_client=httpx.AsyncClient(
|
||||
verify=system_ssl_context(),
|
||||
timeout=10,
|
||||
),
|
||||
http_client=get_shared_http_client(timeout=10),
|
||||
)
|
||||
await api.chat.completions.create(
|
||||
model=config.get("default_model", "gpt-4o-mini"),
|
||||
|
||||
308
shared/src/ss_tools/shared/_llm_http.py
Normal file
308
shared/src/ss_tools/shared/_llm_http.py
Normal file
@@ -0,0 +1,308 @@
|
||||
# #region SharedLlmHttpClient [C:3] [TYPE Module] [SEMANTICS shared,http,client,ssl,llm]
|
||||
# @defgroup Shared Shared lightweight utilities for backend and agent.
|
||||
# @BRIEF Singleton httpx.AsyncClient with system SSL context for all LLM/API HTTP calls.
|
||||
# Provides connection pooling, proper SSL verification (capath), configurable timeout.
|
||||
# @LAYER Infrastructure
|
||||
# @INVARIANT No dependency on FastAPI, SQLAlchemy, Gradio, LangChain, pydantic.
|
||||
# @INVARIANT All HTTP clients use system_ssl_context() for SSL verification.
|
||||
# @RELATION DEPENDS_ON -> [CoreSslTrust]
|
||||
# @RATIONALE All HTTP calls to LLM providers and external APIs must use the system CA
|
||||
# store (capath) instead of certifi to respect corporate certificates installed at
|
||||
# container startup. This module provides a single source of truth for HTTP client
|
||||
# creation, eliminating the pattern of per-module httpx.AsyncClient(timeout=N) without
|
||||
# SSL context that silently uses certifi and fails with corporate CA certs.
|
||||
# @REJECTED Per-module httpx.AsyncClient singletons were rejected — each one was a copy
|
||||
# that could forget SSL context; module-level _http_client variables scattered across
|
||||
# 5+ files created connection pooling fragmentation.
|
||||
# @REJECTED Passing verify=True (certifi default) was rejected — it passes with public CAs
|
||||
# but silently fails with corporate intermediate CAs installed in /etc/ssl/certs.
|
||||
# Only capath-based ssl.create_default_context() works with OpenSSL 3.x intermediates.
|
||||
|
||||
import asyncio
|
||||
import ssl
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from .logger import logger
|
||||
from .ssl import httpx_verify
|
||||
|
||||
# Module-level singleton clients, lazily initialized
|
||||
_http_client_180: httpx.AsyncClient | None = None
|
||||
_http_client_10: httpx.AsyncClient | None = None
|
||||
|
||||
|
||||
# #region SharedLlmHttpClient.GetSharedClient [C:2] [TYPE Function] [SEMANTICS shared,http,client,get]
|
||||
# @ingroup Shared
|
||||
# @BRIEF Get or create a module-level httpx.AsyncClient singleton with system SSL context.
|
||||
# Uses shared ssl module (capath) for certificate verification.
|
||||
# @POST Returns httpx.AsyncClient with verify=system_ssl_context() and specified timeout.
|
||||
# @SIDE_EFFECT Lazily creates the client on first call and caches it for reuse.
|
||||
def get_shared_http_client(timeout: float = 180.0) -> httpx.AsyncClient:
|
||||
"""Get or create a shared httpx.AsyncClient singleton with system SSL context.
|
||||
|
||||
Uses the system CA store (capath) for SSL verification — NOT certifi.
|
||||
This ensures corporate CA certificates installed at container startup
|
||||
are trusted for all HTTP calls.
|
||||
|
||||
Args:
|
||||
timeout: Total timeout in seconds (default 180).
|
||||
Use 10 for lightweight config fetches, 180 for LLM API calls.
|
||||
|
||||
Returns:
|
||||
httpx.AsyncClient with SSL context and specified timeout.
|
||||
"""
|
||||
global _http_client_180, _http_client_10
|
||||
|
||||
# Cache two common timeout configurations to avoid creating new clients
|
||||
if abs(timeout - 180.0) < 0.1:
|
||||
if _http_client_180 is None:
|
||||
ssl_ctx = httpx_verify()
|
||||
_http_client_180 = httpx.AsyncClient(
|
||||
verify=ssl_ctx,
|
||||
timeout=httpx.Timeout(180.0),
|
||||
)
|
||||
logger.reason(
|
||||
"Created shared HTTP client (180s timeout)",
|
||||
extra={"src": "SharedLlmHttpClient", "ssl": "system_ssl_context"},
|
||||
)
|
||||
return _http_client_180
|
||||
|
||||
if abs(timeout - 10.0) < 0.1:
|
||||
if _http_client_10 is None:
|
||||
ssl_ctx = httpx_verify()
|
||||
_http_client_10 = httpx.AsyncClient(
|
||||
verify=ssl_ctx,
|
||||
timeout=httpx.Timeout(10.0),
|
||||
)
|
||||
logger.reason(
|
||||
"Created shared HTTP client (10s timeout)",
|
||||
extra={"src": "SharedLlmHttpClient", "ssl": "system_ssl_context"},
|
||||
)
|
||||
return _http_client_10
|
||||
|
||||
# Custom timeout — create a new client (not cached)
|
||||
ssl_ctx = httpx_verify()
|
||||
client = httpx.AsyncClient(
|
||||
verify=ssl_ctx,
|
||||
timeout=httpx.Timeout(timeout),
|
||||
)
|
||||
logger.reason(
|
||||
"Created shared HTTP client (custom timeout)",
|
||||
extra={"src": "SharedLlmHttpClient", "timeout": timeout, "ssl": "system_ssl_context"},
|
||||
)
|
||||
return client
|
||||
# #endregion SharedLlmHttpClient.GetSharedClient
|
||||
|
||||
|
||||
# #region SharedLlmHttpClient.SanitizeUrl [C:1] [TYPE Function] [SEMANTICS shared,url,sanitize]
|
||||
# @ingroup Shared
|
||||
# @BRIEF Strip embedded credentials from URL for safe logging.
|
||||
# @POST Returns URL with user:pass@ portion removed, preserving host:port.
|
||||
def sanitize_url(url: str) -> str:
|
||||
"""Strip embedded credentials from URL for safe logging."""
|
||||
if not url:
|
||||
return url
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
parsed = urlsplit(url)
|
||||
if parsed.username or parsed.password:
|
||||
safe_netloc = parsed.hostname
|
||||
if parsed.port:
|
||||
safe_netloc += f":{parsed.port}"
|
||||
parsed = parsed._replace(netloc=safe_netloc)
|
||||
return urlunsplit(parsed)
|
||||
# #endregion SharedLlmHttpClient.SanitizeUrl
|
||||
|
||||
|
||||
# #region SharedLlmHttpClient.CallOpenaiCompatible [C:3] [TYPE Function] [SEMANTICS shared,llm,http,openai,async]
|
||||
# @ingroup Shared
|
||||
# @BRIEF Call OpenAI-compatible API asynchronously with rate-limit handling and structured output fallback.
|
||||
# @PRE Valid API endpoint, key, model, and prompt.
|
||||
# @POST Returns (response text, finish_reason).
|
||||
# @SIDE_EFFECT Async HTTP POST to LLM API with optional retry on 429.
|
||||
# @REJECTED Keeping sync requests.post — would block async event loop during LLM calls.
|
||||
# Per-request httpx.AsyncClient — loses connection pooling.
|
||||
async def call_openai_compatible(
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model: str,
|
||||
prompt: str,
|
||||
provider_type: str = "openai",
|
||||
max_tokens: int = 8192,
|
||||
disable_reasoning: bool = False,
|
||||
) -> tuple[str, str | None]:
|
||||
"""Call OpenAI-compatible API for LLM requests (async)."""
|
||||
if not base_url:
|
||||
raise ValueError("LLM provider has no base_url configured")
|
||||
|
||||
# Normalise base_url: strip trailing /v1 to avoid double /v1
|
||||
base = base_url.rstrip("/")
|
||||
if base.endswith("/v1"):
|
||||
base = base[:-3]
|
||||
url = f"{base}/v1/chat/completions"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
system_content = (
|
||||
"You are a database content translation assistant. "
|
||||
"Translate the provided text accurately, preserving data semantics. "
|
||||
"Respond directly with ONLY the JSON result. "
|
||||
"Do NOT include any reasoning, thinking, chain-of-thought, analysis, or explanation. "
|
||||
"Output ONLY valid JSON."
|
||||
)
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_content},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"temperature": 0.1,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
|
||||
if provider_type in ("openai", "openai_compatible", "kilo", "openrouter", "litellm"):
|
||||
if not disable_reasoning:
|
||||
payload["response_format"] = {"type": "json_object"}
|
||||
|
||||
if disable_reasoning:
|
||||
if provider_type not in ("kilo", "openrouter", "litellm"):
|
||||
payload["reasoning_effort"] = "none"
|
||||
payload["max_tokens"] = max_tokens
|
||||
|
||||
client = get_shared_http_client()
|
||||
response, response_text = await _do_http_request(client, url, headers, payload)
|
||||
await _handle_response_format_fallback(client, response, response_text, payload, url, headers)
|
||||
|
||||
if not response.is_success:
|
||||
logger.explore(
|
||||
f"LLM API error status={response.status_code} model={payload.get('model')} body={response_text[:2000]}",
|
||||
extra={"src": "SharedLlmHttpClient"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
choices = data.get("choices", [])
|
||||
if not choices:
|
||||
logger.explore(
|
||||
"LLM returned no choices",
|
||||
extra={
|
||||
"src": "SharedLlmHttpClient",
|
||||
"response_keys": list(data.keys()),
|
||||
"response_preview": str(data)[:2000],
|
||||
},
|
||||
)
|
||||
raise ValueError("LLM returned no choices")
|
||||
|
||||
try:
|
||||
finish_reason = choices[0].get("finish_reason") or "none"
|
||||
msg = choices[0].get("message") or {}
|
||||
except (TypeError, AttributeError) as e:
|
||||
logger.explore(
|
||||
"TypeError processing LLM response choices",
|
||||
extra={
|
||||
"src": "SharedLlmHttpClient",
|
||||
"error": str(e),
|
||||
"choices_0_type": type(choices[0]).__name__ if choices else "N/A",
|
||||
"choices_0_repr": repr(choices[0])[:2000] if choices else "N/A",
|
||||
"data_type": type(data).__name__,
|
||||
"data_preview": str(data)[:2000],
|
||||
},
|
||||
)
|
||||
raise ValueError(f"LLM response processing failed: {e}")
|
||||
|
||||
refusal = msg.get("refusal") if isinstance(msg, dict) else None
|
||||
if refusal:
|
||||
logger.explore(
|
||||
"LLM refused to respond",
|
||||
extra={
|
||||
"src": "SharedLlmHttpClient",
|
||||
"refusal": str(refusal)[:500],
|
||||
"finish_reason": finish_reason,
|
||||
},
|
||||
)
|
||||
raise ValueError(f"LLM refused to respond: {refusal}")
|
||||
|
||||
content = msg.get("content") if isinstance(msg, dict) else ""
|
||||
if not content and isinstance(msg, dict):
|
||||
content = msg.get("content") or ""
|
||||
|
||||
if not content:
|
||||
logger.explore(
|
||||
"LLM returned empty content",
|
||||
extra={
|
||||
"src": "SharedLlmHttpClient",
|
||||
"finish_reason": finish_reason,
|
||||
"msg_keys": list(msg.keys()) if isinstance(msg, dict) else [],
|
||||
"response_preview": str(data)[:2000],
|
||||
},
|
||||
)
|
||||
raise ValueError("LLM returned empty content")
|
||||
|
||||
return content, finish_reason
|
||||
# #endregion SharedLlmHttpClient.CallOpenaiCompatible
|
||||
|
||||
|
||||
# #region SharedLlmHttpClient.DoHttpRequest [C:1] [TYPE Function] [SEMANTICS shared,http,request,retry]
|
||||
async def _do_http_request(
|
||||
client: httpx.AsyncClient,
|
||||
url: str,
|
||||
headers: dict,
|
||||
payload: dict,
|
||||
) -> tuple[httpx.Response, str]:
|
||||
"""Make async HTTP POST with rate-limit (429) retry handling."""
|
||||
_max_retry_429 = 3
|
||||
_retry_count_429 = 0
|
||||
while _retry_count_429 < _max_retry_429:
|
||||
response = await client.post(url, headers=headers, json=payload)
|
||||
response_text = response.text
|
||||
if response.status_code == 429:
|
||||
_retry_count_429 += 1
|
||||
retry_after = response.headers.get("Retry-After")
|
||||
if retry_after:
|
||||
try:
|
||||
wait = int(retry_after)
|
||||
except (ValueError, TypeError):
|
||||
wait = 2**_retry_count_429
|
||||
else:
|
||||
wait = 2**_retry_count_429
|
||||
logger.explore(
|
||||
f"Rate limited (429), retry {_retry_count_429}/{_max_retry_429} after {wait}s",
|
||||
extra={"src": "SharedLlmHttpClient", "retry_after": retry_after, "wait": wait},
|
||||
)
|
||||
await asyncio.sleep(wait)
|
||||
if _retry_count_429 >= _max_retry_429:
|
||||
break
|
||||
else:
|
||||
break
|
||||
return response, response_text
|
||||
# #endregion SharedLlmHttpClient.DoHttpRequest
|
||||
|
||||
|
||||
# #region SharedLlmHttpClient.HandleResponseFormatFallback [C:1] [TYPE Function] [SEMANTICS shared,http,response,fallback]
|
||||
async def _handle_response_format_fallback(
|
||||
client: httpx.AsyncClient,
|
||||
response: httpx.Response,
|
||||
response_text: str,
|
||||
payload: dict,
|
||||
url: str,
|
||||
headers: dict,
|
||||
) -> None:
|
||||
"""Handle 400 errors from structured_outputs not being supported."""
|
||||
_patterns = ("response_format", "structured_outputs", "structured", "json_object")
|
||||
if not response.is_success and response.status_code == 400 and any(p in (response_text or "").lower() for p in _patterns):
|
||||
logger.explore(
|
||||
"Structured outputs not supported, retrying without response_format",
|
||||
extra={"src": "SharedLlmHttpClient"},
|
||||
)
|
||||
payload.pop("response_format", None)
|
||||
new_response = await client.post(url, headers=headers, json=payload)
|
||||
# Mutate the original response object with new data
|
||||
response.status_code = new_response.status_code
|
||||
response._content = new_response.content
|
||||
response.encoding = new_response.encoding
|
||||
response.headers = new_response.headers
|
||||
# #endregion SharedLlmHttpClient.HandleResponseFormatFallback
|
||||
# #endregion SharedLlmHttpClient
|
||||
Reference in New Issue
Block a user