# #region LLMAsyncHttpClient [C:4] [TYPE Module] [SEMANTICS translate, llm, http, openai, async, retry, rate-limit] # @ingroup Translate # @BRIEF Async HTTP client for OpenAI-compatible LLM API calls using httpx.AsyncClient # with rate-limit handling and structured output fallback. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [EXT:httpx:AsyncClient] # @PRE Valid API endpoint, key, model, and prompt. # @POST Returns (response text, finish_reason) tuple. # @SIDE_EFFECT Async HTTP POST to LLM API with optional retry on 429. # @RATIONALE Async migration of _llm_http.py to use httpx.AsyncClient instead of sync requests. # Uses asyncio.sleep for 429 backoff instead of time.sleep. Module-level httpx client singleton # for connection reuse. response.ok -> response.is_success for httpx.Response compatibility. # @REJECTED Keeping sync requests.post — would block async event loop during LLM calls. # Per-request httpx.AsyncClient — loses connection pooling. response.ok — httpx.Response has no # .ok attribute (only is_success), causes 'Response' object has no attribute 'ok'. import asyncio import os import ssl from typing import Any import httpx from ...core.cot_logger import log from ...core.logger import logger from ._utils import _sanitize_url # Module-level httpx client, lazily initialized for connection reuse _http_client: httpx.AsyncClient | None = None # Default provider and max_tokens constants DEFAULT_PROVIDER_TYPE: str = "openai" DEFAULT_MAX_TOKENS: int = 8192 # #region _get_verify [C:1] [TYPE Function] [SEMANTICS translate, ssl, verify] # @BRIEF Resolve SSL verification context via centralized core.ssl helper. # @RATIONALE Используем capath=/etc/ssl/certs/ через ssl.create_default_context, # потому что OpenSSL 3.x не использует intermediate CA сертификаты из cafile # для построения цепочки (verify code 20). capath с хеш-симлинками работает # корректно (verify code 0). Возвращаем SSLContext вместо строки — httpx 0.28.x # депрекейтит строковый путь в verify=. # @REJECTED cafile отвергнут — OpenSSL 3.x не использует intermediate CA # из единого bundle-файла. Только capath с хеш-симлинками даёт code 0. # @REJECTED Строковый путь "/etc/ssl/certs/" отвергнут — httpx 0.28.x депрекейтит # строки в verify=, требует SSLContext. # @REJECTED LLM_SSL_VERIFY=false escape hatch отвергнут — централизованный SSL # всегда использует системное хранилище без возможности отключения. # @POST Returns ssl.SSLContext with capath (never False). def _get_verify() -> ssl.SSLContext: from ...core.ssl import httpx_verify return httpx_verify() # #endregion _get_verify # #region _get_http_client [C:1] [TYPE Function] # @BRIEF Get or create the module-level httpx.AsyncClient singleton. # @POST Returns httpx.AsyncClient with SSL verify and 180s timeout. async def _get_http_client() -> httpx.AsyncClient: global _http_client if _http_client is None: ssl_verify = _get_verify() _http_client = httpx.AsyncClient( verify=ssl_verify, timeout=httpx.Timeout(180.0), ) return _http_client # #endregion _get_http_client # #region call_openai_compatible [C:3] [TYPE Function] [SEMANTICS translate, llm, http, openai, async] # @ingroup Translate # @BRIEF Call OpenAI-compatible API asynchronously with rate-limit handling and structured output fallback. # @PRE Valid API endpoint, key, model, and prompt. # @POST Returns (response text, finish_reason). # @SIDE_EFFECT Async HTTP POST to LLM API. async def call_openai_compatible( base_url: str, api_key: str, model: str, prompt: str, provider_type: str = "openai", max_tokens: int = 8192, disable_reasoning: bool = False, ) -> tuple[str, str | None]: """Call OpenAI-compatible API for batch translation (async).""" if not base_url: raise ValueError("LLM provider has no base_url configured") # Normalise base_url: strip trailing /v1 to avoid double /v1 base = base_url.rstrip("/") if base.endswith("/v1"): base = base[:-3] url = f"{base}/v1/chat/completions" headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", } system_content = "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics. Respond directly with ONLY the JSON result. Do NOT include any reasoning, thinking, chain-of-thought, analysis, or explanation. Output ONLY valid JSON." payload: dict[str, Any] = { "model": model, "messages": [ {"role": "system", "content": system_content}, {"role": "user", "content": prompt}, ], "temperature": 0.1, "max_tokens": max_tokens, } if provider_type in ("openai", "openai_compatible", "kilo", "openrouter", "litellm"): if not disable_reasoning: payload["response_format"] = {"type": "json_object"} if disable_reasoning: if provider_type not in ("kilo", "openrouter", "litellm"): payload["reasoning_effort"] = "none" payload["max_tokens"] = max_tokens logger.reason(f"LLM request url={_sanitize_url(base_url)} model={payload.get('model')} provider_type={provider_type} response_format={'yes' if 'response_format' in payload else 'no'} prompt_len={len(prompt)}") response, response_text = await _do_http_request(url, headers, payload) await _handle_response_format_fallback(response, response_text, payload, url, headers) if not response.is_success: logger.explore(f"LLM API error status={response.status_code} model={payload.get('model')} body={response_text[:2000]}") response.raise_for_status() data = response.json() choices = data.get("choices", []) if not choices: logger.explore( "LLM returned no choices", extra={ "src": "executor", "response_keys": list(data.keys()), "response_preview": str(data)[:2000], }, ) raise ValueError("LLM returned no choices") try: finish_reason = choices[0].get("finish_reason") or "none" msg = choices[0].get("message") or {} except (TypeError, AttributeError) as e: logger.explore( "TypeError processing LLM response choices", extra={ "src": "executor_diag", "error": str(e), "choices_0_type": type(choices[0]).__name__ if choices else "N/A", "choices_0_repr": repr(choices[0])[:2000] if choices else "N/A", "data_type": type(data).__name__, "data_preview": str(data)[:2000], }, ) raise ValueError(f"LLM response processing failed: {e}") # Log provider token usage for batch sizing calibration usage = data.get("usage") or {} if usage: logger.reason( "LLM provider usage", { "prompt_tokens": usage.get("prompt_tokens"), "completion_tokens": usage.get("completion_tokens"), "total_tokens": usage.get("total_tokens"), "finish_reason": finish_reason, "max_tokens_sent": max_tokens, "chars_sent": len(prompt), }, ) refusal = msg.get("refusal") if isinstance(msg, dict) else None if refusal: logger.explore( "LLM refused to respond", extra={ "src": "executor", "refusal": str(refusal)[:500], "finish_reason": finish_reason, }, ) raise ValueError(f"LLM refused to respond: {refusal}") content = msg.get("content") if isinstance(msg, dict) else "" if not content and isinstance(msg, dict): content = msg.get("content") or "" logger.reason(f"LLM response finish_reason={finish_reason} content_len={len(content)} msg_keys={list(msg.keys()) if isinstance(msg, dict) else []}") if not content: logger.explore( "LLM returned empty content", extra={ "src": "executor", "finish_reason": finish_reason, "msg_keys": list(msg.keys()) if isinstance(msg, dict) else [], "response_preview": str(data)[:2000], }, ) raise ValueError("LLM returned empty content") return content, finish_reason # #endregion call_openai_compatible # #region _do_http_request [C:1] [TYPE Function] async def _do_http_request(url: str, headers: dict, payload: dict) -> tuple[httpx.Response, str]: """Make async HTTP POST with rate-limit (429) retry handling.""" client = await _get_http_client() _max_retry_429 = 3 _retry_count_429 = 0 while _retry_count_429 < _max_retry_429: response = await client.post(url, headers=headers, json=payload) response_text = response.text if response.status_code == 429: _retry_count_429 += 1 retry_after = response.headers.get("Retry-After") if retry_after: try: wait = int(retry_after) except (ValueError, TypeError): wait = 2**_retry_count_429 else: wait = 2**_retry_count_429 logger.explore(f"Rate limited (429), retry {_retry_count_429}/{_max_retry_429} after {wait}s", extra={"src": "executor", "retry_after": retry_after, "wait": wait}) await asyncio.sleep(wait) if _retry_count_429 >= _max_retry_429: break else: break return response, response_text # #endregion _do_http_request # #region _handle_response_format_fallback [C:1] [TYPE Function] async def _handle_response_format_fallback( response: httpx.Response, response_text: str, payload: dict, url: str, headers: dict, ) -> None: """Handle 400 errors from structured_outputs not being supported.""" _patterns = ("response_format", "structured_outputs", "structured", "json_object") if not response.is_success and response.status_code == 400 and any(p in (response_text or "").lower() for p in _patterns): client = await _get_http_client() logger.explore("Structured outputs not supported, retrying without response_format", extra={"src": "executor"}) payload.pop("response_format", None) new_response = await client.post(url, headers=headers, json=payload) # Mutate the original response object with new data response.status_code = new_response.status_code response._content = new_response.content response.encoding = new_response.encoding response.headers = new_response.headers # #endregion _handle_response_format_fallback # #endregion LLMAsyncHttpClient