ss-tools/backend/src/plugins/translate/_llm_async_http.py

# #region LLMAsyncHttpClient [C:4] [TYPE Module] [SEMANTICS translate, llm, http, openai, async, retry, rate-limit]
# @defgroup Translate Module group.
# @BRIEF Async HTTP client for OpenAI-compatible LLM API calls using httpx.AsyncClient
#         with rate-limit handling and structured output fallback.
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [EXT:httpx:AsyncClient]
# @PRE Valid API endpoint, key, model, and prompt.
# @POST Returns (response text, finish_reason) tuple.
# @SIDE_EFFECT Async HTTP POST to LLM API with optional retry on 429.
# @RATIONALE Async migration of _llm_http.py to use httpx.AsyncClient instead of sync requests.
#   Uses asyncio.sleep for 429 backoff instead of time.sleep. Module-level httpx client singleton
#   for connection reuse. response.ok -> response.is_success for httpx.Response compatibility.
# @REJECTED Keeping sync requests.post — would block async event loop during LLM calls.
#   Per-request httpx.AsyncClient — loses connection pooling. response.ok — httpx.Response has no
#   .ok attribute (only is_success), causes 'Response' object has no attribute 'ok'.

import asyncio
import os
import ssl
from typing import Any

import httpx

from ...core.cot_logger import log
from ...core.logger import logger
from ._utils import _sanitize_url

# Module-level httpx client, lazily initialized for connection reuse
_http_client: httpx.AsyncClient | None = None

# Default provider and max_tokens constants
DEFAULT_PROVIDER_TYPE: str = "openai"
DEFAULT_MAX_TOKENS: int = 8192


# #region _get_verify [C:1] [TYPE Function] [SEMANTICS translate, ssl, verify]
# @BRIEF Resolve SSL verification context from LLM_SSL_VERIFY env var.
# @RATIONALE Используем capath=/etc/ssl/certs/ через ssl.create_default_context,
#   потому что OpenSSL 3.x не использует intermediate CA сертификаты из cafile
#   для построения цепочки (verify code 20). capath с хеш-симлинками работает
#   корректно (verify code 0). Возвращаем SSLContext вместо строки — httpx 0.28.x
#   депрекейтит строковый путь в verify=.
# @REJECTED cafile отвергнут — OpenSSL 3.x не использует intermediate CA
#   из единого bundle-файла. Только capath с хеш-симлинками даёт code 0.
# @REJECTED Строковый путь "/etc/ssl/certs/" отвергнут — httpx 0.28.x депрекейтит
#   строки в verify=, требует SSLContext.
# @POST Returns ssl.SSLContext with capath when enabled, False when disabled.
def _get_verify() -> ssl.SSLContext | bool:
    raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
    if raw in ("false", "0", "no", "off"):
        return False
    return ssl.create_default_context(capath="/etc/ssl/certs/")
# #endregion _get_verify


# #region _get_http_client [C:1] [TYPE Function]
# @BRIEF Get or create the module-level httpx.AsyncClient singleton.
# @POST Returns httpx.AsyncClient with SSL verify and 180s timeout.
async def _get_http_client() -> httpx.AsyncClient:
    global _http_client
    if _http_client is None:
        ssl_verify = _get_verify()
        if ssl_verify is False:
            log("LLMAsyncHttpClient", "EXPLORE",
                "TLS verification disabled via LLM_SSL_VERIFY=false",
                error="TLS verification disabled — traffic to LLM provider is unencrypted")
        _http_client = httpx.AsyncClient(
            verify=ssl_verify,
            timeout=httpx.Timeout(180.0),
        )
    return _http_client
# #endregion _get_http_client


# #region call_openai_compatible [C:3] [TYPE Function] [SEMANTICS translate, llm, http, openai, async]
# @ingroup Translate
# @BRIEF Call OpenAI-compatible API asynchronously with rate-limit handling and structured output fallback.
# @PRE Valid API endpoint, key, model, and prompt.
# @POST Returns (response text, finish_reason).
# @SIDE_EFFECT Async HTTP POST to LLM API.
async def call_openai_compatible(
    base_url: str,
    api_key: str,
    model: str,
    prompt: str,
    provider_type: str = "openai",
    max_tokens: int = 8192,
    disable_reasoning: bool = False,
) -> tuple[str, str | None]:
    """Call OpenAI-compatible API for batch translation (async)."""
    if not base_url:
        raise ValueError("LLM provider has no base_url configured")

    # Normalise base_url: strip trailing /v1 to avoid double /v1
    base = base_url.rstrip("/")
    if base.endswith("/v1"):
        base = base[:-3]
    url = f"{base}/v1/chat/completions"
    headers = {
        "Authorization": f"Bearer {api_key}",
        "Content-Type": "application/json",
    }
    system_content = (
        "You are a database content translation assistant. "
        "Translate the provided text accurately, preserving data semantics. "
        "Respond directly with ONLY the JSON result. "
        "Do NOT include any reasoning, thinking, chain-of-thought, analysis, "
        "or explanation. Output ONLY valid JSON."
    )

    payload: dict[str, Any] = {
        "model": model,
        "messages": [
            {"role": "system", "content": system_content},
            {"role": "user", "content": prompt},
        ],
        "temperature": 0.1,
        "max_tokens": max_tokens,
    }

    if provider_type in ("openai", "openai_compatible", "kilo", "openrouter", "litellm"):
        if not disable_reasoning:
            payload["response_format"] = {"type": "json_object"}

    if disable_reasoning:
        if provider_type not in ("kilo", "openrouter", "litellm"):
            payload["reasoning_effort"] = "none"
        payload["max_tokens"] = max_tokens

    logger.reason(
        f"LLM request url={_sanitize_url(base_url)} model={payload.get('model')} "
        f"provider_type={provider_type} "
        f"response_format={'yes' if 'response_format' in payload else 'no'} "
        f"prompt_len={len(prompt)}"
    )

    response, response_text = await _do_http_request(url, headers, payload)
    await _handle_response_format_fallback(response, response_text, payload, url, headers)

    if not response.is_success:
        logger.explore(
            f"LLM API error status={response.status_code} "
            f"model={payload.get('model')} "
            f"body={response_text[:2000]}"
        )
    response.raise_for_status()
    data = response.json()

    choices = data.get("choices", [])
    if not choices:
        logger.explore("LLM returned no choices", extra={
            "src": "executor", "response_keys": list(data.keys()),
            "response_preview": str(data)[:2000],
        })
        raise ValueError("LLM returned no choices")

    try:
        finish_reason = choices[0].get("finish_reason") or "none"
        msg = choices[0].get("message") or {}
    except (TypeError, AttributeError) as e:
        logger.explore("TypeError processing LLM response choices", extra={
            "src": "executor_diag", "error": str(e),
            "choices_0_type": type(choices[0]).__name__ if choices else "N/A",
            "choices_0_repr": repr(choices[0])[:2000] if choices else "N/A",
            "data_type": type(data).__name__, "data_preview": str(data)[:2000],
        })
        raise ValueError(f"LLM response processing failed: {e}")

    # Log provider token usage for batch sizing calibration
    usage = data.get("usage") or {}
    if usage:
        logger.reason(
            "LLM provider usage",
            {
                "prompt_tokens": usage.get("prompt_tokens"),
                "completion_tokens": usage.get("completion_tokens"),
                "total_tokens": usage.get("total_tokens"),
                "finish_reason": finish_reason,
                "max_tokens_sent": max_tokens,
                "chars_sent": len(prompt),
            },
        )

    refusal = msg.get("refusal") if isinstance(msg, dict) else None
    if refusal:
        logger.explore("LLM refused to respond", extra={
            "src": "executor", "refusal": str(refusal)[:500], "finish_reason": finish_reason,
        })
        raise ValueError(f"LLM refused to respond: {refusal}")

    content = msg.get("content") if isinstance(msg, dict) else ""
    if not content and isinstance(msg, dict):
        content = msg.get("content") or ""

    logger.reason(
        f"LLM response finish_reason={finish_reason} content_len={len(content)} "
        f"msg_keys={list(msg.keys()) if isinstance(msg, dict) else []}"
    )
    if not content:
        logger.explore("LLM returned empty content", extra={
            "src": "executor", "finish_reason": finish_reason,
            "msg_keys": list(msg.keys()) if isinstance(msg, dict) else [],
            "response_preview": str(data)[:2000],
        })
        raise ValueError("LLM returned empty content")

    return content, finish_reason
# #endregion call_openai_compatible


# #region _do_http_request [C:1] [TYPE Function]
async def _do_http_request(url: str, headers: dict, payload: dict) -> tuple[httpx.Response, str]:
    """Make async HTTP POST with rate-limit (429) retry handling."""
    client = await _get_http_client()
    _max_retry_429 = 3
    _retry_count_429 = 0
    while _retry_count_429 < _max_retry_429:
        response = await client.post(url, headers=headers, json=payload)
        response_text = response.text
        if response.status_code == 429:
            _retry_count_429 += 1
            retry_after = response.headers.get("Retry-After")
            if retry_after:
                try:
                    wait = int(retry_after)
                except (ValueError, TypeError):
                    wait = 2 ** _retry_count_429
            else:
                wait = 2 ** _retry_count_429
            logger.explore(f"Rate limited (429), retry {_retry_count_429}/{_max_retry_429} after {wait}s",
                           extra={"src": "executor", "retry_after": retry_after, "wait": wait})
            await asyncio.sleep(wait)
            if _retry_count_429 >= _max_retry_429:
                break
        else:
            break
    return response, response_text
# #endregion _do_http_request


# #region _handle_response_format_fallback [C:1] [TYPE Function]
async def _handle_response_format_fallback(
    response: httpx.Response, response_text: str, payload: dict, url: str, headers: dict,
) -> None:
    """Handle 400 errors from structured_outputs not being supported."""
    _patterns = ("response_format", "structured_outputs", "structured", "json_object")
    if (
        not response.is_success
        and response.status_code == 400
        and any(p in (response_text or "").lower() for p in _patterns)
    ):
        client = await _get_http_client()
        logger.explore("Structured outputs not supported, retrying without response_format",
                       extra={"src": "executor"})
        payload.pop("response_format", None)
        new_response = await client.post(url, headers=headers, json=payload)
        # Mutate the original response object with new data
        response.status_code = new_response.status_code
        response._content = new_response.content
        response.encoding = new_response.encoding
        response.headers = new_response.headers
# #endregion _handle_response_format_fallback
# #endregion LLMAsyncHttpClient