feat(translate): language detection, async HTTP LLM, history model, agent improvements

- Add async HTTP-based LLM transport (_llm_async_http.py)
- Add orthogonal LLM call tests
- Improve language detection (_lang_detect.py) and batch insert
- Update translate schemas, service utils, preview constants/prompts
- Add TranslateHistoryModel with pagination and filtering
- Update agent confirmation, persistence, langgraph setup, run, tools
- Improve LLM health checking in shared module
- Update translate runs API, history route
This commit is contained in:
2026-07-08 19:35:49 +03:00
parent 97a64ce056
commit c488a63dc0
30 changed files with 1148 additions and 550 deletions

View File

@@ -15,6 +15,7 @@ from typing import Any
from langchain_openai import ChatOpenAI from langchain_openai import ChatOpenAI
from ss_tools.agent._llm_params import chat_openai_kwargs from ss_tools.agent._llm_params import chat_openai_kwargs
from ss_tools.shared._llm_http import get_shared_http_client
from ss_tools.agent._tool_resolver import ( from ss_tools.agent._tool_resolver import (
extract_tool_call_from_state, extract_tool_call_from_state,
find_tool, find_tool,
@@ -287,12 +288,15 @@ async def _format_tool_output_via_llm(
config = await _fetch_llm_config() config = await _fetch_llm_config()
if config and config.get("configured"): if config and config.get("configured"):
try: try:
llm = ChatOpenAI(**chat_openai_kwargs( llm = ChatOpenAI(
model=config.get("default_model", "gpt-4o-mini"), http_client=get_shared_http_client(),
base_url=config.get("base_url", "https://api.openai.com/v1"), **chat_openai_kwargs(
api_key=config["api_key"], model=config.get("default_model", "gpt-4o-mini"),
max_tokens=1024, base_url=config.get("base_url", "https://api.openai.com/v1"),
)) api_key=config["api_key"],
max_tokens=1024,
),
)
prompt = ( prompt = (
f"Tool '{tool_name}' returned this data:\n\n{text}\n\n" f"Tool '{tool_name}' returned this data:\n\n{text}\n\n"
"Summarize this data in a concise, human-readable format. " "Summarize this data in a concise, human-readable format. "

View File

@@ -14,10 +14,9 @@ import re
from typing import Any from typing import Any
import uuid import uuid
import httpx
from ss_tools.agent._config import AGENT_PREFETCH_DASHBOARD_LIMIT as _PREFETCH_LIMIT, FASTAPI_URL, SERVICE_JWT as _SERVICE_JWT from ss_tools.agent._config import AGENT_PREFETCH_DASHBOARD_LIMIT as _PREFETCH_LIMIT, FASTAPI_URL, SERVICE_JWT as _SERVICE_JWT
from ss_tools.agent._llm_params import add_temperature_if_supported from ss_tools.agent._llm_params import add_temperature_if_supported
from ss_tools.shared._llm_http import get_shared_http_client
from ss_tools.shared.logger import logger from ss_tools.shared.logger import logger
SAVE_API_URL = FASTAPI_URL + "/api/agent/conversations/save" SAVE_API_URL = FASTAPI_URL + "/api/agent/conversations/save"
@@ -119,10 +118,10 @@ async def _get_llm_config() -> dict[str, Any] | None:
headers = {"Content-Type": "application/json"} headers = {"Content-Type": "application/json"}
if service_token: if service_token:
headers["Authorization"] = f"Bearer {service_token}" headers["Authorization"] = f"Bearer {service_token}"
async with httpx.AsyncClient(timeout=5) as client: client = get_shared_http_client(timeout=10)
resp = await client.get(f"{fastapi_url}/api/agent/llm-config", headers=headers) resp = await client.get(f"{fastapi_url}/api/agent/llm-config", headers=headers)
if resp.status_code == 200: if resp.status_code == 200:
return resp.json() return resp.json()
except Exception: except Exception:
pass pass
return None return None
@@ -152,10 +151,10 @@ async def _call_llm_for_title(user_text: str) -> str | None:
if base.endswith("/v1"): if base.endswith("/v1"):
base = base[:-3] base = base[:-3]
api_url = base + "/v1/chat/completions" api_url = base + "/v1/chat/completions"
async with httpx.AsyncClient(timeout=10) as client: client = get_shared_http_client(timeout=180)
resp = await client.post(api_url, json=payload, headers=headers) resp = await client.post(api_url, json=payload, headers=headers)
if resp.status_code != 200: if resp.status_code != 200:
return None return None
data = resp.json() data = resp.json()
title = data.get("choices", [{}])[0].get("message", {}).get("content", "") title = data.get("choices", [{}])[0].get("message", {}).get("content", "")
if title: if title:
@@ -188,8 +187,8 @@ async def generate_llm_title(conv_id: str, user_text: str) -> None:
if _SERVICE_JWT: if _SERVICE_JWT:
headers["Authorization"] = f"Bearer {_SERVICE_JWT}" headers["Authorization"] = f"Bearer {_SERVICE_JWT}"
payload = {"conversation_id": conv_id, "title": title, "user_id": "admin", "messages": []} payload = {"conversation_id": conv_id, "title": title, "user_id": "admin", "messages": []}
async with httpx.AsyncClient(timeout=5) as client: client = get_shared_http_client(timeout=10)
await client.post(SAVE_API_URL, json=payload, headers=headers) await client.post(SAVE_API_URL, json=payload, headers=headers)
logger.reflect("LLM title updated", payload={"conv_id": conv_id, "title": title[:40]}, extra={"src": "AgentChat.Persistence"}) logger.reflect("LLM title updated", payload={"conv_id": conv_id, "title": title[:40]}, extra={"src": "AgentChat.Persistence"})
except Exception as e: except Exception as e:
logger.explore("LLM title save failed", payload={"conv_id": conv_id}, error=str(e), extra={"src": "AgentChat.Persistence"}) logger.explore("LLM title save failed", payload={"conv_id": conv_id}, error=str(e), extra={"src": "AgentChat.Persistence"})
@@ -204,14 +203,14 @@ async def generate_llm_title(conv_id: str, user_text: str) -> None:
async def prefetch_dashboards(env_id: str) -> str: async def prefetch_dashboards(env_id: str) -> str:
try: try:
from ss_tools.agent.tools import FASTAPI_URL, _dual_auth_headers from ss_tools.agent.tools import FASTAPI_URL, _dual_auth_headers
async with httpx.AsyncClient(timeout=10) as client: client = get_shared_http_client(timeout=10)
resp = await client.get( resp = await client.get(
f"{FASTAPI_URL}/api/dashboards", f"{FASTAPI_URL}/api/dashboards",
params={"q": "", "env_id": env_id or ""}, params={"q": "", "env_id": env_id or ""},
headers=_dual_auth_headers(), headers=_dual_auth_headers(),
) )
if resp.status_code != 200: if resp.status_code != 200:
return "" return ""
data = resp.json() data = resp.json()
dashboards = data.get("dashboards", []) dashboards = data.get("dashboards", [])
if not dashboards: if not dashboards:
@@ -243,14 +242,14 @@ async def prefetch_dashboards(env_id: str) -> str:
async def prefetch_databases(env_id: str) -> str: async def prefetch_databases(env_id: str) -> str:
try: try:
from ss_tools.agent.tools import FASTAPI_URL, _dual_auth_headers from ss_tools.agent.tools import FASTAPI_URL, _dual_auth_headers
async with httpx.AsyncClient(timeout=10) as client: client = get_shared_http_client(timeout=10)
resp = await client.get( resp = await client.get(
f"{FASTAPI_URL}/api/agent/superset/databases", f"{FASTAPI_URL}/api/agent/superset/databases",
params={"environment_id": env_id or ""}, params={"environment_id": env_id or ""},
headers=_dual_auth_headers(), headers=_dual_auth_headers(),
) )
if resp.status_code != 200: if resp.status_code != 200:
return "" return ""
databases = resp.json() databases = resp.json()
if not databases: if not databases:
return "No databases found." return "No databases found."
@@ -287,8 +286,8 @@ async def save_conversation(conv_id: str, user_text: str, user_id: str = "admin"
if assistant_text: if assistant_text:
messages.append({"id": str(uuid.uuid4()), "conversation_id": conv_id, "role": "assistant", "text": assistant_text.strip(), "state": None, "created_at": datetime.utcnow().isoformat()}) messages.append({"id": str(uuid.uuid4()), "conversation_id": conv_id, "role": "assistant", "text": assistant_text.strip(), "state": None, "created_at": datetime.utcnow().isoformat()})
payload = {"conversation_id": conv_id, "title": clean_title(user_text)[:TITLE_MAX_LENGTH], "user_id": user_id, "messages": messages} payload = {"conversation_id": conv_id, "title": clean_title(user_text)[:TITLE_MAX_LENGTH], "user_id": user_id, "messages": messages}
async with httpx.AsyncClient(timeout=10) as client: client = get_shared_http_client(timeout=10)
await client.post(SAVE_API_URL, json=payload, headers=headers) await client.post(SAVE_API_URL, json=payload, headers=headers)
logger.reflect("Conversation saved", payload={"conv_id": conv_id, "user_id": user_id, "messages": len(messages)}, extra={"src": "AgentChat.Persistence"}) logger.reflect("Conversation saved", payload={"conv_id": conv_id, "user_id": user_id, "messages": len(messages)}, extra={"src": "AgentChat.Persistence"})
except Exception as e: except Exception as e:
logger.explore("Save conversation failed", payload={"conv_id": conv_id}, error=str(e), extra={"src": "AgentChat.Persistence"}) logger.explore("Save conversation failed", payload={"conv_id": conv_id}, error=str(e), extra={"src": "AgentChat.Persistence"})

View File

@@ -10,7 +10,6 @@
import inspect as _inspect import inspect as _inspect
import os import os
import httpx
from langchain_openai import ChatOpenAI from langchain_openai import ChatOpenAI
from langgraph.checkpoint.memory import InMemorySaver from langgraph.checkpoint.memory import InMemorySaver
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
@@ -23,6 +22,7 @@ import pydantic_core as _pydantic_core
from ss_tools.agent._config import AGENT_CONFIRM_TOOLS, AGENT_INTERRUPT_BEFORE as _INTERRUPT_BEFORE, FASTAPI_URL from ss_tools.agent._config import AGENT_CONFIRM_TOOLS, AGENT_INTERRUPT_BEFORE as _INTERRUPT_BEFORE, FASTAPI_URL
from ss_tools.agent._llm_params import chat_openai_kwargs from ss_tools.agent._llm_params import chat_openai_kwargs
from ss_tools.shared._llm_http import get_shared_http_client
from ss_tools.shared.logger import logger from ss_tools.shared.logger import logger
_original_transform = _openai_transform._async_transform_recursive _original_transform = _openai_transform._async_transform_recursive
@@ -90,13 +90,13 @@ async def _fetch_llm_config() -> dict | None:
global _llm_config global _llm_config
try: try:
fastapi_url = FASTAPI_URL fastapi_url = FASTAPI_URL
async with httpx.AsyncClient(timeout=5) as client: client = get_shared_http_client(timeout=10)
resp = await client.get(f"{fastapi_url}/api/agent/llm-config") resp = await client.get(f"{fastapi_url}/api/agent/llm-config")
if resp.status_code == 200: if resp.status_code == 200:
config = resp.json() config = resp.json()
if config.get("configured"): if config.get("configured"):
_llm_config = config _llm_config = config
return config return config
except Exception as e: except Exception as e:
logger.explore("Failed to fetch LLM config from FastAPI", error=str(e), extra={"src": "AgentChat.LangGraph.Setup"}) logger.explore("Failed to fetch LLM config from FastAPI", error=str(e), extra={"src": "AgentChat.LangGraph.Setup"})
return _llm_config return _llm_config
@@ -133,7 +133,10 @@ async def create_agent(tools: list, env_id: str | None = None, interrupt_before:
else: else:
raise RuntimeError("No LLM provider configured in backend. Configure one via Settings → AI Providers in the web UI.") raise RuntimeError("No LLM provider configured in backend. Configure one via Settings → AI Providers in the web UI.")
logger.reason("Creating LangGraph agent", payload={"model": model, "tools_count": len(tools), "env_id": env_id}, extra={"src": "AgentChat.LangGraph.Setup"}) logger.reason("Creating LangGraph agent", payload={"model": model, "tools_count": len(tools), "env_id": env_id}, extra={"src": "AgentChat.LangGraph.Setup"})
llm = ChatOpenAI(**chat_openai_kwargs(model=model, base_url=base_url, api_key=api_key, max_tokens=2048)) llm = ChatOpenAI(
http_client=get_shared_http_client(),
**chat_openai_kwargs(model=model, base_url=base_url, api_key=api_key, max_tokens=2048),
)
prompt = ( prompt = (
"You are a Superset Tools assistant. You have access to tools for searching " "You are a Superset Tools assistant. You have access to tools for searching "
"dashboards, managing maintenance, running migrations and backups, " "dashboards, managing maintenance, running migrations and backups, "

View File

@@ -22,6 +22,7 @@ from ss_tools.agent._config import (
) )
from ss_tools.shared.cot_logger import seed_trace_id from ss_tools.shared.cot_logger import seed_trace_id
from ss_tools.shared.logger import logger from ss_tools.shared.logger import logger
from ss_tools.shared.ssl import httpx_verify
def _find_free_port(start_port: int, max_attempts: int = 100) -> int: def _find_free_port(start_port: int, max_attempts: int = 100) -> int:
@@ -46,9 +47,15 @@ def _fetch_llm_config() -> dict | None:
service_token = SERVICE_JWT service_token = SERVICE_JWT
headers = {"Authorization": f"Bearer {service_token}"} if service_token else {} headers = {"Authorization": f"Bearer {service_token}"} if service_token else {}
ssl_ctx = httpx_verify()
for attempt in range(6): for attempt in range(6):
try: try:
resp = httpx.get(f"{FASTAPI_URL}/api/agent/llm-config", headers=headers, timeout=5) resp = httpx.get(
f"{FASTAPI_URL}/api/agent/llm-config",
headers=headers,
timeout=5,
verify=ssl_ctx,
)
resp.raise_for_status() resp.raise_for_status()
config = resp.json() config = resp.json()
if config.get("configured"): if config.get("configured"):

View File

@@ -19,6 +19,7 @@ from pydantic import BaseModel, Field
from ss_tools.agent._config import FASTAPI_URL, SERVICE_JWT as _SERVICE_JWT from ss_tools.agent._config import FASTAPI_URL, SERVICE_JWT as _SERVICE_JWT
from ss_tools.agent.context import get_service_jwt, get_user_jwt, get_user_role from ss_tools.agent.context import get_service_jwt, get_user_jwt, get_user_role
from ss_tools.shared._llm_http import get_shared_http_client
from ss_tools.shared.logger import logger from ss_tools.shared.logger import logger
TOOL_RESPONSE_LIMIT = 4000 TOOL_RESPONSE_LIMIT = 4000
@@ -249,19 +250,19 @@ async def _execute_with_timeout(tool_name: str, tool_fn, is_write: bool = False,
# @BRIEF Async HTTP GET to FastAPI with dual-auth headers. # @BRIEF Async HTTP GET to FastAPI with dual-auth headers.
async def _get(path: str, params: dict[str, Any] | None = None) -> httpx.Response: async def _get(path: str, params: dict[str, Any] | None = None) -> httpx.Response:
async def _request() -> httpx.Response: async def _request() -> httpx.Response:
async with httpx.AsyncClient(timeout=TOOL_TIMEOUT_SECONDS) as client: client = get_shared_http_client(timeout=TOOL_TIMEOUT_SECONDS)
resp = await client.get( resp = await client.get(
f"{FASTAPI_URL}{path}", f"{FASTAPI_URL}{path}",
params=params, params=params,
headers=_dual_auth_headers(), headers=_dual_auth_headers(),
)
if resp.status_code in {502, 503, 504}:
raise httpx.HTTPStatusError(
f"Transient FastAPI error {resp.status_code}",
request=resp.request,
response=resp,
) )
if resp.status_code in {502, 503, 504}: return resp
raise httpx.HTTPStatusError(
f"Transient FastAPI error {resp.status_code}",
request=resp.request,
response=resp,
)
return resp
try: try:
return await _execute_with_timeout(path, lambda: _retry_read_tool(path, _request), timeout_s=TOOL_TIMEOUT_SECONDS) return await _execute_with_timeout(path, lambda: _retry_read_tool(path, _request), timeout_s=TOOL_TIMEOUT_SECONDS)
@@ -279,13 +280,13 @@ async def _post(
params: dict[str, Any] | None = None, params: dict[str, Any] | None = None,
) -> httpx.Response: ) -> httpx.Response:
async def _request() -> httpx.Response: async def _request() -> httpx.Response:
async with httpx.AsyncClient(timeout=TOOL_TIMEOUT_SECONDS) as client: client = get_shared_http_client(timeout=TOOL_TIMEOUT_SECONDS)
return await client.post( return await client.post(
f"{FASTAPI_URL}{path}", f"{FASTAPI_URL}{path}",
json=payload or {}, json=payload or {},
params=params, params=params,
headers=_dual_auth_headers(), headers=_dual_auth_headers(),
) )
try: try:
return await _execute_with_timeout(path, _request, is_write=True, timeout_s=TOOL_TIMEOUT_SECONDS) return await _execute_with_timeout(path, _request, is_write=True, timeout_s=TOOL_TIMEOUT_SECONDS)

View File

@@ -527,10 +527,13 @@ async def probe_max_images(
if not model: if not model:
raise HTTPException(status_code=400, detail="Provider has no default model set") raise HTTPException(status_code=400, detail="Provider has no default model set")
# Build the client # Build the client with system SSL context
from ss_tools.shared._llm_http import get_shared_http_client
client = AsyncOpenAI( client = AsyncOpenAI(
api_key=api_key, api_key=api_key,
base_url=db_provider.base_url, base_url=db_provider.base_url,
http_client=get_shared_http_client(timeout=180),
) )
def build_content(n_images: int) -> list[dict]: def build_content(n_images: int) -> list[dict]:

View File

@@ -593,6 +593,22 @@ def _ensure_translation_jobs_columns(bind_engine):
extra={"error": str(migration_error)}, extra={"error": str(migration_error)},
) )
if "include_source_reference" not in existing_columns:
try:
with bind_engine.begin() as connection:
connection.execute(
text(
"ALTER TABLE translation_jobs "
"ADD COLUMN include_source_reference BOOLEAN NOT NULL DEFAULT TRUE"
)
)
logger.reflect("Added include_source_reference column to translation_jobs")
except Exception as migration_error:
logger.explore(
"Failed to add include_source_reference to translation_jobs",
extra={"error": str(migration_error)},
)
def _ensure_dataset_review_session_columns(bind_engine): def _ensure_dataset_review_session_columns(bind_engine):
with belief_scope("_ensure_dataset_review_session_columns"): with belief_scope("_ensure_dataset_review_session_columns"):

View File

@@ -28,6 +28,7 @@ def make_job(**overrides) -> TranslationJob:
job.target_schema = "public" job.target_schema = "public"
job.target_table = "translations" job.target_table = "translations"
job.target_languages = ["ru"] job.target_languages = ["ru"]
job.include_source_reference = True
job.context_columns = [] job.context_columns = []
job.target_dialect = "clickhouse" job.target_dialect = "clickhouse"
job.database_dialect = None job.database_dialect = None
@@ -151,6 +152,24 @@ class TestBuildInsertRows:
# Only the original row — language is skipped because en == detected # Only the original row — language is skipped because en == detected
assert len(rows) == 1 assert len(rows) == 1
def test_no_source_reference_skips_matching_language_entirely(self):
"""When source references are disabled, src_lang == target_lang inserts nothing."""
job = make_job(target_key_cols=["id"], include_source_reference=False)
rec = make_record(languages=[make_lang("ru", "Оплачено 22.06", detected="ru")])
rows = _build_insert_rows([rec], job, "translated_text", "ru", [])
assert rows == []
def test_no_source_reference_inserts_only_real_translation(self):
"""Reference row is omitted, non-source translation row remains."""
job = make_job(target_key_cols=["id"], include_source_reference=False)
rec = make_record(languages=[make_lang("ru", "Привет", detected="en")])
rows = _build_insert_rows([rec], job, "translated_text", "ru", [])
assert len(rows) == 1
assert rows[0]["is_original"] == 0
assert rows[0]["lang_code"] == "ru"
def test_source_column_values(self): def test_source_column_values(self):
job = make_job(target_key_cols=["id"]) job = make_job(target_key_cols=["id"])
rec = make_record() rec = make_record()

View File

@@ -47,6 +47,13 @@ async def insert_batch_to_target(
context_keys = _build_context_keys(job, effective_target) context_keys = _build_context_keys(job, effective_target)
rows_for_sql = _build_insert_rows(records, job, effective_target, primary_language, context_keys) rows_for_sql = _build_insert_rows(records, job, effective_target, primary_language, context_keys)
if not rows_for_sql:
logger.reason(
f"Batch {batch_id[:12]} has no rows to insert after language filtering",
{"batch_id": batch_id, "records": len(records)},
)
return
if not columns: if not columns:
columns = [effective_target or "translated_text"] columns = [effective_target or "translated_text"]
rows_for_sql = [{columns[0]: rec.target_sql or ""} for rec in records] rows_for_sql = [{columns[0]: rec.target_sql or ""} for rec in records]
@@ -140,6 +147,7 @@ def _build_insert_rows(
) -> list[dict[str, object]]: ) -> list[dict[str, object]]:
"""Build row dicts for the INSERT SQL statement.""" """Build row dicts for the INSERT SQL statement."""
rows_for_sql: list[dict[str, object]] = [] rows_for_sql: list[dict[str, object]] = []
include_source_reference = getattr(job, "include_source_reference", True)
for rec in records: for rec in records:
source_data = rec.source_data or {} source_data = rec.source_data or {}
detected_src_lang = "und" detected_src_lang = "und"
@@ -166,13 +174,14 @@ def _build_insert_rows(
base_row[job.target_source_language_column] = detected_src_lang base_row[job.target_source_language_column] = detected_src_lang
base_row["context"] = json.dumps(context_data, ensure_ascii=False) base_row["context"] = json.dumps(context_data, ensure_ascii=False)
original_row = dict(base_row) if include_source_reference:
if effective_target: original_row = dict(base_row)
original_row[effective_target] = rec.source_sql or "" if effective_target:
if job.target_language_column: original_row[effective_target] = rec.source_sql or ""
original_row[job.target_language_column] = detected_src_lang if job.target_language_column:
original_row["is_original"] = 1 original_row[job.target_language_column] = detected_src_lang
rows_for_sql.append(original_row) original_row["is_original"] = 1
rows_for_sql.append(original_row)
if rec.languages and len(rec.languages) > 0: if rec.languages and len(rec.languages) > 0:
for lang in rec.languages: for lang in rec.languages:
@@ -187,6 +196,8 @@ def _build_insert_rows(
trans_row["is_original"] = 0 trans_row["is_original"] = 0
rows_for_sql.append(trans_row) rows_for_sql.append(trans_row)
else: else:
if str(primary_language).lower() == str(detected_src_lang).lower():
continue
fallback = dict(base_row) fallback = dict(base_row)
if effective_target: if effective_target:
fallback[effective_target] = rec.target_sql or "" fallback[effective_target] = rec.target_sql or ""

View File

@@ -20,6 +20,7 @@ from lingua import Language, LanguageDetector, LanguageDetectorBuilder
_CYRILLIC_RE = re.compile(r"[\u0400-\u04FF\u0500-\u052F]") _CYRILLIC_RE = re.compile(r"[\u0400-\u04FF\u0500-\u052F]")
# ASCII letter range (Latin-based) # ASCII letter range (Latin-based)
_ASCII_LETTER_RE = re.compile(r"[a-zA-Z]") _ASCII_LETTER_RE = re.compile(r"[a-zA-Z]")
_DIGIT_RE = re.compile(r"\d")
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Build BCP-47 → lingua Language mapping from the lingua Language enum # Build BCP-47 → lingua Language mapping from the lingua Language enum
@@ -181,6 +182,43 @@ def _character_block_fallback(
# #endregion _character_block_fallback # #endregion _character_block_fallback
# #region _mark_suspicious_detections_und [C:2] [TYPE Function] [SEMANTICS language, detection, llm-fallback]
# @BRIEF Downgrade low-signal detections to "und" so LLM can arbitrate source language.
# @RATIONALE lingua is reliable on full sentences, but short accounting strings with dates,
# amounts, or mixed scripts are frequently misclassified (ru text as uk/it/en).
# Returning "und" routes the row through LLM with explicit language-detection
# instructions instead of trusting a brittle local guess.
def _mark_suspicious_detections_und(
texts: list[str],
results: list[str],
target_languages: list[str] | None,
) -> list[str]:
"""Mark suspicious non-target detections as und for LLM-assisted verification."""
if not target_languages or "ru" not in {t.lower() for t in target_languages}:
return results
for i, (text, result) in enumerate(zip(texts, results)):
result_l = (result or "und").lower()
if result_l in ("und", "ru") or not text:
continue
ascii_letters = len(_ASCII_LETTER_RE.findall(text))
cyrillic_letters = len(_CYRILLIC_RE.findall(text))
if cyrillic_letters == 0:
continue
total_letters = ascii_letters + cyrillic_letters
digit_count = len(_DIGIT_RE.findall(text))
short_text = total_letters <= 20
number_heavy = total_letters > 0 and digit_count / total_letters >= 0.35
if short_text or number_heavy:
results[i] = "und"
return results
# #endregion _mark_suspicious_detections_und
# #region batch_detect [C:3] [TYPE Function] [SEMANTICS language, detection, batch] # #region batch_detect [C:3] [TYPE Function] [SEMANTICS language, detection, batch]
# @ingroup Translate # @ingroup Translate
# @BRIEF Detect language for multiple texts in batch. Builds detector if not provided. # @BRIEF Detect language for multiple texts in batch. Builds detector if not provided.
@@ -206,7 +244,7 @@ def batch_detect(
detector = get_detector(target_languages) detector = get_detector(target_languages)
results = [detect_language(t, detector) for t in texts] results = [detect_language(t, detector) for t in texts]
results = _character_block_fallback(texts, results, target_languages) results = _mark_suspicious_detections_und(texts, results, target_languages)
return results return results
# #endregion batch_detect # #endregion batch_detect
# #endregion LanguageDetectService # #endregion LanguageDetectService

View File

@@ -15,254 +15,26 @@
# .ok attribute (only is_success), causes 'Response' object has no attribute 'ok'. # .ok attribute (only is_success), causes 'Response' object has no attribute 'ok'.
import asyncio import asyncio
import os
import ssl
from typing import Any from typing import Any
import httpx import httpx
from ...core.cot_logger import log from ...core.cot_logger import log
from ...core.logger import logger from ...core.logger import logger
from ._utils import _sanitize_url from ss_tools.shared._llm_http import (
get_shared_http_client,
# Module-level httpx client, lazily initialized for connection reuse sanitize_url,
_http_client: httpx.AsyncClient | None = None call_openai_compatible,
)
# Default provider and max_tokens constants # Default provider and max_tokens constants
DEFAULT_PROVIDER_TYPE: str = "openai" DEFAULT_PROVIDER_TYPE: str = "openai"
DEFAULT_MAX_TOKENS: int = 8192 DEFAULT_MAX_TOKENS: int = 8192
# Re-export for backward compatibility with _llm_call.py and preview_executor.py
# #region _get_verify [C:1] [TYPE Function] [SEMANTICS translate, ssl, verify] __all__ = [
# @BRIEF Resolve SSL verification context via centralized core.ssl helper. "call_openai_compatible",
# @RATIONALE Используем capath=/etc/ssl/certs/ через ssl.create_default_context, "DEFAULT_PROVIDER_TYPE",
# потому что OpenSSL 3.x не использует intermediate CA сертификаты из cafile "DEFAULT_MAX_TOKENS",
# для построения цепочки (verify code 20). capath с хеш-симлинками работает ]
# корректно (verify code 0). Возвращаем SSLContext вместо строки — httpx 0.28.x
# депрекейтит строковый путь в verify=.
# @REJECTED cafile отвергнут — OpenSSL 3.x не использует intermediate CA
# из единого bundle-файла. Только capath с хеш-симлинками даёт code 0.
# @REJECTED Строковый путь "/etc/ssl/certs/" отвергнут — httpx 0.28.x депрекейтит
# строки в verify=, требует SSLContext.
# @REJECTED LLM_SSL_VERIFY=false escape hatch отвергнут — централизованный SSL
# всегда использует системное хранилище без возможности отключения.
# @POST Returns ssl.SSLContext with capath (never False).
def _get_verify() -> ssl.SSLContext:
from ...core.ssl import httpx_verify
return httpx_verify()
# #endregion _get_verify
# #region _get_http_client [C:1] [TYPE Function]
# @BRIEF Get or create the module-level httpx.AsyncClient singleton.
# @POST Returns httpx.AsyncClient with SSL verify and 180s timeout.
async def _get_http_client() -> httpx.AsyncClient:
global _http_client
if _http_client is None:
ssl_verify = _get_verify()
_http_client = httpx.AsyncClient(
verify=ssl_verify,
timeout=httpx.Timeout(180.0),
)
return _http_client
# #endregion _get_http_client
# #region call_openai_compatible [C:3] [TYPE Function] [SEMANTICS translate, llm, http, openai, async]
# @ingroup Translate
# @BRIEF Call OpenAI-compatible API asynchronously with rate-limit handling and structured output fallback.
# @PRE Valid API endpoint, key, model, and prompt.
# @POST Returns (response text, finish_reason).
# @SIDE_EFFECT Async HTTP POST to LLM API.
async def call_openai_compatible(
base_url: str,
api_key: str,
model: str,
prompt: str,
provider_type: str = "openai",
max_tokens: int = 8192,
disable_reasoning: bool = False,
) -> tuple[str, str | None]:
"""Call OpenAI-compatible API for batch translation (async)."""
if not base_url:
raise ValueError("LLM provider has no base_url configured")
# Normalise base_url: strip trailing /v1 to avoid double /v1
base = base_url.rstrip("/")
if base.endswith("/v1"):
base = base[:-3]
url = f"{base}/v1/chat/completions"
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
}
system_content = "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics. Respond directly with ONLY the JSON result. Do NOT include any reasoning, thinking, chain-of-thought, analysis, or explanation. Output ONLY valid JSON."
payload: dict[str, Any] = {
"model": model,
"messages": [
{"role": "system", "content": system_content},
{"role": "user", "content": prompt},
],
"temperature": 0.1,
"max_tokens": max_tokens,
}
if provider_type in ("openai", "openai_compatible", "kilo", "openrouter", "litellm"):
if not disable_reasoning:
payload["response_format"] = {"type": "json_object"}
if disable_reasoning:
if provider_type not in ("kilo", "openrouter", "litellm"):
payload["reasoning_effort"] = "none"
payload["max_tokens"] = max_tokens
logger.reason(f"LLM request url={_sanitize_url(base_url)} model={payload.get('model')} provider_type={provider_type} response_format={'yes' if 'response_format' in payload else 'no'} prompt_len={len(prompt)}")
response, response_text = await _do_http_request(url, headers, payload)
await _handle_response_format_fallback(response, response_text, payload, url, headers)
if not response.is_success:
logger.explore(f"LLM API error status={response.status_code} model={payload.get('model')} body={response_text[:2000]}")
response.raise_for_status()
data = response.json()
choices = data.get("choices", [])
if not choices:
logger.explore(
"LLM returned no choices",
extra={
"src": "executor",
"response_keys": list(data.keys()),
"response_preview": str(data)[:2000],
},
)
raise ValueError("LLM returned no choices")
try:
finish_reason = choices[0].get("finish_reason") or "none"
msg = choices[0].get("message") or {}
except (TypeError, AttributeError) as e:
logger.explore(
"TypeError processing LLM response choices",
extra={
"src": "executor_diag",
"error": str(e),
"choices_0_type": type(choices[0]).__name__ if choices else "N/A",
"choices_0_repr": repr(choices[0])[:2000] if choices else "N/A",
"data_type": type(data).__name__,
"data_preview": str(data)[:2000],
},
)
raise ValueError(f"LLM response processing failed: {e}")
# Log provider token usage for batch sizing calibration
usage = data.get("usage") or {}
if usage:
logger.reason(
"LLM provider usage",
{
"prompt_tokens": usage.get("prompt_tokens"),
"completion_tokens": usage.get("completion_tokens"),
"total_tokens": usage.get("total_tokens"),
"finish_reason": finish_reason,
"max_tokens_sent": max_tokens,
"chars_sent": len(prompt),
},
)
refusal = msg.get("refusal") if isinstance(msg, dict) else None
if refusal:
logger.explore(
"LLM refused to respond",
extra={
"src": "executor",
"refusal": str(refusal)[:500],
"finish_reason": finish_reason,
},
)
raise ValueError(f"LLM refused to respond: {refusal}")
content = msg.get("content") if isinstance(msg, dict) else ""
if not content and isinstance(msg, dict):
content = msg.get("content") or ""
logger.reason(f"LLM response finish_reason={finish_reason} content_len={len(content)} msg_keys={list(msg.keys()) if isinstance(msg, dict) else []}")
if not content:
logger.explore(
"LLM returned empty content",
extra={
"src": "executor",
"finish_reason": finish_reason,
"msg_keys": list(msg.keys()) if isinstance(msg, dict) else [],
"response_preview": str(data)[:2000],
},
)
raise ValueError("LLM returned empty content")
return content, finish_reason
# #endregion call_openai_compatible
# #region _do_http_request [C:1] [TYPE Function]
async def _do_http_request(url: str, headers: dict, payload: dict) -> tuple[httpx.Response, str]:
"""Make async HTTP POST with rate-limit (429) retry handling."""
client = await _get_http_client()
_max_retry_429 = 3
_retry_count_429 = 0
while _retry_count_429 < _max_retry_429:
response = await client.post(url, headers=headers, json=payload)
response_text = response.text
if response.status_code == 429:
_retry_count_429 += 1
retry_after = response.headers.get("Retry-After")
if retry_after:
try:
wait = int(retry_after)
except (ValueError, TypeError):
wait = 2**_retry_count_429
else:
wait = 2**_retry_count_429
logger.explore(f"Rate limited (429), retry {_retry_count_429}/{_max_retry_429} after {wait}s", extra={"src": "executor", "retry_after": retry_after, "wait": wait})
await asyncio.sleep(wait)
if _retry_count_429 >= _max_retry_429:
break
else:
break
return response, response_text
# #endregion _do_http_request
# #region _handle_response_format_fallback [C:1] [TYPE Function]
async def _handle_response_format_fallback(
response: httpx.Response,
response_text: str,
payload: dict,
url: str,
headers: dict,
) -> None:
"""Handle 400 errors from structured_outputs not being supported."""
_patterns = ("response_format", "structured_outputs", "structured", "json_object")
if not response.is_success and response.status_code == 400 and any(p in (response_text or "").lower() for p in _patterns):
client = await _get_http_client()
logger.explore("Structured outputs not supported, retrying without response_format", extra={"src": "executor"})
payload.pop("response_format", None)
new_response = await client.post(url, headers=headers, json=payload)
# Mutate the original response object with new data
response.status_code = new_response.status_code
response._content = new_response.content
response.encoding = new_response.encoding
response.headers = new_response.headers
# #endregion _handle_response_format_fallback
# #endregion LLMAsyncHttpClient # #endregion LLMAsyncHttpClient

View File

@@ -3,8 +3,8 @@
# @BRIEF LLM interaction for batch translation: call provider with retry, handle truncation # @BRIEF LLM interaction for batch translation: call provider with retry, handle truncation
# by recursive splitting, enforce dictionary post-processing. Orchestrates HTTP calls # by recursive splitting, enforce dictionary post-processing. Orchestrates HTTP calls
# (_llm_http) and response parsing (_llm_parse). # (_llm_http) and response parsing (_llm_parse).
# Language detection is now handled locally (lingua) — LLM prompt no longer # Language detection is handled locally first; ambiguous rows keep "und" and
# requests detected_source_language; local _detected_lang takes priority. # the LLM response supplies detected_source_language for arbitration.
# @LAYER Domain # @LAYER Domain
# @RELATION DEPENDS_ON -> [LLMProviderService] # @RELATION DEPENDS_ON -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [TranslationRecord], [TranslationLanguage] # @RELATION DEPENDS_ON -> [TranslationRecord], [TranslationLanguage]
@@ -167,20 +167,37 @@ class LLMTranslationService:
# #region _build_prompt [C:2] [TYPE Function] [SEMANTICS translate, llm, prompt] # #region _build_prompt [C:2] [TYPE Function] [SEMANTICS translate, llm, prompt]
# @BRIEF Build the full LLM prompt from batch rows, dictionary, and target languages. # @BRIEF Build the full LLM prompt from batch rows, dictionary, and target languages.
# Context columns from job.context_columns are included in rows_json when configured.
@staticmethod @staticmethod
def _build_prompt(job, batch_rows, dictionary_section, target_languages): def _build_prompt(job, batch_rows, dictionary_section, target_languages):
target_languages_str = ", ".join(target_languages) target_languages_str = ", ".join(target_languages)
context_cols = job.context_columns or []
context_hint = ""
if context_cols:
context_hint = (
f"Context columns: {', '.join(context_cols)}\n"
"Consider these context fields when determining the meaning of the text.\n\n"
)
rows_json = json.dumps([ rows_json = json.dumps([
{"row_id": str(row.get("row_index", idx)), "text": row.get("source_text", "")} {
"row_id": str(row.get("row_index", idx)),
"text": row.get("source_text", ""),
"detected_source_language": row.get("_detected_lang", "und") or "und",
**({"context": {
col: str((row.get("source_data") or {}).get(col) or "")
for col in context_cols
}} if context_cols else {})
}
for idx, row in enumerate(batch_rows) for idx, row in enumerate(batch_rows)
], indent=2) ], indent=2)
return render_prompt(DEFAULT_EXECUTION_PROMPT_TEMPLATE, { return render_prompt(DEFAULT_EXECUTION_PROMPT_TEMPLATE, {
"source_language": job.source_dialect or "SQL",
"target_language": target_languages_str,
"target_languages": target_languages_str, "target_languages": target_languages_str,
"source_dialect": job.source_dialect or "", "target_language": target_languages_str,
"target_dialect": job.target_dialect or "",
"translation_column": job.translation_column or "", "translation_column": job.translation_column or "",
"context_hint": context_hint,
"dictionary_section": dictionary_section, "dictionary_section": dictionary_section,
"rows_json": rows_json, "rows_json": rows_json,
"row_count": str(len(batch_rows)), "row_count": str(len(batch_rows)),
@@ -339,12 +356,14 @@ class LLMTranslationService:
self.db.add(record) self.db.add(record)
for lang_code in target_languages: for lang_code in target_languages:
if detected_lang != "und" and str(lang_code).lower() == str(detected_lang).lower(): if detected_lang != "und" and str(lang_code).lower() == str(detected_lang).lower():
continue val = source_text
val = plv.get(lang_code, "") else:
val = plv.get(lang_code, "")
needs_review = (detected_lang == "und")
self.db.add(TranslationLanguage( self.db.add(TranslationLanguage(
id=str(uuid.uuid4()), record_id=record.id, language_code=lang_code, id=str(uuid.uuid4()), record_id=record.id, language_code=lang_code,
source_language_detected=detected_lang, translated_value=val or "", source_language_detected=detected_lang, translated_value=val or "",
final_value=val or "", status="translated", needs_review=(detected_lang == "und"), final_value=val or "", status="translated", needs_review=needs_review,
)) ))
logger.reason( logger.reason(
@@ -440,8 +459,9 @@ class LLMTranslationService:
self.db.add(record) self.db.add(record)
for lang_code in target_languages: for lang_code in target_languages:
if detected_lang != "und" and str(lang_code).lower() == str(detected_lang).lower(): if detected_lang != "und" and str(lang_code).lower() == str(detected_lang).lower():
continue val = source_text
val = plv.get(lang_code, "") else:
val = plv.get(lang_code, "")
needs_review = (detected_lang == "und") needs_review = (detected_lang == "und")
if needs_review: if needs_review:
logger.explore("undetected language", {"record_id": row_id, "language_code": lang_code, "text": source_text[:100]}) logger.explore("undetected language", {"record_id": row_id, "language_code": lang_code, "text": source_text[:100]})

View File

@@ -62,6 +62,7 @@ def build_rows(
) -> list[dict[str, object]]: ) -> list[dict[str, object]]:
"""Build row data for SQL INSERT with per-language expansion.""" """Build row data for SQL INSERT with per-language expansion."""
rows_for_sql: list[dict[str, object]] = [] rows_for_sql: list[dict[str, object]] = []
include_source_reference = getattr(job, "include_source_reference", True)
for rec in records: for rec in records:
source_data = rec.source_data or {} source_data = rec.source_data or {}
detected_src_lang = "und" detected_src_lang = "und"
@@ -84,13 +85,14 @@ def build_rows(
base_row[job.target_source_language_column] = detected_src_lang base_row[job.target_source_language_column] = detected_src_lang
base_row["context"] = json.dumps(context_data, ensure_ascii=False) base_row["context"] = json.dumps(context_data, ensure_ascii=False)
original_row = dict(base_row) if include_source_reference:
if effective_target: original_row = dict(base_row)
original_row[effective_target] = rec.source_sql or "" if effective_target:
if job.target_language_column: original_row[effective_target] = rec.source_sql or ""
original_row[job.target_language_column] = detected_src_lang if job.target_language_column:
original_row["is_original"] = 1 original_row[job.target_language_column] = detected_src_lang
rows_for_sql.append(original_row) original_row["is_original"] = 1
rows_for_sql.append(original_row)
if rec.languages and len(rec.languages) > 0: if rec.languages and len(rec.languages) > 0:
for lang in rec.languages: for lang in rec.languages:
@@ -105,6 +107,8 @@ def build_rows(
trans_row["is_original"] = 0 trans_row["is_original"] = 0
rows_for_sql.append(trans_row) rows_for_sql.append(trans_row)
else: else:
if str(primary_language).lower() == str(detected_src_lang).lower():
continue
fallback_row = dict(base_row) fallback_row = dict(base_row)
if effective_target: if effective_target:
fallback_row[effective_target] = rec.target_sql or "" fallback_row[effective_target] = rec.target_sql or ""

View File

@@ -1,23 +1,26 @@
# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [C:1] [TYPE Block] [SEMANTICS prompt, template] # #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [C:1] [TYPE Block] [SEMANTICS prompt, template]
# @BRIEF Default LLM prompt template for full execution (batch translation). # @BRIEF Default LLM prompt template for full execution (batch translation).
# Language detection is handled locally (lingua) — prompt does not request # Language detection is handled locally first; ambiguous/und rows ask the LLM
# detected_source_language, saving LLM tokens. # to return detected_source_language for source-language arbitration.
# Supports both single-language and multi-language modes via {target_languages} placeholder. # Supports both single-language and multi-language modes via {target_languages} placeholder.
DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = ( DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
"Translate the following database content from {source_language} to the following language(s): {target_languages}.\n\n" "Translate the following content to the following language(s): {target_languages}.\n\n"
"Source dialect: {source_dialect}\n"
"Target dialect(s): {target_dialect}\n"
"Column to translate: {translation_column}\n\n" "Column to translate: {translation_column}\n\n"
"{context_hint}"
"{dictionary_section}" "{dictionary_section}"
"IMPORTANT — You MUST use the terminology dictionary above. " "IMPORTANT — You MUST use the terminology dictionary above. "
"For any source term listed in the dictionary, you MUST use its exact target translation. " "For any source term listed in the dictionary, you MUST use its exact target translation. "
"Do not translate dictionary terms differently. " "Do not translate dictionary terms differently. "
"This is mandatory, not optional.\n\n" "This is mandatory, not optional.\n\n"
"For each row, provide an accurate translation of the text into each target language.\n\n" "For each row, provide an accurate translation of the text into each target language.\n"
"Each row contains detected_source_language from local detection. "
"If it is 'und', determine the source language yourself and return it as detected_source_language. "
"If the source language is the same as a target language, return the original text for that target language.\n\n"
"Rows to translate:\n{rows_json}\n\n" "Rows to translate:\n{rows_json}\n\n"
"Respond with a JSON object in this exact format:\n" "Respond with a JSON object in this exact format:\n"
'{{"rows": [{{"row_id": "<row_index>", "<language_code_1>": "<translation_in_lang_1>", "<language_code_2>": "<translation_in_lang_2>"}}]}}\n' '{{"rows": [{{"row_id": "<row_index>", "detected_source_language": "<bcp47_or_und>", "<language_code_1>": "<translation_in_lang_1>", "<language_code_2>": "<translation_in_lang_2>"}}]}}\n'
"Include detected_source_language for EVERY row. "
"Include a separate key for EACH target language code with the translated text in that language.\n" "Include a separate key for EACH target language code with the translated text in that language.\n"
"Each row_id must match the index provided. Return exactly {row_count} entries." "Each row_id must match the index provided. Return exactly {row_count} entries."
) )
@@ -28,8 +31,7 @@ DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
# @BRIEF Default LLM prompt template for preview (sample translation). # @BRIEF Default LLM prompt template for preview (sample translation).
DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = ( DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
"Translate the following database content.\n\n" "Translate the following content.\n\n"
"Source dialect: {source_dialect}\n"
"Column to translate: {translation_column}\n\n" "Column to translate: {translation_column}\n\n"
"{dictionary_section}" "{dictionary_section}"
"IMPORTANT — You MUST use the terminology dictionary above. " "IMPORTANT — You MUST use the terminology dictionary above. "

View File

@@ -92,10 +92,8 @@ class PreviewPromptBuilder:
target_languages_str = ", ".join(target_languages) target_languages_str = ", ".join(target_languages)
template = prompt_template or DEFAULT_PREVIEW_PROMPT_TEMPLATE template = prompt_template or DEFAULT_PREVIEW_PROMPT_TEMPLATE
prompt = render_prompt(template, { prompt = render_prompt(template, {
"source_language": job.source_dialect or "SQL",
"target_language": target_languages_str,
"source_dialect": job.source_dialect or "",
"target_languages": target_languages_str, "target_languages": target_languages_str,
"target_language": target_languages_str,
"translation_column": job.translation_column or "", "translation_column": job.translation_column or "",
"dictionary_section": dictionary_section, "dictionary_section": dictionary_section,
"rows_json": rows_json, "rows_json": rows_json,

View File

@@ -54,6 +54,7 @@ def job_to_response(job: TranslationJob, dict_ids: list[str] | None = None) -> T
target_language_column=job.target_language_column, target_language_column=job.target_language_column,
target_source_column=job.target_source_column, target_source_column=job.target_source_column,
target_source_language_column=job.target_source_language_column, target_source_language_column=job.target_source_language_column,
include_source_reference=job.include_source_reference,
context_columns=job.context_columns or [], context_columns=job.context_columns or [],
target_languages=job.target_languages, target_languages=job.target_languages,
provider_id=job.provider_id, provider_id=job.provider_id,

View File

@@ -32,8 +32,8 @@ def _validate_bcp47_list(v: list[str] | None) -> list[str] | None:
class TranslateJobCreate(BaseModel): class TranslateJobCreate(BaseModel):
name: str name: str
description: str | None = None description: str | None = None
source_dialect: str = Field(..., description="Source database dialect (e.g. postgresql, clickhouse)") source_dialect: str = Field("", description="Source database dialect (e.g. postgresql, clickhouse) — auto-filled from database_dialect if empty")
target_dialect: str = Field(..., description="Target database dialect (e.g. postgresql, clickhouse)") target_dialect: str = Field("", description="Target database dialect (e.g. postgresql, clickhouse) — auto-filled from database_dialect if empty")
database_dialect: str | None = Field(None, description="Detected dialect from Superset connection at save time") database_dialect: str | None = Field(None, description="Detected dialect from Superset connection at save time")
source_datasource_id: str | None = Field(None, description="Superset datasource ID") source_datasource_id: str | None = Field(None, description="Superset datasource ID")
source_table: str | None = Field(None, description="Source table name") source_table: str | None = Field(None, description="Source table name")
@@ -46,6 +46,7 @@ class TranslateJobCreate(BaseModel):
target_language_column: str | None = Field(None, description="Target column for language code (e.g. 'ru', 'en')") target_language_column: str | None = Field(None, description="Target column for language code (e.g. 'ru', 'en')")
target_source_column: str | None = Field(None, description="Target column for source/original text") target_source_column: str | None = Field(None, description="Target column for source/original text")
target_source_language_column: str | None = Field(None, description="Target column for detected source language (BCP-47)") target_source_language_column: str | None = Field(None, description="Target column for detected source language (BCP-47)")
include_source_reference: bool = Field(True, description="If true, insert original/source rows alongside translations")
context_columns: list[str] | None = Field(default_factory=list, description="Context column names") context_columns: list[str] | None = Field(default_factory=list, description="Context column names")
target_language: str | None = Field(None, description="Target language code [DEPRECATED: use target_languages]") target_language: str | None = Field(None, description="Target language code [DEPRECATED: use target_languages]")
target_languages: list[str] | None = Field(default_factory=list, description="List of BCP-47 target language codes") target_languages: list[str] | None = Field(default_factory=list, description="List of BCP-47 target language codes")
@@ -90,6 +91,7 @@ class TranslateJobUpdate(BaseModel):
target_language_column: str | None = None target_language_column: str | None = None
target_source_column: str | None = None target_source_column: str | None = None
target_source_language_column: str | None = None target_source_language_column: str | None = None
include_source_reference: bool | None = None
context_columns: list[str] | None = None context_columns: list[str] | None = None
target_language: str | None = None target_language: str | None = None
target_languages: list[str] | None = None target_languages: list[str] | None = None
@@ -129,6 +131,7 @@ class TranslateJobResponse(BaseModel):
target_language_column: str | None = None target_language_column: str | None = None
target_source_column: str | None = None target_source_column: str | None = None
target_source_language_column: str | None = None target_source_language_column: str | None = None
include_source_reference: bool = True
context_columns: list[str] | None = None context_columns: list[str] | None = None
# source_language removed — deprecated, per-row auto-detected # source_language removed — deprecated, per-row auto-detected
target_languages: list[str] | None = None target_languages: list[str] | None = None

View File

@@ -17,6 +17,7 @@ from src.plugins.translate._lang_detect import (
get_detector, get_detector,
detect_language, detect_language,
_character_block_fallback, _character_block_fallback,
_mark_suspicious_detections_und,
batch_detect, batch_detect,
) )
@@ -184,6 +185,39 @@ class TestCharacterBlockFallback:
assert results[0] == "und" assert results[0] == "und"
class TestSuspiciousDetections:
"""Suspicious short Cyrillic detections are routed to LLM as und."""
def test_short_cyrillic_uk_for_ru_target_becomes_und(self):
results = _mark_suspicious_detections_und(
["Оплачено 22.06"],
["uk"],
["ru"],
)
assert results == ["und"]
def test_short_cyrillic_it_for_ru_target_becomes_und(self):
results = _mark_suspicious_detections_und(
["Нет"],
["it"],
["ru"],
)
assert results == ["und"]
def test_long_mixed_text_keeps_detection(self):
text = "Contract with KZ Trading for Financing in China Необходимо подтверждение депозита"
results = _mark_suspicious_detections_und([text], ["en"], ["ru"])
assert results == ["en"]
def test_non_ru_target_keeps_detection(self):
results = _mark_suspicious_detections_und(
["Оплачено 22.06"],
["uk"],
["en"],
)
assert results == ["uk"]
class TestBatchDetect: class TestBatchDetect:
"""batch_detect.""" """batch_detect."""

View File

@@ -19,7 +19,7 @@ import pytest
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_llm_rate_limit_uses_asyncio_sleep(): async def test_llm_rate_limit_uses_asyncio_sleep():
"""Verify call_openai_compatible retries on 429 using asyncio.sleep, not time.sleep.""" """Verify call_openai_compatible retries on 429 using asyncio.sleep, not time.sleep."""
from src.plugins.translate._llm_async_http import call_openai_compatible from ss_tools.shared._llm_http import call_openai_compatible
call_count = 0 call_count = 0
@@ -30,6 +30,7 @@ async def test_llm_rate_limit_uses_asyncio_sleep():
resp = MagicMock(spec=httpx.Response) resp = MagicMock(spec=httpx.Response)
resp.status_code = 429 resp.status_code = 429
resp.ok = False resp.ok = False
resp.is_success = False
resp.text = "Rate limited" resp.text = "Rate limited"
resp.headers = {"Retry-After": "1"} resp.headers = {"Retry-After": "1"}
resp.json.return_value = {} resp.json.return_value = {}
@@ -37,6 +38,7 @@ async def test_llm_rate_limit_uses_asyncio_sleep():
resp = MagicMock(spec=httpx.Response) resp = MagicMock(spec=httpx.Response)
resp.status_code = 200 resp.status_code = 200
resp.ok = True resp.ok = True
resp.is_success = True
resp.text = '{"choices":[{"message":{"content":"{\\"translated\\": \\"hola\\"}"},"finish_reason":"stop"}]}' resp.text = '{"choices":[{"message":{"content":"{\\"translated\\": \\"hola\\"}"},"finish_reason":"stop"}]}'
resp.json.return_value = { resp.json.return_value = {
"choices": [{"message": {"content": '{"translated": "hola"}'}, "finish_reason": "stop"}] "choices": [{"message": {"content": '{"translated": "hola"}'}, "finish_reason": "stop"}]
@@ -46,7 +48,7 @@ async def test_llm_rate_limit_uses_asyncio_sleep():
mock_client = AsyncMock() mock_client = AsyncMock()
mock_client.post = AsyncMock(side_effect=mock_post_side_effect) mock_client.post = AsyncMock(side_effect=mock_post_side_effect)
with patch("src.plugins.translate._llm_async_http._get_http_client", return_value=mock_client): with patch("ss_tools.shared._llm_http.get_shared_http_client", return_value=mock_client):
# Patch asyncio.sleep to verify it's called (not time.sleep) # Patch asyncio.sleep to verify it's called (not time.sleep)
sleep_calls = [] sleep_calls = []
original_sleep = asyncio.sleep original_sleep = asyncio.sleep
@@ -55,7 +57,7 @@ async def test_llm_rate_limit_uses_asyncio_sleep():
sleep_calls.append(seconds) sleep_calls.append(seconds)
await original_sleep(0) # Don't actually wait await original_sleep(0) # Don't actually wait
with patch("src.plugins.translate._llm_async_http.asyncio.sleep", side_effect=tracking_sleep): with patch("ss_tools.shared._llm_http.asyncio.sleep", side_effect=tracking_sleep):
content, finish_reason = await call_openai_compatible( content, finish_reason = await call_openai_compatible(
base_url="http://fake-llm.local", base_url="http://fake-llm.local",
api_key="test-key", api_key="test-key",
@@ -75,7 +77,7 @@ async def test_llm_rate_limit_uses_asyncio_sleep():
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_call_openai_compatible_no_base_url(): async def test_call_openai_compatible_no_base_url():
"""Verify empty base_url raises ValueError immediately.""" """Verify empty base_url raises ValueError immediately."""
from src.plugins.translate._llm_async_http import call_openai_compatible from ss_tools.shared._llm_http import call_openai_compatible
with pytest.raises(ValueError, match="no base_url"): with pytest.raises(ValueError, match="no base_url"):
await call_openai_compatible( await call_openai_compatible(
@@ -84,6 +86,8 @@ async def test_call_openai_compatible_no_base_url():
model="gpt-4", model="gpt-4",
prompt="hello", prompt="hello",
) )
# #endregion test_call_openai_compatible_no_base_url # #endregion test_call_openai_compatible_no_base_url
@@ -93,24 +97,29 @@ async def test_call_openai_compatible_no_base_url():
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_call_openai_compatible_empty_choices(): async def test_call_openai_compatible_empty_choices():
"""Verify LLM response with empty choices raises ValueError.""" """Verify LLM response with empty choices raises ValueError."""
from src.plugins.translate._llm_async_http import call_openai_compatible from ss_tools.shared._llm_http import call_openai_compatible
mock_response = MagicMock(spec=httpx.Response) mock_response = MagicMock(spec=httpx.Response)
mock_response.status_code = 200 mock_response.status_code = 200
mock_response.ok = True mock_response.is_success = True
mock_response.json.return_value = {"choices": []} mock_response.json.return_value = {"choices": []}
mock_response.text = '{"choices": []}'
mock_client = AsyncMock() mock_client = AsyncMock()
mock_client.post = AsyncMock(return_value=mock_response) mock_client.post = AsyncMock(return_value=mock_response)
with patch("src.plugins.translate._llm_async_http._get_http_client", return_value=mock_client): with patch("ss_tools.shared._llm_http.get_shared_http_client", return_value=mock_client):
with pytest.raises(ValueError, match="no choices"): with patch("ss_tools.shared._llm_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))):
await call_openai_compatible( with patch("ss_tools.shared._llm_http._handle_response_format_fallback", AsyncMock()):
base_url="http://fake-llm.local", with pytest.raises(ValueError, match="no choices"):
api_key="test-key", await call_openai_compatible(
model="gpt-4", base_url="http://fake-llm.local",
prompt="hello", api_key="test-key",
) model="gpt-4",
prompt="hello",
)
# #endregion test_call_openai_compatible_empty_choices # #endregion test_call_openai_compatible_empty_choices
@@ -120,26 +129,31 @@ async def test_call_openai_compatible_empty_choices():
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_call_openai_compatible_refusal(): async def test_call_openai_compatible_refusal():
"""Verify LLM refusal response raises ValueError.""" """Verify LLM refusal response raises ValueError."""
from src.plugins.translate._llm_async_http import call_openai_compatible from ss_tools.shared._llm_http import call_openai_compatible
mock_response = MagicMock(spec=httpx.Response) mock_response = MagicMock(spec=httpx.Response)
mock_response.status_code = 200 mock_response.status_code = 200
mock_response.ok = True mock_response.is_success = True
mock_response.json.return_value = { mock_response.json.return_value = {
"choices": [{"message": {"content": "", "refusal": "I cannot translate this"}, "finish_reason": "stop"}] "choices": [{"message": {"content": "", "refusal": "I cannot translate this"}, "finish_reason": "stop"}]
} }
mock_response.text = '{"choices": [{"message": {"content": "", "refusal": "I cannot translate this"}, "finish_reason": "stop"}]}'
mock_client = AsyncMock() mock_client = AsyncMock()
mock_client.post = AsyncMock(return_value=mock_response) mock_client.post = AsyncMock(return_value=mock_response)
with patch("src.plugins.translate._llm_async_http._get_http_client", return_value=mock_client): with patch("ss_tools.shared._llm_http.get_shared_http_client", return_value=mock_client):
with pytest.raises(ValueError, match="refused"): with patch("ss_tools.shared._llm_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))):
await call_openai_compatible( with patch("ss_tools.shared._llm_http._handle_response_format_fallback", AsyncMock()):
base_url="http://fake-llm.local", with pytest.raises(ValueError, match="refused"):
api_key="test-key", await call_openai_compatible(
model="gpt-4", base_url="http://fake-llm.local",
prompt="hello", api_key="test-key",
) model="gpt-4",
prompt="hello",
)
# #endregion test_call_openai_compatible_refusal # #endregion test_call_openai_compatible_refusal
@@ -149,7 +163,7 @@ async def test_call_openai_compatible_refusal():
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_retry_does_not_block_event_loop(): async def test_retry_does_not_block_event_loop():
"""Verify that 429 retry backoff allows other coroutines to run.""" """Verify that 429 retry backoff allows other coroutines to run."""
from src.plugins.translate._llm_async_http import _do_http_request from ss_tools.shared._llm_http import _do_http_request
concurrent_ran = False concurrent_ran = False
@@ -166,25 +180,24 @@ async def test_retry_does_not_block_event_loop():
if call_count == 1: if call_count == 1:
resp = MagicMock(spec=httpx.Response) resp = MagicMock(spec=httpx.Response)
resp.status_code = 429 resp.status_code = 429
resp.ok = False resp.is_success = False
resp.text = "Rate limited" resp.text = "Rate limited"
resp.headers = {"Retry-After": "1"} resp.headers = {"Retry-After": "1"}
return resp return resp
resp = MagicMock(spec=httpx.Response) resp = MagicMock(spec=httpx.Response)
resp.status_code = 200 resp.status_code = 200
resp.ok = True resp.is_success = True
resp.text = "ok" resp.text = "ok"
return resp return resp
mock_client = AsyncMock() mock_client = AsyncMock()
mock_client.post = AsyncMock(side_effect=mock_post) mock_client.post = AsyncMock(side_effect=mock_post)
with patch("src.plugins.translate._llm_async_http._get_http_client", return_value=mock_client): with patch("ss_tools.shared._llm_http.asyncio.sleep", new_callable=AsyncMock) as mock_sleep:
with patch("src.plugins.translate._llm_async_http.asyncio.sleep", new_callable=AsyncMock) as mock_sleep: # Run the HTTP request alongside a concurrent task
# Run the HTTP request alongside a concurrent task http_task = asyncio.create_task(_do_http_request(mock_client, "http://fake", {}, {}))
http_task = asyncio.create_task(_do_http_request("http://fake", {}, {})) conc_task = asyncio.create_task(concurrent_task())
conc_task = asyncio.create_task(concurrent_task()) await asyncio.gather(http_task, conc_task)
await asyncio.gather(http_task, conc_task)
assert concurrent_ran, "Concurrent task was blocked by 429 retry" assert concurrent_ran, "Concurrent task was blocked by 429 retry"
assert mock_sleep.called, "asyncio.sleep was not called during 429 retry" assert mock_sleep.called, "asyncio.sleep was not called during 429 retry"

View File

@@ -17,9 +17,9 @@ from unittest.mock import AsyncMock, MagicMock, patch
import httpx import httpx
from src.plugins.translate._llm_async_http import ( from ss_tools.shared.ssl import httpx_verify, system_ssl_context
_get_verify, from ss_tools.shared._llm_http import get_shared_http_client
_get_http_client, from ss_tools.shared._llm_http import (
call_openai_compatible, call_openai_compatible,
_do_http_request, _do_http_request,
_handle_response_format_fallback, _handle_response_format_fallback,
@@ -27,34 +27,37 @@ from src.plugins.translate._llm_async_http import (
class TestGetVerify: class TestGetVerify:
"""_get_verify — SSL verification context.""" """SSL verification context (now from shared module)."""
def test_returns_ssl_context(self): def test_returns_ssl_context(self):
"""Always returns SSLContext (no env-based disable).""" """httpx_verify returns SSLContext (no env-based disable)."""
result = _get_verify() result = httpx_verify()
assert isinstance(result, ssl.SSLContext) assert isinstance(result, ssl.SSLContext)
def test_system_context_not_false(self):
"""system_ssl_context never returns False."""
result = system_ssl_context()
assert result is not False
def test_ignores_llm_ssl_verify_env(self): def test_ignores_llm_ssl_verify_env(self):
"""LLM_SSL_VERIFY env is no longer read — central ssl helper is used.""" """LLM_SSL_VERIFY env is no longer read — central ssl helper is used."""
with patch.dict(os.environ, {"LLM_SSL_VERIFY": "false"}): with patch.dict(os.environ, {"LLM_SSL_VERIFY": "false"}):
result = _get_verify() result = system_ssl_context()
assert isinstance(result, ssl.SSLContext), "must not return False" assert isinstance(result, ssl.SSLContext), "must not return False"
class TestGetHttpClient: class TestGetHttpClient:
"""_get_http_client — module-level singleton.""" """get_shared_http_client — module-level singleton."""
@pytest.mark.asyncio def test_creates_client(self):
async def test_creates_client(self):
"""Creates client on first call.""" """Creates client on first call."""
client = await _get_http_client() client = get_shared_http_client(timeout=180)
assert isinstance(client, httpx.AsyncClient) assert isinstance(client, httpx.AsyncClient)
@pytest.mark.asyncio def test_reuses_client(self):
async def test_reuses_client(self):
"""Returns same client on second call.""" """Returns same client on second call."""
client1 = await _get_http_client() client1 = get_shared_http_client(timeout=180)
client2 = await _get_http_client() client2 = get_shared_http_client(timeout=180)
assert client1 is client2 assert client1 is client2
@@ -84,8 +87,8 @@ class TestCallOpenaiCompatible:
} }
mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"content": "Hello, world!"}}]}' mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"content": "Hello, world!"}}]}'
with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): with patch("ss_tools.shared._llm_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))):
with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): with patch("ss_tools.shared._llm_http._handle_response_format_fallback", AsyncMock()):
content, finish_reason = await call_openai_compatible( content, finish_reason = await call_openai_compatible(
"https://api.openai.com", "https://api.openai.com",
"sk-test", "sk-test",
@@ -104,8 +107,8 @@ class TestCallOpenaiCompatible:
mock_response.json.return_value = {"choices": []} mock_response.json.return_value = {"choices": []}
mock_response.text = '{"choices": []}' mock_response.text = '{"choices": []}'
with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): with patch("ss_tools.shared._llm_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))):
with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): with patch("ss_tools.shared._llm_http._handle_response_format_fallback", AsyncMock()):
with pytest.raises(ValueError, match="no choices"): with pytest.raises(ValueError, match="no choices"):
await call_openai_compatible( await call_openai_compatible(
"https://api.openai.com", "https://api.openai.com",
@@ -130,8 +133,8 @@ class TestCallOpenaiCompatible:
} }
mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"content": ""}}]}' mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"content": ""}}]}'
with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): with patch("ss_tools.shared._llm_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))):
with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): with patch("ss_tools.shared._llm_http._handle_response_format_fallback", AsyncMock()):
with pytest.raises(ValueError, match="empty content"): with pytest.raises(ValueError, match="empty content"):
await call_openai_compatible( await call_openai_compatible(
"https://api.openai.com", "https://api.openai.com",
@@ -151,8 +154,8 @@ class TestCallOpenaiCompatible:
} }
mock_response.text = '{"choices": [null]}' mock_response.text = '{"choices": [null]}'
with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): with patch("ss_tools.shared._llm_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))):
with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): with patch("ss_tools.shared._llm_http._handle_response_format_fallback", AsyncMock()):
with pytest.raises(ValueError, match="LLM response processing failed"): with pytest.raises(ValueError, match="LLM response processing failed"):
await call_openai_compatible( await call_openai_compatible(
"https://api.openai.com", "https://api.openai.com",
@@ -177,8 +180,8 @@ class TestCallOpenaiCompatible:
} }
mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"refusal": "I cannot answer that", "content": null}}]}' mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"refusal": "I cannot answer that", "content": null}}]}'
with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): with patch("ss_tools.shared._llm_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))):
with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): with patch("ss_tools.shared._llm_http._handle_response_format_fallback", AsyncMock()):
with pytest.raises(ValueError, match="refused"): with pytest.raises(ValueError, match="refused"):
await call_openai_compatible( await call_openai_compatible(
"https://api.openai.com", "https://api.openai.com",
@@ -200,8 +203,8 @@ class TestCallOpenaiCompatible:
) )
mock_response.text = "Internal Server Error" mock_response.text = "Internal Server Error"
with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): with patch("ss_tools.shared._llm_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))):
with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): with patch("ss_tools.shared._llm_http._handle_response_format_fallback", AsyncMock()):
with pytest.raises(httpx.HTTPStatusError): with pytest.raises(httpx.HTTPStatusError):
await call_openai_compatible( await call_openai_compatible(
"https://api.openai.com", "https://api.openai.com",
@@ -222,8 +225,8 @@ class TestCallOpenaiCompatible:
} }
mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"content": "hi"}}]}' mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"content": "hi"}}]}'
with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): with patch("ss_tools.shared._llm_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))):
with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): with patch("ss_tools.shared._llm_http._handle_response_format_fallback", AsyncMock()):
# disable_reasoning=False, so response_format should be set # disable_reasoning=False, so response_format should be set
content, _ = await call_openai_compatible( content, _ = await call_openai_compatible(
"https://api.openai.com", "https://api.openai.com",
@@ -247,8 +250,8 @@ class TestCallOpenaiCompatible:
} }
mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"content": "hi"}}]}' mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"content": "hi"}}]}'
with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): with patch("ss_tools.shared._llm_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))):
with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): with patch("ss_tools.shared._llm_http._handle_response_format_fallback", AsyncMock()):
content, _ = await call_openai_compatible( content, _ = await call_openai_compatible(
"https://api.openai.com", "https://api.openai.com",
"sk-test", "sk-test",
@@ -262,28 +265,29 @@ class TestCallOpenaiCompatible:
class TestDoHttpRequest: class TestDoHttpRequest:
"""_do_http_request — async HTTP POST with rate-limit retry.""" """_do_http_request — async HTTP POST with rate-limit retry."""
@pytest.fixture
def mock_client(self):
return AsyncMock(spec=httpx.AsyncClient)
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_success(self): async def test_success(self, mock_client):
"""Single successful request.""" """Single successful request."""
mock_response = MagicMock(spec=httpx.Response) mock_response = MagicMock(spec=httpx.Response)
mock_response.status_code = 200 mock_response.status_code = 200
mock_response.text = "OK" mock_response.text = "OK"
mock_client.post.return_value = mock_response
with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: response, text = await _do_http_request(
mock_client = AsyncMock() mock_client,
mock_get.return_value = mock_client "https://api.openai.com/chat/completions",
mock_client.post.return_value = mock_response {"Authorization": "Bearer test"},
{"model": "gpt-4o"},
response, text = await _do_http_request( )
"https://api.openai.com/chat/completions", assert response.status_code == 200
{"Authorization": "Bearer test"}, assert text == "OK"
{"model": "gpt-4o"},
)
assert response.status_code == 200
assert text == "OK"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_rate_limit_retry(self): async def test_rate_limit_retry(self, mock_client):
"""429 triggers retry.""" """429 triggers retry."""
mock_429 = MagicMock(spec=httpx.Response) mock_429 = MagicMock(spec=httpx.Response)
mock_429.status_code = 429 mock_429.status_code = 429
@@ -294,23 +298,21 @@ class TestDoHttpRequest:
mock_200.status_code = 200 mock_200.status_code = 200
mock_200.text = "OK" mock_200.text = "OK"
with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: mock_client.post.side_effect = [mock_429, mock_200]
mock_client = AsyncMock()
mock_get.return_value = mock_client
mock_client.post.side_effect = [mock_429, mock_200]
with patch("src.plugins.translate._llm_async_http.asyncio.sleep", AsyncMock()): with patch("ss_tools.shared._llm_http.asyncio.sleep", AsyncMock()):
response, text = await _do_http_request( response, text = await _do_http_request(
"https://api.openai.com/chat/completions", mock_client,
{"Authorization": "Bearer test"}, "https://api.openai.com/chat/completions",
{"model": "gpt-4o"}, {"Authorization": "Bearer test"},
) {"model": "gpt-4o"},
assert response.status_code == 200 )
assert text == "OK" assert response.status_code == 200
assert mock_client.post.call_count == 2 assert text == "OK"
assert mock_client.post.call_count == 2
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_rate_limit_with_retry_after(self): async def test_rate_limit_with_retry_after(self, mock_client):
"""Retry-After header is respected.""" """Retry-After header is respected."""
mock_429 = MagicMock(spec=httpx.Response) mock_429 = MagicMock(spec=httpx.Response)
mock_429.status_code = 429 mock_429.status_code = 429
@@ -321,46 +323,41 @@ class TestDoHttpRequest:
mock_200.status_code = 200 mock_200.status_code = 200
mock_200.text = "OK" mock_200.text = "OK"
with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: mock_client.post.side_effect = [mock_429, mock_200]
mock_client = AsyncMock()
mock_get.return_value = mock_client
mock_client.post.side_effect = [mock_429, mock_200]
with patch("src.plugins.translate._llm_async_http.asyncio.sleep", AsyncMock()) as mock_sleep: with patch("ss_tools.shared._llm_http.asyncio.sleep", AsyncMock()) as mock_sleep:
response, _ = await _do_http_request( response, _ = await _do_http_request(
"https://api.openai.com/chat/completions", mock_client,
{"Authorization": "Bearer test"}, "https://api.openai.com/chat/completions",
{"model": "gpt-4o"}, {"Authorization": "Bearer test"},
) {"model": "gpt-4o"},
assert response.status_code == 200 )
# Verify sleep was called with retry-after value assert response.status_code == 200
mock_sleep.assert_called_once_with(2) mock_sleep.assert_called_once_with(2)
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_rate_limit_exhausted(self): async def test_rate_limit_exhausted(self, mock_client):
"""All 429 retries exhausted, returns last 429.""" """All 429 retries exhausted, returns last 429."""
mock_429 = MagicMock(spec=httpx.Response) mock_429 = MagicMock(spec=httpx.Response)
mock_429.status_code = 429 mock_429.status_code = 429
mock_429.headers = {} mock_429.headers = {}
mock_429.text = "Rate limited" mock_429.text = "Rate limited"
with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: mock_client.post.return_value = mock_429 # Always 429
mock_client = AsyncMock()
mock_get.return_value = mock_client
mock_client.post.return_value = mock_429 # Always 429
with patch("src.plugins.translate._llm_async_http.asyncio.sleep", AsyncMock()): with patch("ss_tools.shared._llm_http.asyncio.sleep", AsyncMock()):
response, _ = await _do_http_request( response, _ = await _do_http_request(
"https://api.openai.com/chat/completions", mock_client,
{"Authorization": "Bearer test"}, "https://api.openai.com/chat/completions",
{"model": "gpt-4o"}, {"Authorization": "Bearer test"},
) {"model": "gpt-4o"},
assert response.status_code == 429 )
assert mock_client.post.call_count == 3 # max 3 retries assert response.status_code == 429
assert mock_client.post.call_count == 3 # max 3 retries
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_rate_limit_invalid_retry_after(self): async def test_rate_limit_invalid_retry_after(self, mock_client):
"""Retry-After with non-int value falls back to exponential backoff (lines 215-216).""" """Retry-After with non-int value falls back to exponential backoff."""
mock_429 = MagicMock(spec=httpx.Response) mock_429 = MagicMock(spec=httpx.Response)
mock_429.status_code = 429 mock_429.status_code = 429
mock_429.headers = {"Retry-After": "abc"} mock_429.headers = {"Retry-After": "abc"}
@@ -370,27 +367,29 @@ class TestDoHttpRequest:
mock_200.status_code = 200 mock_200.status_code = 200
mock_200.text = "OK" mock_200.text = "OK"
with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: mock_client.post.side_effect = [mock_429, mock_200]
mock_client = AsyncMock()
mock_get.return_value = mock_client
mock_client.post.side_effect = [mock_429, mock_200]
with patch("src.plugins.translate._llm_async_http.asyncio.sleep", AsyncMock()) as mock_sleep: with patch("ss_tools.shared._llm_http.asyncio.sleep", AsyncMock()) as mock_sleep:
response, _ = await _do_http_request( response, _ = await _do_http_request(
"https://api.openai.com/chat/completions", mock_client,
{"Authorization": "Bearer test"}, "https://api.openai.com/chat/completions",
{"model": "gpt-4o"}, {"Authorization": "Bearer test"},
) {"model": "gpt-4o"},
assert response.status_code == 200 )
# Should fall back to exponential backoff: 2^1 = 2 assert response.status_code == 200
mock_sleep.assert_called_once_with(2) # Should fall back to exponential backoff: 2^1 = 2
mock_sleep.assert_called_once_with(2)
class TestHandleResponseFormatFallback: class TestHandleResponseFormatFallback:
"""_handle_response_format_fallback — structured output fallback.""" """_handle_response_format_fallback — structured output fallback."""
@pytest.fixture
def mock_client(self):
return AsyncMock(spec=httpx.AsyncClient)
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_400_response_format_error(self): async def test_400_response_format_error(self, mock_client):
"""400 error with response_format pattern triggers retry.""" """400 error with response_format pattern triggers retry."""
mock_400 = MagicMock(spec=httpx.Response) mock_400 = MagicMock(spec=httpx.Response)
mock_400.status_code = 400 mock_400.status_code = 400
@@ -404,27 +403,25 @@ class TestHandleResponseFormatFallback:
mock_200.encoding = "utf-8" mock_200.encoding = "utf-8"
mock_200.headers = {} mock_200.headers = {}
mock_client.post.return_value = mock_200
payload = {"model": "gpt-4o", "response_format": {"type": "json_object"}} payload = {"model": "gpt-4o", "response_format": {"type": "json_object"}}
with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: await _handle_response_format_fallback(
mock_client = AsyncMock() mock_client,
mock_get.return_value = mock_client mock_400,
mock_client.post.return_value = mock_200 mock_400.text,
payload,
await _handle_response_format_fallback( "https://api.openai.com",
mock_400, {},
mock_400.text, )
payload, # Verify response_format was popped
"https://api.openai.com", assert "response_format" not in payload
{}, # Verify status_code was updated
) assert mock_400.status_code == 200
# Verify response_format was popped
assert "response_format" not in payload
# Verify status_code was updated
assert mock_400.status_code == 200
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_non_400_noop(self): async def test_non_400_noop(self, mock_client):
"""Non-400 error does nothing.""" """Non-400 error does nothing."""
mock_500 = MagicMock(spec=httpx.Response) mock_500 = MagicMock(spec=httpx.Response)
mock_500.status_code = 500 mock_500.status_code = 500
@@ -433,24 +430,21 @@ class TestHandleResponseFormatFallback:
payload = {"model": "gpt-4o", "response_format": {"type": "json_object"}} payload = {"model": "gpt-4o", "response_format": {"type": "json_object"}}
with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: await _handle_response_format_fallback(
mock_client = AsyncMock() mock_client,
mock_get.return_value = mock_client mock_500,
mock_500.text,
await _handle_response_format_fallback( payload,
mock_500, "https://api.openai.com",
mock_500.text, {},
payload, )
"https://api.openai.com", # Payload unchanged
{}, assert "response_format" in payload
) # No retry call
# Payload unchanged mock_client.post.assert_not_called()
assert "response_format" in payload
# No retry call
mock_client.post.assert_not_called()
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_400_no_pattern_noop(self): async def test_400_no_pattern_noop(self, mock_client):
"""400 error without response_format pattern does nothing.""" """400 error without response_format pattern does nothing."""
mock_400 = MagicMock(spec=httpx.Response) mock_400 = MagicMock(spec=httpx.Response)
mock_400.status_code = 400 mock_400.status_code = 400
@@ -459,19 +453,16 @@ class TestHandleResponseFormatFallback:
payload = {"model": "gpt-4o", "response_format": {"type": "json_object"}} payload = {"model": "gpt-4o", "response_format": {"type": "json_object"}}
with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: await _handle_response_format_fallback(
mock_client = AsyncMock() mock_client,
mock_get.return_value = mock_client mock_400,
mock_400.text,
await _handle_response_format_fallback( payload,
mock_400, "https://api.openai.com",
mock_400.text, {},
payload, )
"https://api.openai.com", assert "response_format" in payload
{}, mock_client.post.assert_not_called()
)
assert "response_format" in payload
mock_client.post.assert_not_called()
# #endregion Test.LLMAsyncHttpClient # #endregion Test.LLMAsyncHttpClient

View File

@@ -0,0 +1,259 @@
# #region Test.LLMTranslationService.BuildPrompt [C:3] [TYPE Module] [SEMANTICS test, llm, translate, prompt, context]
# @BRIEF Orthogonal verification of _build_prompt context column changes:
# rows_json enrichment, context_hint computation, template dialect/source-language removal.
# @RELATION BINDS_TO -> [LLMTranslationService._build_prompt]
# @TEST_EDGE: no_context_columns -> No 'context' key in rows_json, empty context_hint, no deprecated dialect markers
# @TEST_EDGE: context_columns_present -> 'context' key with source_data values, non-empty context_hint
# @TEST_EDGE: context_columns_missing_source_data -> Graceful fallback with empty string values per column
import json
from unittest.mock import MagicMock
from src.models.translate import TranslationLanguage
from src.models.translate import TranslationJob
from src.plugins.translate._llm_call import LLMTranslationService
from .conftest import JOB_ID
def _make_job(session, context_columns=None, translation_column="title"):
"""Create or update a TranslationJob for _build_prompt testing.
Each test invokes this against the function-scoped db_session fixture,
so job state is isolated per test.
"""
job = session.query(TranslationJob).filter(TranslationJob.id == JOB_ID).first()
if job is None:
job = TranslationJob(
id=JOB_ID, name="BuildPrompt Test",
source_dialect="en", target_dialect="fr",
status="ACTIVE", translation_column=translation_column,
)
job.context_columns = context_columns or []
session.add(job)
else:
job.context_columns = context_columns or []
job.translation_column = translation_column
session.commit()
return job
def _batch_rows(count=2, *, source_data_present=True):
"""Build batch rows; when source_data_present=False, omit the source_data key entirely."""
rows = []
for i in range(count):
row: dict = {
"row_index": str(i),
"source_text": f"Hello world {i}",
}
if source_data_present:
row["source_data"] = {
"author": f"Author_{i}",
"date": f"2024-01-0{i+1}",
"table": f"tbl_{i}",
}
rows.append(row)
return rows
class TestBuildPromptContextColumns:
"""Orthogonal verification of _build_prompt context column behaviour.
Covers the three orthogonal projections of context column handling:
(1) absent → no enrichment, (2) present → full enrichment,
(3) partially absent source_data → graceful fallback.
"""
# #region test_no_context_columns [C:2] [TYPE Function]
# @BRIEF Without context_columns: rows_json has no 'context' key, context_hint absent, no legacy dialect markers.
def test_no_context_columns(self, db_session):
"""Job has no context_columns → prompt stays lean, template changes verified."""
job = _make_job(db_session, context_columns=[])
rows = _batch_rows(2)
prompt = LLMTranslationService._build_prompt(job, rows, "", ["fr"])
# Rows JSON: no "context" key injected
assert '"context"' not in prompt, (
"rows_json must NOT contain 'context' key when context_columns is empty; "
"found the key in rendered prompt"
)
# Context hint: absent from the rendered prompt
assert "Context columns:" not in prompt, (
"context_hint section must be absent when no context_columns configured"
)
# Regression: removed source_dialect / target_dialect / source_language from template
assert "Source dialect" not in prompt, (
"Prompt must NOT contain removed 'Source dialect' marker"
)
assert "Translate from" not in prompt, (
"Prompt must NOT contain removed 'Translate from' marker"
)
# Sanity: row data still present
assert "Hello world 0" in prompt, "Row text must be present"
assert "Hello world 1" in prompt, "Row text must be present"
assert '"detected_source_language": "und"' in prompt, (
"rows_json must carry local detected_source_language for LLM arbitration"
)
assert "Include detected_source_language for EVERY row" in prompt, (
"prompt must require LLM to return detected_source_language"
)
# #endregion test_no_context_columns
# #region test_with_context_columns [C:2] [TYPE Function]
# @BRIEF With context_columns=["author","date"]: rows_json includes 'context' dict with source_data values, context_hint lists columns.
def test_with_context_columns(self, db_session):
"""Job has context_columns=['author','date'] → prompt enriched with context fields."""
job = _make_job(db_session, context_columns=["author", "date"])
rows = _batch_rows(2)
prompt = LLMTranslationService._build_prompt(job, rows, "", ["fr"])
# Rows JSON: must contain "context" key with correct source_data values
assert '"context"' in prompt, (
"rows_json must contain 'context' key when context_columns configured"
)
assert '"author": "Author_0"' in prompt, (
"context.author must match source_data value for row 0"
)
assert '"date": "2024-01-01"' in prompt, (
"context.date must match source_data value for row 0"
)
assert '"author": "Author_1"' in prompt, (
"context.author must match source_data value for row 1"
)
# Context hint: must mention the configured column names
assert "Context columns: author, date" in prompt, (
"context_hint must list configured context columns"
)
assert "Consider these context fields" in prompt, (
"context_hint must include guidance phrase about context fields"
)
# #endregion test_with_context_columns
# #region test_context_columns_missing_source_data [C:2] [TYPE Function]
# @BRIEF Context columns set but row has no source_data key → graceful fallback with empty-string values.
def test_context_columns_missing_source_data(self, db_session):
"""When source_data key is entirely absent, context values fall back to empty strings."""
job = _make_job(db_session, context_columns=["author", "date"])
rows = _batch_rows(1, source_data_present=False)
prompt = LLMTranslationService._build_prompt(job, rows, "", ["fr"])
# "context" key must still be present (structure is maintained)
assert '"context"' in prompt, (
"rows_json must still include 'context' key when source_data is missing"
)
# Each column falls back to an empty string (not null, not missing)
assert '"author": ""' in prompt, (
"context.author must fallback to empty string when source_data absent"
)
assert '"date": ""' in prompt, (
"context.date must fallback to empty string when source_data absent"
)
# Row text must still be present (basic structure intact)
assert "Hello world 0" in prompt, (
"Row text must be present even when source_data is missing"
)
# table field from _batch_rows(source_data_present=False) should NOT leak
assert '"table"' not in prompt, (
"Only context_columns fields should appear in context, not all source_data fields"
)
# #endregion test_context_columns_missing_source_data
# #region test_context_columns_empty_list [C:2] [TYPE Function]
# @BRIEF Edge case: empty context_columns list (same as None / not set).
def test_context_columns_empty_list(self, db_session):
"""Empty list is treated identically to None — no context enrichment."""
job = _make_job(db_session, context_columns=[])
rows = _batch_rows(1)
prompt = LLMTranslationService._build_prompt(job, rows, "", ["fr"])
assert '"context"' not in prompt, (
"Empty context_columns must behave like None — no context key injected"
)
assert "Hello world 0" in prompt, "Row text must be present"
# #endregion test_context_columns_empty_list
# #region test_context_columns_single_column [C:2] [TYPE Function]
# @BRIEF Single context column: correct context dict with exactly one key.
def test_context_columns_single_column(self, db_session):
"""A single context column produces a context dict with one key."""
job = _make_job(db_session, context_columns=["author"])
rows = _batch_rows(1)
prompt = LLMTranslationService._build_prompt(job, rows, "", ["fr"])
assert '"context"' in prompt, "Single context column must produce context key"
assert '"author": "Author_0"' in prompt, "Value must match source_data"
assert '"date"' not in prompt, "Unlisted column must not appear in context"
assert "Context columns: author" in prompt, "context_hint must list the single column"
# #endregion test_context_columns_single_column
# #region test_context_columns_still_passes_other_placeholders [C:2] [TYPE Function]
# @BRIEF Context column changes must not break other template placeholders.
def test_context_columns_still_passes_other_placeholders(self, db_session):
"""Verify target_languages, translation_column, and dictionary still render correctly."""
job = _make_job(db_session, context_columns=["author"], translation_column="description")
rows = _batch_rows(1)
prompt = LLMTranslationService._build_prompt(job, rows, "DICT_SECTION\n", ["fr", "de"])
# Target languages
assert "fr, de" in prompt, "target_languages must be present"
# Translation column
assert "description" in prompt, "translation_column must be present"
# Dictionary section
assert "DICT_SECTION" in prompt, "dictionary_section must be present"
# Row count
assert "Return exactly 1 entries" in prompt, "row_count must be correct"
# Context hint and context data still render
assert "Context columns: author" in prompt, "context_hint must appear with context columns"
assert '"context"' in prompt, "context key must appear with context columns"
# #endregion test_context_columns_still_passes_other_placeholders
class TestCreateRecordsLanguageArbitration:
"""Regression tests for LLM-assisted source-language detection."""
# #region test_llm_detected_same_language_creates_skip_marker [C:2] [TYPE Function]
# @BRIEF und local detection + LLM-detected same target creates marker language entry.
def test_llm_detected_same_language_creates_skip_marker(self):
"""When LLM detects ru for target ru, insert stage must skip translation row."""
db = MagicMock()
service = LLMTranslationService(db)
rows = [{
"row_index": "0",
"source_text": "Оплачено 22.06",
"_detected_lang": "und",
"source_data": {"id": "1"},
"_source_hash": "hash",
}]
translations = {
"0": {
"detected_source_language": "ru",
"ru": "Оплачено 22.06",
}
}
result = service._create_records_from_translations(
rows, "run-1", "batch-1", ["ru"], translations, [], 0,
)
assert result["successful"] == 1
language_entries = [
call.args[0] for call in db.add.call_args_list
if isinstance(call.args[0], TranslationLanguage)
]
assert len(language_entries) == 1
assert language_entries[0].language_code == "ru"
assert language_entries[0].source_language_detected == "ru"
assert language_entries[0].final_value == "Оплачено 22.06"
# #endregion test_llm_detected_same_language_creates_skip_marker
# #endregion Test.LLMTranslationService.BuildPrompt

View File

@@ -31,6 +31,7 @@ def mock_job():
job.target_language_column = "lang" job.target_language_column = "lang"
job.target_source_column = "source_text" job.target_source_column = "source_text"
job.target_source_language_column = "src_lang" job.target_source_language_column = "src_lang"
job.include_source_reference = True
job.context_columns = ["category", "brand"] job.context_columns = ["category", "brand"]
job.translation_column = "name" job.translation_column = "name"
return job return job
@@ -322,6 +323,44 @@ class TestBuildRows:
translated_langs = [r["lang"] for r in rows if r["is_original"] == 0] translated_langs = [r["lang"] for r in rows if r["is_original"] == 0]
assert translated_langs == ["ru"] assert translated_langs == ["ru"]
def test_no_source_reference_skips_src_language_entirely(self, mock_job):
"""No reference row + source language matching target -> no insert rows."""
mock_job.include_source_reference = False
lang_ru = self._make_lang("ru", final="Оплачено 22.06", src_lang="ru")
rec = self._make_record(
source_data={"product_id": 1, "store_id": "S1"},
source_sql="Оплачено 22.06",
languages=[lang_ru],
)
rows = build_rows(
records=[rec],
job=mock_job,
effective_target="translated_name",
primary_language="ru",
context_keys=[],
)
assert rows == []
def test_no_source_reference_keeps_non_source_translation(self, mock_job):
"""No reference row still inserts real target-language translations."""
mock_job.include_source_reference = False
lang_ru = self._make_lang("ru", final="Привет", src_lang="en")
rec = self._make_record(
source_data={"product_id": 1, "store_id": "S1"},
source_sql="Hello",
languages=[lang_ru],
)
rows = build_rows(
records=[rec],
job=mock_job,
effective_target="translated_name",
primary_language="ru",
context_keys=[],
)
assert len(rows) == 1
assert rows[0]["is_original"] == 0
assert rows[0]["lang"] == "ru"
# ── Line 100: final_value vs translated_value fallback ── # ── Line 100: final_value vs translated_value fallback ──
def test_translation_uses_final_value_then_translated_value(self, mock_job): def test_translation_uses_final_value_then_translated_value(self, mock_job):

View File

@@ -272,7 +272,7 @@ describe('fetchAllRuns', () => {
expect(mockApi.fetchApi).toHaveBeenCalledWith('/translate/runs'); expect(mockApi.fetchApi).toHaveBeenCalledWith('/translate/runs');
}); });
it('fetches all runs with job_id, status, trigger_type, and created_by filters', async () => { it('fetches all runs with job_id, status, trigger_type, created_by, and date filters', async () => {
mockApi.fetchApi.mockResolvedValue({ runs: [{ id: 'r1' }] }); mockApi.fetchApi.mockResolvedValue({ runs: [{ id: 'r1' }] });
const { fetchAllRuns } = await import('$lib/api/translate/runs.js'); const { fetchAllRuns } = await import('$lib/api/translate/runs.js');
const result = await fetchAllRuns({ const result = await fetchAllRuns({
@@ -280,12 +280,14 @@ describe('fetchAllRuns', () => {
status: 'FAILED', status: 'FAILED',
trigger_type: 'manual', trigger_type: 'manual',
created_by: 'user1', created_by: 'user1',
date_from: '2026-07-01',
date_to: '2026-07-08',
page: 1, page: 1,
page_size: 25, page_size: 25,
}); });
expect(result).toEqual({ runs: [{ id: 'r1' }] }); expect(result).toEqual({ runs: [{ id: 'r1' }] });
expect(mockApi.fetchApi).toHaveBeenCalledWith( expect(mockApi.fetchApi).toHaveBeenCalledWith(
'/translate/runs?page=1&page_size=25&job_id=job-1&status=FAILED&trigger_type=manual&created_by=user1', '/translate/runs?page=1&page_size=25&job_id=job-1&status=FAILED&trigger_type=manual&created_by=user1&date_from=2026-07-01&date_to=2026-07-08',
); );
}); });

View File

@@ -29,6 +29,8 @@ export interface AllRunsQueryOptions extends RunQueryOptions {
status?: string; status?: string;
trigger_type?: string; trigger_type?: string;
created_by?: string; created_by?: string;
date_from?: string;
date_to?: string;
} }
// #region triggerRun [C:2] [TYPE Function] [SEMANTICS translate, runs, trigger] // #region triggerRun [C:2] [TYPE Function] [SEMANTICS translate, runs, trigger]
@@ -171,6 +173,8 @@ export async function fetchAllRuns<T = unknown>(options: AllRunsQueryOptions = {
if (options.status) params.append('status', options.status); if (options.status) params.append('status', options.status);
if (options.trigger_type) params.append('trigger_type', options.trigger_type); if (options.trigger_type) params.append('trigger_type', options.trigger_type);
if (options.created_by) params.append('created_by', options.created_by); if (options.created_by) params.append('created_by', options.created_by);
if (options.date_from) params.append('date_from', options.date_from);
if (options.date_to) params.append('date_to', options.date_to);
const query = params.toString(); const query = params.toString();
return await api.fetchApi<T>(`/translate/runs${query ? `?${query}` : ''}`); return await api.fetchApi<T>(`/translate/runs${query ? `?${query}` : ''}`);
} catch (error) { } catch (error) {

View File

@@ -54,7 +54,15 @@ export class TranslateHistoryModel {
this.isLoading = true; this.isLoading = true;
this.uxState = 'loading'; this.uxState = 'loading';
try { try {
const result = await fetchAllRuns({ page: this.currentPage, page_size: this.pageSize, job_id: this.filterJobId || undefined, status: this.filterStatus || undefined, trigger_type: this.filterTrigger || undefined }); const result = await fetchAllRuns({
page: this.currentPage,
page_size: this.pageSize,
job_id: this.filterJobId || undefined,
status: this.filterStatus || undefined,
trigger_type: this.filterTrigger || undefined,
date_from: this.filterDateFrom || undefined,
date_to: this.filterDateTo || undefined,
});
this.runs = (result as { items?: Record<string, unknown>[] })?.items || []; this.runs = (result as { items?: Record<string, unknown>[] })?.items || [];
this.total = (result as { total?: number })?.total || 0; this.total = (result as { total?: number })?.total || 0;
this.uxState = this.runs.length === 0 ? 'empty' : 'populated'; this.uxState = this.runs.length === 0 ? 'empty' : 'populated';
@@ -72,8 +80,20 @@ export class TranslateHistoryModel {
async loadJobs(): Promise<void> { async loadJobs(): Promise<void> {
try { try {
const result = await fetchJobs({ page_size: 100 }); const pageSize = 100;
this.jobs = Array.isArray(result) ? result : ((result as { items?: unknown[]; results?: unknown[] })?.items || (result as { items?: unknown[]; results?: unknown[] })?.results || []) as Record<string, unknown>[]; const first = await fetchJobs({ page: 1, page_size: pageSize });
const firstJobs = this._extractJobs(first);
const total = Array.isArray(first) ? firstJobs.length : ((first as { total?: number })?.total || firstJobs.length);
const jobs = [...firstJobs];
for (let page = 2; jobs.length < total; page += 1) {
const result = await fetchJobs({ page, page_size: pageSize });
const pageJobs = this._extractJobs(result);
if (pageJobs.length === 0) break;
jobs.push(...pageJobs);
}
this.jobs = jobs;
} catch { /* non-critical */ } } catch { /* non-critical */ }
} }
@@ -158,5 +178,11 @@ export class TranslateHistoryModel {
total_records: this.metrics.reduce((s: number, m: Record<string, unknown>) => s + ((m.total_records as number) || 0), 0), total_records: this.metrics.reduce((s: number, m: Record<string, unknown>) => s + ((m.total_records as number) || 0), 0),
}; };
} }
private _extractJobs(result: unknown): Record<string, unknown>[] {
if (Array.isArray(result)) return result as Record<string, unknown>[];
const payload = result as { items?: unknown[]; results?: unknown[] };
return (payload?.items || payload?.results || []) as Record<string, unknown>[];
}
} }
// #endregion Translate.HistoryModel // #endregion Translate.HistoryModel

View File

@@ -86,10 +86,16 @@ describe("TranslateHistoryModel — Data Loading", () => {
// #region TranslateHistoryModel.LoadRuns [C:2] [TYPE Test] // #region TranslateHistoryModel.LoadRuns [C:2] [TYPE Test]
it("loadRuns sets loading state and populates runs", async () => { it("loadRuns sets loading state and populates runs", async () => {
vi.mocked(fetchAllRuns).mockResolvedValue({ items: [{ id: "r1" }, { id: "r2" }], total: 2 }); vi.mocked(fetchAllRuns).mockResolvedValue({ items: [{ id: "r1" }, { id: "r2" }], total: 2 });
model.filterDateFrom = "2026-07-01";
model.filterDateTo = "2026-07-08";
const promise = model.loadRuns(); const promise = model.loadRuns();
expect(model.isLoading).toBe(true); expect(model.isLoading).toBe(true);
expect(model.uxState).toBe("loading"); expect(model.uxState).toBe("loading");
await promise; await promise;
expect(fetchAllRuns).toHaveBeenCalledWith(expect.objectContaining({
date_from: "2026-07-01",
date_to: "2026-07-08",
}));
expect(model.runs).toHaveLength(2); expect(model.runs).toHaveLength(2);
expect(model.total).toBe(2); expect(model.total).toBe(2);
expect(model.uxState).toBe("populated"); expect(model.uxState).toBe("populated");
@@ -138,6 +144,16 @@ describe("TranslateHistoryModel — Data Loading", () => {
expect(model.jobs).toHaveLength(1); expect(model.jobs).toHaveLength(1);
}); });
it("loadJobs fetches all pages when total exceeds first page", async () => {
vi.mocked(fetchJobs)
.mockResolvedValueOnce({ items: Array.from({ length: 100 }, (_, i) => ({ id: `j${i}` })), total: 101 })
.mockResolvedValueOnce({ items: [{ id: "j100", name: "Job 100" }], total: 101 });
await model.loadJobs();
expect(fetchJobs).toHaveBeenCalledWith({ page: 1, page_size: 100 });
expect(fetchJobs).toHaveBeenCalledWith({ page: 2, page_size: 100 });
expect(model.jobs).toHaveLength(101);
});
it("loadJobs accepts results format result", async () => { it("loadJobs accepts results format result", async () => {
vi.mocked(fetchJobs).mockResolvedValue({ results: [{ id: "j1", name: "Job 1" }] }); vi.mocked(fetchJobs).mockResolvedValue({ results: [{ id: "j1", name: "Job 1" }] });
await model.loadJobs(); await model.loadJobs();

View File

@@ -53,7 +53,7 @@
<!-- Filters --> <!-- Filters -->
<div class="bg-surface-card border border-border rounded-lg p-4 mb-4"> <div class="bg-surface-card border border-border rounded-lg p-4 mb-4">
<div class="grid grid-cols-2 sm:grid-cols-5 gap-3"> <div class="grid grid-cols-2 sm:grid-cols-6 gap-3">
<select bind:value={m.filterJobId} class="px-3 py-2 border border-border-strong rounded-lg text-sm"> <select bind:value={m.filterJobId} class="px-3 py-2 border border-border-strong rounded-lg text-sm">
<option value="">{$t.translate?.history?.all_jobs}</option> <option value="">{$t.translate?.history?.all_jobs}</option>
{#each m.jobs as job}<option value={job.id}>{job.name}</option>{/each} {#each m.jobs as job}<option value={job.id}>{job.name}</option>{/each}
@@ -73,6 +73,7 @@
<option value="baseline_expired">{$t.translate?.history?.trigger_baseline_expired}</option> <option value="baseline_expired">{$t.translate?.history?.trigger_baseline_expired}</option>
</select> </select>
<input type="date" bind:value={m.filterDateFrom} class="px-3 py-2 border border-border-strong rounded-lg text-sm" placeholder={$t.translate?.common?.from} /> <input type="date" bind:value={m.filterDateFrom} class="px-3 py-2 border border-border-strong rounded-lg text-sm" placeholder={$t.translate?.common?.from} />
<input type="date" bind:value={m.filterDateTo} class="px-3 py-2 border border-border-strong rounded-lg text-sm" placeholder={$t.translate?.common?.to} />
<button onclick={() => { page0 = 0; m.applyFilters(); }} class="px-4 py-2 text-sm bg-primary text-white rounded-lg hover:bg-primary-hover">{$t.translate?.history?.filter}</button> <button onclick={() => { page0 = 0; m.applyFilters(); }} class="px-4 py-2 text-sm bg-primary text-white rounded-lg hover:bg-primary-hover">{$t.translate?.history?.filter}</button>
</div> </div>
</div> </div>
@@ -90,6 +91,11 @@
<!-- Populated / Detail --> <!-- Populated / Detail -->
{:else if m.uxState === 'populated' || m.uxState === 'detail_open'} {:else if m.uxState === 'populated' || m.uxState === 'detail_open'}
<div class="mb-3 text-sm text-text-muted">
{_('translate.history.showing')
.replace('{count}', String(Math.min(m.currentPage * m.pageSize, m.total)))
.replace('{total}', String(m.total))}
</div>
<div class="space-y-2"> <div class="space-y-2">
{#each m.runs as run} {#each m.runs as run}
<div onclick={() => m.openDetail(run)} class="bg-surface-card border border-border rounded-lg p-3 hover:shadow-sm hover:border-border-strong transition-all cursor-pointer"> <div onclick={() => m.openDetail(run)} class="bg-surface-card border border-border rounded-lg p-3 hover:shadow-sm hover:border-border-strong transition-all cursor-pointer">

View File

@@ -3,6 +3,8 @@
# Contains: cot_logger (stdlib-only trace propagation), # Contains: cot_logger (stdlib-only trace propagation),
# ssl (stdlib-only SSL context), # ssl (stdlib-only SSL context),
# logger (lightweight JSON logger, no pydantic), # logger (lightweight JSON logger, no pydantic),
# _llm_http (shared httpx.AsyncClient with system SSL),
# _llm_health (LLM health probe, openai+httpx). # _llm_health (LLM health probe, openai+httpx).
# @INVARIANT No dependency on FastAPI, SQLAlchemy, Gradio, LangChain. # @INVARIANT No dependency on FastAPI, SQLAlchemy, Gradio, LangChain.
# @INVARIANT All HTTP clients use system_ssl_context() from ssl module.
# #endregion ss_tools.shared # #endregion ss_tools.shared

View File

@@ -27,6 +27,7 @@ from openai import (
RateLimitError, RateLimitError,
) )
from ._llm_http import get_shared_http_client
from .logger import logger from .logger import logger
# ── LLM provider health cache ───────────────────────────────────────── # ── LLM provider health cache ─────────────────────────────────────────
@@ -50,19 +51,19 @@ async def _check_llm_provider_health() -> str:
# Fetch LLM config from backend's own API (same as agent container does) # Fetch LLM config from backend's own API (same as agent container does)
try: try:
fastapi_url = os.getenv("FASTAPI_URL", "http://localhost:8000") fastapi_url = os.getenv("FASTAPI_URL", "http://localhost:8000")
async with httpx.AsyncClient(timeout=5) as client: client = get_shared_http_client(timeout=10)
resp = await client.get(f"{fastapi_url}/api/agent/llm-config") resp = await client.get(f"{fastapi_url}/api/agent/llm-config")
if resp.status_code != 200: if resp.status_code != 200:
_llm_status["status"] = "unavailable" _llm_status["status"] = "unavailable"
_llm_status["last_error"] = f"LLM config endpoint returned {resp.status_code}" _llm_status["last_error"] = f"LLM config endpoint returned {resp.status_code}"
_llm_status["last_check_ts"] = time.time() _llm_status["last_check_ts"] = time.time()
return "unavailable" return "unavailable"
config = resp.json() config = resp.json()
if not config or not config.get("configured"): if not config or not config.get("configured"):
_llm_status["status"] = "unavailable" _llm_status["status"] = "unavailable"
_llm_status["last_error"] = "No LLM provider configured" _llm_status["last_error"] = "No LLM provider configured"
_llm_status["last_check_ts"] = time.time() _llm_status["last_check_ts"] = time.time()
return "unavailable" return "unavailable"
except Exception as exc: except Exception as exc:
_llm_status["status"] = "unavailable" _llm_status["status"] = "unavailable"
_llm_status["last_error"] = f"Failed to fetch LLM config: {exc}" _llm_status["last_error"] = f"Failed to fetch LLM config: {exc}"
@@ -72,15 +73,11 @@ async def _check_llm_provider_health() -> str:
# Probe LLM API using AsyncOpenAI (available in both backend and agent containers) # Probe LLM API using AsyncOpenAI (available in both backend and agent containers)
try: try:
from openai import AsyncOpenAI from openai import AsyncOpenAI
from .ssl import system_ssl_context
api = AsyncOpenAI( api = AsyncOpenAI(
api_key=config.get("api_key", ""), api_key=config.get("api_key", ""),
base_url=config.get("base_url", "https://api.openai.com/v1"), base_url=config.get("base_url", "https://api.openai.com/v1"),
http_client=httpx.AsyncClient( http_client=get_shared_http_client(timeout=10),
verify=system_ssl_context(),
timeout=10,
),
) )
await api.chat.completions.create( await api.chat.completions.create(
model=config.get("default_model", "gpt-4o-mini"), model=config.get("default_model", "gpt-4o-mini"),

View File

@@ -0,0 +1,308 @@
# #region SharedLlmHttpClient [C:3] [TYPE Module] [SEMANTICS shared,http,client,ssl,llm]
# @defgroup Shared Shared lightweight utilities for backend and agent.
# @BRIEF Singleton httpx.AsyncClient with system SSL context for all LLM/API HTTP calls.
# Provides connection pooling, proper SSL verification (capath), configurable timeout.
# @LAYER Infrastructure
# @INVARIANT No dependency on FastAPI, SQLAlchemy, Gradio, LangChain, pydantic.
# @INVARIANT All HTTP clients use system_ssl_context() for SSL verification.
# @RELATION DEPENDS_ON -> [CoreSslTrust]
# @RATIONALE All HTTP calls to LLM providers and external APIs must use the system CA
# store (capath) instead of certifi to respect corporate certificates installed at
# container startup. This module provides a single source of truth for HTTP client
# creation, eliminating the pattern of per-module httpx.AsyncClient(timeout=N) without
# SSL context that silently uses certifi and fails with corporate CA certs.
# @REJECTED Per-module httpx.AsyncClient singletons were rejected — each one was a copy
# that could forget SSL context; module-level _http_client variables scattered across
# 5+ files created connection pooling fragmentation.
# @REJECTED Passing verify=True (certifi default) was rejected — it passes with public CAs
# but silently fails with corporate intermediate CAs installed in /etc/ssl/certs.
# Only capath-based ssl.create_default_context() works with OpenSSL 3.x intermediates.
import asyncio
import ssl
from typing import Any
import httpx
from .logger import logger
from .ssl import httpx_verify
# Module-level singleton clients, lazily initialized
_http_client_180: httpx.AsyncClient | None = None
_http_client_10: httpx.AsyncClient | None = None
# #region SharedLlmHttpClient.GetSharedClient [C:2] [TYPE Function] [SEMANTICS shared,http,client,get]
# @ingroup Shared
# @BRIEF Get or create a module-level httpx.AsyncClient singleton with system SSL context.
# Uses shared ssl module (capath) for certificate verification.
# @POST Returns httpx.AsyncClient with verify=system_ssl_context() and specified timeout.
# @SIDE_EFFECT Lazily creates the client on first call and caches it for reuse.
def get_shared_http_client(timeout: float = 180.0) -> httpx.AsyncClient:
"""Get or create a shared httpx.AsyncClient singleton with system SSL context.
Uses the system CA store (capath) for SSL verification — NOT certifi.
This ensures corporate CA certificates installed at container startup
are trusted for all HTTP calls.
Args:
timeout: Total timeout in seconds (default 180).
Use 10 for lightweight config fetches, 180 for LLM API calls.
Returns:
httpx.AsyncClient with SSL context and specified timeout.
"""
global _http_client_180, _http_client_10
# Cache two common timeout configurations to avoid creating new clients
if abs(timeout - 180.0) < 0.1:
if _http_client_180 is None:
ssl_ctx = httpx_verify()
_http_client_180 = httpx.AsyncClient(
verify=ssl_ctx,
timeout=httpx.Timeout(180.0),
)
logger.reason(
"Created shared HTTP client (180s timeout)",
extra={"src": "SharedLlmHttpClient", "ssl": "system_ssl_context"},
)
return _http_client_180
if abs(timeout - 10.0) < 0.1:
if _http_client_10 is None:
ssl_ctx = httpx_verify()
_http_client_10 = httpx.AsyncClient(
verify=ssl_ctx,
timeout=httpx.Timeout(10.0),
)
logger.reason(
"Created shared HTTP client (10s timeout)",
extra={"src": "SharedLlmHttpClient", "ssl": "system_ssl_context"},
)
return _http_client_10
# Custom timeout — create a new client (not cached)
ssl_ctx = httpx_verify()
client = httpx.AsyncClient(
verify=ssl_ctx,
timeout=httpx.Timeout(timeout),
)
logger.reason(
"Created shared HTTP client (custom timeout)",
extra={"src": "SharedLlmHttpClient", "timeout": timeout, "ssl": "system_ssl_context"},
)
return client
# #endregion SharedLlmHttpClient.GetSharedClient
# #region SharedLlmHttpClient.SanitizeUrl [C:1] [TYPE Function] [SEMANTICS shared,url,sanitize]
# @ingroup Shared
# @BRIEF Strip embedded credentials from URL for safe logging.
# @POST Returns URL with user:pass@ portion removed, preserving host:port.
def sanitize_url(url: str) -> str:
"""Strip embedded credentials from URL for safe logging."""
if not url:
return url
from urllib.parse import urlsplit, urlunsplit
parsed = urlsplit(url)
if parsed.username or parsed.password:
safe_netloc = parsed.hostname
if parsed.port:
safe_netloc += f":{parsed.port}"
parsed = parsed._replace(netloc=safe_netloc)
return urlunsplit(parsed)
# #endregion SharedLlmHttpClient.SanitizeUrl
# #region SharedLlmHttpClient.CallOpenaiCompatible [C:3] [TYPE Function] [SEMANTICS shared,llm,http,openai,async]
# @ingroup Shared
# @BRIEF Call OpenAI-compatible API asynchronously with rate-limit handling and structured output fallback.
# @PRE Valid API endpoint, key, model, and prompt.
# @POST Returns (response text, finish_reason).
# @SIDE_EFFECT Async HTTP POST to LLM API with optional retry on 429.
# @REJECTED Keeping sync requests.post — would block async event loop during LLM calls.
# Per-request httpx.AsyncClient — loses connection pooling.
async def call_openai_compatible(
base_url: str,
api_key: str,
model: str,
prompt: str,
provider_type: str = "openai",
max_tokens: int = 8192,
disable_reasoning: bool = False,
) -> tuple[str, str | None]:
"""Call OpenAI-compatible API for LLM requests (async)."""
if not base_url:
raise ValueError("LLM provider has no base_url configured")
# Normalise base_url: strip trailing /v1 to avoid double /v1
base = base_url.rstrip("/")
if base.endswith("/v1"):
base = base[:-3]
url = f"{base}/v1/chat/completions"
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
}
system_content = (
"You are a database content translation assistant. "
"Translate the provided text accurately, preserving data semantics. "
"Respond directly with ONLY the JSON result. "
"Do NOT include any reasoning, thinking, chain-of-thought, analysis, or explanation. "
"Output ONLY valid JSON."
)
payload: dict[str, Any] = {
"model": model,
"messages": [
{"role": "system", "content": system_content},
{"role": "user", "content": prompt},
],
"temperature": 0.1,
"max_tokens": max_tokens,
}
if provider_type in ("openai", "openai_compatible", "kilo", "openrouter", "litellm"):
if not disable_reasoning:
payload["response_format"] = {"type": "json_object"}
if disable_reasoning:
if provider_type not in ("kilo", "openrouter", "litellm"):
payload["reasoning_effort"] = "none"
payload["max_tokens"] = max_tokens
client = get_shared_http_client()
response, response_text = await _do_http_request(client, url, headers, payload)
await _handle_response_format_fallback(client, response, response_text, payload, url, headers)
if not response.is_success:
logger.explore(
f"LLM API error status={response.status_code} model={payload.get('model')} body={response_text[:2000]}",
extra={"src": "SharedLlmHttpClient"},
)
response.raise_for_status()
data = response.json()
choices = data.get("choices", [])
if not choices:
logger.explore(
"LLM returned no choices",
extra={
"src": "SharedLlmHttpClient",
"response_keys": list(data.keys()),
"response_preview": str(data)[:2000],
},
)
raise ValueError("LLM returned no choices")
try:
finish_reason = choices[0].get("finish_reason") or "none"
msg = choices[0].get("message") or {}
except (TypeError, AttributeError) as e:
logger.explore(
"TypeError processing LLM response choices",
extra={
"src": "SharedLlmHttpClient",
"error": str(e),
"choices_0_type": type(choices[0]).__name__ if choices else "N/A",
"choices_0_repr": repr(choices[0])[:2000] if choices else "N/A",
"data_type": type(data).__name__,
"data_preview": str(data)[:2000],
},
)
raise ValueError(f"LLM response processing failed: {e}")
refusal = msg.get("refusal") if isinstance(msg, dict) else None
if refusal:
logger.explore(
"LLM refused to respond",
extra={
"src": "SharedLlmHttpClient",
"refusal": str(refusal)[:500],
"finish_reason": finish_reason,
},
)
raise ValueError(f"LLM refused to respond: {refusal}")
content = msg.get("content") if isinstance(msg, dict) else ""
if not content and isinstance(msg, dict):
content = msg.get("content") or ""
if not content:
logger.explore(
"LLM returned empty content",
extra={
"src": "SharedLlmHttpClient",
"finish_reason": finish_reason,
"msg_keys": list(msg.keys()) if isinstance(msg, dict) else [],
"response_preview": str(data)[:2000],
},
)
raise ValueError("LLM returned empty content")
return content, finish_reason
# #endregion SharedLlmHttpClient.CallOpenaiCompatible
# #region SharedLlmHttpClient.DoHttpRequest [C:1] [TYPE Function] [SEMANTICS shared,http,request,retry]
async def _do_http_request(
client: httpx.AsyncClient,
url: str,
headers: dict,
payload: dict,
) -> tuple[httpx.Response, str]:
"""Make async HTTP POST with rate-limit (429) retry handling."""
_max_retry_429 = 3
_retry_count_429 = 0
while _retry_count_429 < _max_retry_429:
response = await client.post(url, headers=headers, json=payload)
response_text = response.text
if response.status_code == 429:
_retry_count_429 += 1
retry_after = response.headers.get("Retry-After")
if retry_after:
try:
wait = int(retry_after)
except (ValueError, TypeError):
wait = 2**_retry_count_429
else:
wait = 2**_retry_count_429
logger.explore(
f"Rate limited (429), retry {_retry_count_429}/{_max_retry_429} after {wait}s",
extra={"src": "SharedLlmHttpClient", "retry_after": retry_after, "wait": wait},
)
await asyncio.sleep(wait)
if _retry_count_429 >= _max_retry_429:
break
else:
break
return response, response_text
# #endregion SharedLlmHttpClient.DoHttpRequest
# #region SharedLlmHttpClient.HandleResponseFormatFallback [C:1] [TYPE Function] [SEMANTICS shared,http,response,fallback]
async def _handle_response_format_fallback(
client: httpx.AsyncClient,
response: httpx.Response,
response_text: str,
payload: dict,
url: str,
headers: dict,
) -> None:
"""Handle 400 errors from structured_outputs not being supported."""
_patterns = ("response_format", "structured_outputs", "structured", "json_object")
if not response.is_success and response.status_code == 400 and any(p in (response_text or "").lower() for p in _patterns):
logger.explore(
"Structured outputs not supported, retrying without response_format",
extra={"src": "SharedLlmHttpClient"},
)
payload.pop("response_format", None)
new_response = await client.post(url, headers=headers, json=payload)
# Mutate the original response object with new data
response.status_code = new_response.status_code
response._content = new_response.content
response.encoding = new_response.encoding
response.headers = new_response.headers
# #endregion SharedLlmHttpClient.HandleResponseFormatFallback
# #endregion SharedLlmHttpClient