feat(translate): replace LLM-based language detection with local lingua-detector

- Add _lang_detect.py module wrapping lingua-language-detector (pure Python, 0.076ms/call)
- Add batch_detect() with detector caching for performance
- Pre-filter rows where detected source language matches target (same-language skip)
- Remove detected_source_language from LLM prompt templates (token savings)
- Use local detection for cached rows (was always 'und')
- Preserve backward compat: LLM detected_source_language as fallback when lingua returns 'und'
- Add 24 unit tests for detection, edge cases, caching, and mapping
This commit is contained in:
2026-05-20 14:31:37 +03:00
parent d1fdba5af1
commit 04a20f7d3c
8 changed files with 438 additions and 18 deletions

View File

@@ -23,6 +23,7 @@ from typing import Any
from sqlalchemy.orm import Session
from ...core.config_manager import ConfigManager
from ._lang_detect import detect_language, get_detector
from .preview_constants import DEFAULT_EXECUTION_PROMPT_TEMPLATE, DEFAULT_PREVIEW_PROMPT_TEMPLATE # noqa: F401
from ...core.logger import belief_scope, logger
from ...models.translate import (
@@ -72,7 +73,6 @@ class TranslationPreview:
env_id: str | None = None,
) -> dict[str, Any]:
with belief_scope("TranslationPreview.preview_rows"):
t0 = time.monotonic()
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
if not job:
raise ValueError(f"Translation job '{job_id}' not found")
@@ -110,6 +110,11 @@ class TranslationPreview:
job=job, source_rows=source_rows, sample_size=sample_size, prompt_template=prompt_template,
)
# ★ NEW: Run local language detection on row_meta (no LLM)
detector = get_detector(target_languages)
for meta in prompt_data["row_meta"]:
meta["_detected_lang"] = detect_language(meta["source_text"], detector)
t_llm = time.monotonic()
llm_response = self._executor.call_llm(
job=job, prompt=prompt_data["prompt"], max_tokens=token_budget["max_output_needed"],
@@ -175,7 +180,10 @@ class TranslationPreview:
idx = meta["row_index"]
source_text = meta["source_text"]
translation_data = translations.get(str(idx), {})
detected_lang = translation_data.get("detected_source_language", "und") if isinstance(translation_data, dict) else "und"
# ★ Local detection (from lingua) takes priority over LLM response
detected_lang = meta.get("_detected_lang", "und") or "und"
if detected_lang == "und" and isinstance(translation_data, dict):
detected_lang = translation_data.get("detected_source_language", "und") or "und"
lang_entries: list[TranslationPreviewLanguage] = []
for lang_code in target_languages: