feat(translate): replace LLM-based language detection with local lingua-detector
- Add _lang_detect.py module wrapping lingua-language-detector (pure Python, 0.076ms/call) - Add batch_detect() with detector caching for performance - Pre-filter rows where detected source language matches target (same-language skip) - Remove detected_source_language from LLM prompt templates (token savings) - Use local detection for cached rows (was always 'und') - Preserve backward compat: LLM detected_source_language as fallback when lingua returns 'und' - Add 24 unit tests for detection, edge cases, caching, and mapping
This commit is contained in:
@@ -23,6 +23,7 @@ from typing import Any
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...core.config_manager import ConfigManager
|
||||
from ._lang_detect import detect_language, get_detector
|
||||
from .preview_constants import DEFAULT_EXECUTION_PROMPT_TEMPLATE, DEFAULT_PREVIEW_PROMPT_TEMPLATE # noqa: F401
|
||||
from ...core.logger import belief_scope, logger
|
||||
from ...models.translate import (
|
||||
@@ -72,7 +73,6 @@ class TranslationPreview:
|
||||
env_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
with belief_scope("TranslationPreview.preview_rows"):
|
||||
t0 = time.monotonic()
|
||||
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
|
||||
if not job:
|
||||
raise ValueError(f"Translation job '{job_id}' not found")
|
||||
@@ -110,6 +110,11 @@ class TranslationPreview:
|
||||
job=job, source_rows=source_rows, sample_size=sample_size, prompt_template=prompt_template,
|
||||
)
|
||||
|
||||
# ★ NEW: Run local language detection on row_meta (no LLM)
|
||||
detector = get_detector(target_languages)
|
||||
for meta in prompt_data["row_meta"]:
|
||||
meta["_detected_lang"] = detect_language(meta["source_text"], detector)
|
||||
|
||||
t_llm = time.monotonic()
|
||||
llm_response = self._executor.call_llm(
|
||||
job=job, prompt=prompt_data["prompt"], max_tokens=token_budget["max_output_needed"],
|
||||
@@ -175,7 +180,10 @@ class TranslationPreview:
|
||||
idx = meta["row_index"]
|
||||
source_text = meta["source_text"]
|
||||
translation_data = translations.get(str(idx), {})
|
||||
detected_lang = translation_data.get("detected_source_language", "und") if isinstance(translation_data, dict) else "und"
|
||||
# ★ Local detection (from lingua) takes priority over LLM response
|
||||
detected_lang = meta.get("_detected_lang", "und") or "und"
|
||||
if detected_lang == "und" and isinstance(translation_data, dict):
|
||||
detected_lang = translation_data.get("detected_source_language", "und") or "und"
|
||||
|
||||
lang_entries: list[TranslationPreviewLanguage] = []
|
||||
for lang_code in target_languages:
|
||||
|
||||
Reference in New Issue
Block a user