feat(translate): replace LLM-based language detection with local lingua-detector
- Add _lang_detect.py module wrapping lingua-language-detector (pure Python, 0.076ms/call) - Add batch_detect() with detector caching for performance - Pre-filter rows where detected source language matches target (same-language skip) - Remove detected_source_language from LLM prompt templates (token savings) - Use local detection for cached rows (was always 'und') - Preserve backward compat: LLM detected_source_language as fallback when lingua returns 'und' - Add 24 unit tests for detection, edge cases, caching, and mapping
This commit is contained in:
@@ -1,10 +1,10 @@
|
||||
# #region BatchProcessingService [C:4] [TYPE Module] [SEMANTICS translate, batch, process, classify, cache]
|
||||
# @BRIEF Batch processing for translation: create batch records, classify rows (cache/preview/LLM),
|
||||
# @BRIEF Batch processing for translation: classify rows (same-language/cache/preview/LLM),
|
||||
# call LLM service, persist TranslationRecord/TranslationLanguage rows.
|
||||
# Insert-to-target delegated to _batch_insert.py.
|
||||
# Local language detection (lingua) replaces LLM-based detection.
|
||||
# @LAYER Domain
|
||||
# @RELATION DEPENDS_ON -> [TranslationBatch], [TranslationRecord], [TranslationLanguage]
|
||||
# @RELATION DEPENDS_ON -> [DictionaryManager], [LLMTranslationService]
|
||||
# @RELATION DEPENDS_ON -> [DictionaryManager], [LLMTranslationService], [LanguageDetectService]
|
||||
# @RELATION DEPENDS_ON -> [estimate_token_budget], [ConfigManager]
|
||||
# @PRE DB session is available. Job configuration is valid.
|
||||
# @POST TranslationBatch, TranslationRecord, TranslationLanguage rows created and committed.
|
||||
@@ -24,6 +24,7 @@ from ...core.logger import belief_scope, logger
|
||||
from ...models.translate import TranslationBatch, TranslationJob, TranslationLanguage, TranslationRecord
|
||||
from ...services.llm_provider import LLMProviderService
|
||||
from ._batch_insert import insert_batch_to_target
|
||||
from ._lang_detect import batch_detect, get_detector
|
||||
from ._llm_call import LLMTranslationService
|
||||
from ._token_budget import estimate_token_budget
|
||||
from ._utils import _check_translation_cache, _compute_source_hash, _compute_key_hash
|
||||
@@ -58,15 +59,18 @@ class BatchProcessingService:
|
||||
bid = batch.id
|
||||
result = {"successful": 0, "failed": 0, "skipped": 0, "retries": 0}
|
||||
|
||||
tls = job.target_languages or [job.target_dialect or "en"]
|
||||
tls = [str(tls)] if not isinstance(tls, list) else tls
|
||||
|
||||
# ★ Run local language detection on all rows (heuristic, no LLM)
|
||||
self._detect_languages(batch_rows, tls)
|
||||
|
||||
source_texts = [r.get("source_text", "") for r in batch_rows if r.get("source_text")]
|
||||
rc = batch_rows[0].get("source_data") if batch_rows else None
|
||||
dict_matches = DictionaryManager.filter_for_batch(self.db, source_texts, job.id, row_context=rc)
|
||||
|
||||
self._check_cache(job, batch_rows, dict_snapshot_hash, config_hash)
|
||||
llm_rows, pre_rows = self._classify(job, batch_rows, preview_edits_cache)
|
||||
|
||||
tls = job.target_languages or [job.target_dialect or "en"]
|
||||
tls = [str(tls)] if not isinstance(tls, list) else tls
|
||||
llm_rows, pre_rows = self._classify(batch_rows, preview_edits_cache, tls)
|
||||
|
||||
result["successful"] += self._persist_pre(pre_rows, bid, run_id, tls)
|
||||
if llm_rows:
|
||||
@@ -107,16 +111,34 @@ class BatchProcessingService:
|
||||
row["_cached_lang_values"] = cached
|
||||
logger.reason("Translation cache hit", {"source_hash": h[:12], "langs": list(cached.keys())})
|
||||
|
||||
def _classify(self, job, batch_rows, preview_edits_cache):
|
||||
# ★ Local language detection — replaces LLM-based detection
|
||||
def _detect_languages(self, batch_rows: list[dict], target_languages: list[str]) -> None:
|
||||
"""Run local language detection on all batch rows (no LLM).
|
||||
|
||||
Attaches '_detected_lang' (BCP-47 code or 'und') to each row dict.
|
||||
Uses batch_detect() for efficient multi-text processing.
|
||||
"""
|
||||
texts = [row.get("source_text", "") for row in batch_rows]
|
||||
results = batch_detect(texts, target_languages)
|
||||
for row, lang in zip(batch_rows, results):
|
||||
row["_detected_lang"] = lang
|
||||
|
||||
def _classify(self, batch_rows, preview_edits_cache, tls):
|
||||
llm_rows, pre_rows = [], []
|
||||
tls_lower = [str(t).lower() for t in tls]
|
||||
for row in batch_rows:
|
||||
# ★ NEW: Same-language pre-filter — detected language matches target
|
||||
dl = row.get("_detected_lang")
|
||||
if dl and str(dl).lower() not in ("und", "") and str(dl).lower() in tls_lower:
|
||||
row["_same_language"] = True
|
||||
pre_rows.append(row)
|
||||
continue
|
||||
|
||||
if row.get("approved_translation"):
|
||||
pre_rows.append(row)
|
||||
continue
|
||||
cl = row.get("_cached_lang_values")
|
||||
if cl:
|
||||
tls = job.target_languages or [job.target_dialect or "en"]
|
||||
tls = [str(tls)] if not isinstance(tls, list) else tls
|
||||
if all(lc in cl for lc in tls):
|
||||
pre_rows.append(row)
|
||||
continue
|
||||
@@ -137,7 +159,31 @@ class BatchProcessingService:
|
||||
def _persist_pre(self, pre_rows, bid, run_id, tls):
|
||||
count = 0
|
||||
for row in pre_rows:
|
||||
# ★ Same-language row: source language = target → no translation needed
|
||||
if row.get("_same_language"):
|
||||
dl = str(row["_detected_lang"])
|
||||
source_text = row.get("source_text", "")
|
||||
rec = TranslationRecord(
|
||||
id=str(uuid.uuid4()), batch_id=bid, run_id=run_id,
|
||||
source_sql=source_text, target_sql=source_text,
|
||||
source_object_type="table_row", source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
source_data=row.get("source_data"), source_hash=row.get("_source_hash"),
|
||||
status="SUCCESS",
|
||||
)
|
||||
self.db.add(rec)
|
||||
# One TranslationLanguage for the detected (same) language
|
||||
self.db.add(TranslationLanguage(
|
||||
id=str(uuid.uuid4()), record_id=rec.id, language_code=dl,
|
||||
source_language_detected=dl, translated_value=source_text,
|
||||
final_value=source_text, status="translated", needs_review=False,
|
||||
))
|
||||
count += 1
|
||||
continue
|
||||
|
||||
# Existing: cached / approved rows
|
||||
cl = row.get("_cached_lang_values")
|
||||
detected_lang = row.get("_detected_lang", "und") or "und"
|
||||
rec = TranslationRecord(
|
||||
id=str(uuid.uuid4()), batch_id=bid, run_id=run_id,
|
||||
source_sql=row.get("source_text", ""), target_sql=row.get("approved_translation", ""),
|
||||
@@ -151,7 +197,7 @@ class BatchProcessingService:
|
||||
fv = cl[lc] if (cl and lc in cl) else row.get("approved_translation", "")
|
||||
self.db.add(TranslationLanguage(
|
||||
id=str(uuid.uuid4()), record_id=rec.id, language_code=lc,
|
||||
source_language_detected="und", translated_value=fv,
|
||||
source_language_detected=detected_lang, translated_value=fv,
|
||||
final_value=fv, status="translated", needs_review=False,
|
||||
))
|
||||
count += 1
|
||||
|
||||
Reference in New Issue
Block a user