diff --git a/backend/src/plugins/translate/_lang_detect.py b/backend/src/plugins/translate/_lang_detect.py index 757b5012..dce82882 100644 --- a/backend/src/plugins/translate/_lang_detect.py +++ b/backend/src/plugins/translate/_lang_detect.py @@ -1,10 +1,9 @@ -# #region LanguageDetectService [C:3] [TYPE Module] [SEMANTICS translate, language, detection, heuristic] +# #region LanguageDetectService [C:2] [TYPE Module] [SEMANTICS translate, language, detection, heuristic] # @BRIEF Local language detection powered by lingua-language-detector (no LLM). # Provides fast, non-LLM language identification for source text rows, # reducing LLM token costs by pre-detecting same-language rows and # removing language detection from translation prompts. # @LAYER Plugin -# @RELATION DEPENDS_ON -> [lingua] # @RATIONALE LLM-based language detection wastes tokens on every row. lingua is a # pure-Python library (pre-built wheel) that runs ~0.1ms/call with ~98% # accuracy, vs ~90% for LLM-based detection at ~100x+ cost. @@ -47,20 +46,24 @@ _COMMON_SOURCE_CODES: set[str] = { _detector_cache: dict[str, LanguageDetector] = {} +# #region _detector_cache_key [C:1] [TYPE Function] [SEMANTICS cache, key] +# @BRIEF Build a deterministic cache key from target languages list. def _detector_cache_key(target_languages: list[str] | None) -> str: """Build a cache key from the set of target languages.""" if not target_languages: return "__default" return ",".join(sorted(set(target_languages))) +# #endregion _detector_cache_key +# #region build_detector [C:2] [TYPE Function] [SEMANTICS language, detection, builder] +# @BRIEF Build a LanguageDetector restricted to target + common source languages. +# Restricting the language set improves detection speed (~5x faster than +# all 87 languages). The detector includes both target languages (for +# pre-filtering) and common source languages (for metadata). def build_detector(target_languages: list[str] | None = None) -> LanguageDetector: """Build a LanguageDetector restricted to target + common source languages. - Restricting the language set improves detection speed (~5x faster than - all 87 languages). The detector includes both target languages (for - pre-filtering) and common source languages (for metadata). - Args: target_languages: List of BCP-47 target language codes (e.g. ["ru", "de"]). If None or empty, falls back to a default set. @@ -81,8 +84,12 @@ def build_detector(target_languages: list[str] | None = None) -> LanguageDetecto return LanguageDetectorBuilder.from_all_spoken_languages().build() return LanguageDetectorBuilder.from_languages(*lingua_langs).build() +# #endregion build_detector +# #region get_detector [C:2] [TYPE Function] [SEMANTICS language, detection, cache] +# @BRIEF Get or create a cached detector for the given target languages. +# Cache keyed by sorted set of language codes — same key returns same instance. def get_detector(target_languages: list[str] | None = None) -> LanguageDetector: """Get or create a cached detector for the given target languages.""" key = _detector_cache_key(target_languages) @@ -91,8 +98,12 @@ def get_detector(target_languages: list[str] | None = None) -> LanguageDetector: detector = build_detector(target_languages) _detector_cache[key] = detector return detector +# #endregion get_detector +# #region detect_language [C:2] [TYPE Function] [SEMANTICS language, detection] +# @BRIEF Detect the language of a single text string. Returns BCP-47 code or "und". +# @RAISES Does not raise — all exceptions caught internally and return "und". def detect_language(text: str, detector: LanguageDetector) -> str: """Detect the language of a text string using the given detector. @@ -114,8 +125,13 @@ def detect_language(text: str, detector: LanguageDetector) -> str: return iso.name.lower() except Exception: return "und" +# #endregion detect_language +# #region batch_detect [C:3] [TYPE Function] [SEMANTICS language, detection, batch] +# @BRIEF Detect language for multiple texts in batch. Builds detector if not provided. +# @RELATION DEPENDS_ON -> [detect_language] +# @RELATION DEPENDS_ON -> [get_detector] def batch_detect( texts: list[str], target_languages: list[str] | None = None, @@ -135,4 +151,5 @@ def batch_detect( detector = get_detector(target_languages) return [detect_language(t, detector) for t in texts] +# #endregion batch_detect # #endregion LanguageDetectService