busya
c488a63dc0
- Add async HTTP-based LLM transport (_llm_async_http.py) - Add orthogonal LLM call tests - Improve language detection (_lang_detect.py) and batch insert - Update translate schemas, service utils, preview constants/prompts - Add TranslateHistoryModel with pagination and filtering - Update agent confirmation, persistence, langgraph setup, run, tools - Improve LLM health checking in shared module - Update translate runs API, history route
249 lines
8.0 KiB
Python
249 lines
8.0 KiB
Python
# #region Test.LanguageDetectService [C:2] [TYPE Module] [SEMANTICS test,language,detection]
|
|
# @BRIEF Tests for _lang_detect.py — language detection.
|
|
# @RELATION BINDS_TO -> [LanguageDetectService]
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "src"))
|
|
|
|
import pytest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from lingua import LanguageDetectorBuilder
|
|
|
|
from src.plugins.translate._lang_detect import (
|
|
_detector_cache_key,
|
|
build_detector,
|
|
get_detector,
|
|
detect_language,
|
|
_character_block_fallback,
|
|
_mark_suspicious_detections_und,
|
|
batch_detect,
|
|
)
|
|
|
|
|
|
class TestDetectorCacheKey:
|
|
"""_detector_cache_key."""
|
|
|
|
def test_none(self):
|
|
"""None returns '__default'."""
|
|
assert _detector_cache_key(None) == "__default"
|
|
|
|
def test_empty_list(self):
|
|
"""Empty list returns '__default'."""
|
|
assert _detector_cache_key([]) == "__default"
|
|
|
|
def test_sorted_key(self):
|
|
"""Languages are sorted for deterministic key."""
|
|
key = _detector_cache_key(["fr", "en", "de"])
|
|
assert key == "de,en,fr"
|
|
|
|
def test_duplicates(self):
|
|
"""Duplicates are removed."""
|
|
key = _detector_cache_key(["en", "en", "fr"])
|
|
assert key == "en,fr"
|
|
|
|
|
|
class TestBuildDetector:
|
|
"""build_detector."""
|
|
|
|
def test_with_target_languages(self):
|
|
"""Builds detector with specific languages."""
|
|
detector = build_detector(["ru", "de"])
|
|
assert detector is not None
|
|
|
|
def test_no_target_languages(self):
|
|
"""No target languages builds with common source codes."""
|
|
detector = build_detector(None)
|
|
assert detector is not None
|
|
|
|
def test_empty_target_languages(self):
|
|
"""Empty list builds with common source codes."""
|
|
detector = build_detector([])
|
|
assert detector is not None
|
|
|
|
def test_unknown_language_codes(self):
|
|
"""Unknown codes are ignored."""
|
|
detector = build_detector(["xx", "yy"])
|
|
# Should try to build; if no lingua_langs found, falls back to all spoken
|
|
assert detector is not None
|
|
|
|
def test_fallback_to_all_spoken(self):
|
|
"""When _CODE_TO_LANG has no matches, falls back to all spoken languages (line 93)."""
|
|
with patch("src.plugins.translate._lang_detect._CODE_TO_LANG", {}):
|
|
detector = build_detector(["en"])
|
|
assert detector is not None
|
|
|
|
|
|
class TestGetDetector:
|
|
"""get_detector — cached detector."""
|
|
|
|
def test_cached(self):
|
|
"""Same key returns same detector."""
|
|
with patch("src.plugins.translate._lang_detect._detector_cache", {}):
|
|
d1 = get_detector(["en", "fr"])
|
|
d2 = get_detector(["en", "fr"])
|
|
assert d1 is d2
|
|
|
|
def test_different_keys(self):
|
|
"""Different keys return different detectors."""
|
|
with patch("src.plugins.translate._lang_detect._detector_cache", {}):
|
|
d1 = get_detector(["en"])
|
|
d2 = get_detector(["fr"])
|
|
assert d1 is not d2
|
|
|
|
def test_default_key(self):
|
|
"""None returns a detector."""
|
|
with patch("src.plugins.translate._lang_detect._detector_cache", {}):
|
|
d = get_detector(None)
|
|
assert d is not None
|
|
|
|
|
|
class TestDetectLanguage:
|
|
"""detect_language."""
|
|
|
|
def test_empty_text(self):
|
|
"""Empty text returns 'und'."""
|
|
detector = build_detector(["en"])
|
|
assert detect_language("", detector) == "und"
|
|
|
|
def test_whitespace_only(self):
|
|
"""Whitespace-only returns 'und'."""
|
|
detector = build_detector(["en"])
|
|
assert detect_language(" ", detector) == "und"
|
|
|
|
def test_english_text(self):
|
|
"""English text detected."""
|
|
detector = build_detector(["en", "ru"])
|
|
result = detect_language("Hello, how are you today?", detector)
|
|
assert result == "en"
|
|
|
|
def test_russian_text(self):
|
|
"""Russian text detected."""
|
|
detector = build_detector(["ru", "en"])
|
|
result = detect_language("Привет, как дела?", detector)
|
|
assert result == "ru"
|
|
|
|
def test_exception_returns_und(self):
|
|
"""Exception in detection returns 'und'."""
|
|
detector = MagicMock()
|
|
detector.detect_language_of.side_effect = Exception("error")
|
|
result = detect_language("hello", detector)
|
|
assert result == "und"
|
|
|
|
def test_none_result(self):
|
|
"""None result from detector returns 'und'."""
|
|
detector = MagicMock()
|
|
detector.detect_language_of.return_value = None
|
|
result = detect_language("hello", detector)
|
|
assert result == "und"
|
|
|
|
|
|
class TestCharacterBlockFallback:
|
|
"""_character_block_fallback."""
|
|
|
|
def test_no_target_languages(self):
|
|
"""No target languages returns results unchanged."""
|
|
results = _character_block_fallback(["text"], ["und"], None)
|
|
assert results == ["und"]
|
|
|
|
def test_non_cyrillic_targets(self):
|
|
"""No Cyrillic targets returns results unchanged."""
|
|
results = _character_block_fallback(["text"], ["und"], ["en", "fr"])
|
|
assert results == ["und"]
|
|
|
|
def test_no_und_results(self):
|
|
"""No 'und' results returns unchanged."""
|
|
results = _character_block_fallback(["text"], ["en"], ["ru"])
|
|
assert results == ["en"]
|
|
|
|
def test_cyrillic_text_detected(self):
|
|
"""Cyrillic-dominant text gets assigned ru."""
|
|
results = _character_block_fallback(
|
|
["привет мир текст"],
|
|
["und"],
|
|
["ru", "en"],
|
|
)
|
|
assert results[0] == "ru"
|
|
|
|
def test_non_cyrillic_text_stays_und(self):
|
|
"""Non-Cyrillic text stays 'und'."""
|
|
results = _character_block_fallback(
|
|
["hello world 123"],
|
|
["und"],
|
|
["ru", "en"],
|
|
)
|
|
assert results[0] == "und"
|
|
|
|
def test_empty_text_stays_und(self):
|
|
"""Empty text stays 'und'."""
|
|
results = _character_block_fallback(
|
|
[""],
|
|
["und"],
|
|
["ru"],
|
|
)
|
|
assert results[0] == "und"
|
|
|
|
|
|
class TestSuspiciousDetections:
|
|
"""Suspicious short Cyrillic detections are routed to LLM as und."""
|
|
|
|
def test_short_cyrillic_uk_for_ru_target_becomes_und(self):
|
|
results = _mark_suspicious_detections_und(
|
|
["Оплачено 22.06"],
|
|
["uk"],
|
|
["ru"],
|
|
)
|
|
assert results == ["und"]
|
|
|
|
def test_short_cyrillic_it_for_ru_target_becomes_und(self):
|
|
results = _mark_suspicious_detections_und(
|
|
["Нет"],
|
|
["it"],
|
|
["ru"],
|
|
)
|
|
assert results == ["und"]
|
|
|
|
def test_long_mixed_text_keeps_detection(self):
|
|
text = "Contract with KZ Trading for Financing in China Необходимо подтверждение депозита"
|
|
results = _mark_suspicious_detections_und([text], ["en"], ["ru"])
|
|
assert results == ["en"]
|
|
|
|
def test_non_ru_target_keeps_detection(self):
|
|
results = _mark_suspicious_detections_und(
|
|
["Оплачено 22.06"],
|
|
["uk"],
|
|
["en"],
|
|
)
|
|
assert results == ["uk"]
|
|
|
|
|
|
class TestBatchDetect:
|
|
"""batch_detect."""
|
|
|
|
def test_basic_batch(self):
|
|
"""Detects languages for multiple texts."""
|
|
detector = build_detector(["en", "ru", "fr"])
|
|
results = batch_detect(["Hello", "Привет", "Bonjour"],
|
|
detector=detector)
|
|
assert len(results) == 3
|
|
|
|
def test_builds_detector_when_none(self):
|
|
"""Builds detector when not provided."""
|
|
results = batch_detect(["Hello"], target_languages=["en"])
|
|
assert len(results) == 1
|
|
|
|
def test_empty_texts(self):
|
|
"""Empty texts list returns empty list."""
|
|
results = batch_detect([])
|
|
assert results == []
|
|
|
|
def test_applies_fallback(self):
|
|
"""Cyrillic fallback is applied."""
|
|
results = batch_detect(
|
|
["привет мир"],
|
|
target_languages=["ru"],
|
|
)
|
|
assert len(results) == 1
|
|
# #endregion Test.LanguageDetectService
|