# #region Test.Translate.Utils [C:3] [TYPE Module] [SEMANTICS test,translate,utils,hash,dictionary] # @BRIEF Tests for _utils.py — term normalization, delimiter detection, dictionary enforcement, # source hashing, cache lookup, key hashing, token estimation. # @RELATION BINDS_TO -> [TranslationUtils] import sys import json from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "src")) import pytest from unittest.mock import MagicMock, patch from src.plugins.translate._utils import ( _normalize_term, _detect_delimiter, _enforce_dictionary, _compute_source_hash, _check_translation_cache, _compute_key_hash, estimate_row_tokens, ) class TestNormalizeTerm: """_normalize_term — NFC normalization, lowercasing, whitespace.""" # #region test_normalize_term_basic [C:1] [TYPE Function] def test_normalize_term_basic(self): """Basic string is lowercased and stripped.""" assert _normalize_term(" Hello World ") == "hello world" # #endregion test_normalize_term_basic # #region test_normalize_term_nfc [C:1] [TYPE Function] def test_normalize_term_nfc(self): """NFC normalization is applied.""" assert _normalize_term("café") == "café" # #endregion test_normalize_term_nfc # #region test_normalize_term_empty [C:1] [TYPE Function] def test_normalize_term_empty(self): """Empty string returns empty string.""" assert _normalize_term("") == "" # #endregion test_normalize_term_empty # #region test_normalize_term_none [C:1] [TYPE Function] def test_normalize_term_none(self): """None or whitespace-only returns empty.""" assert _normalize_term(" ") == "" # #endregion test_normalize_term_none class TestDetectDelimiter: """_detect_delimiter — CSV vs TSV detection.""" # #region test_detect_delimiter_csv [C:1] [TYPE Function] def test_detect_delimiter_csv(self): """More commas than tabs → comma.""" assert _detect_delimiter("a,b,c") == "," # #endregion test_detect_delimiter_csv # #region test_detect_delimiter_tsv [C:1] [TYPE Function] def test_detect_delimiter_tsv(self): """More tabs than commas → tab.""" assert _detect_delimiter("a\tb\tc") == "\t" # #endregion test_detect_delimiter_tsv # #region test_detect_delimiter_empty [C:1] [TYPE Function] def test_detect_delimiter_empty(self): """Empty header defaults to comma.""" assert _detect_delimiter("") == "," # #endregion test_detect_delimiter_empty # #region test_detect_delimiter_tie [C:1] [TYPE Function] def test_detect_delimiter_tie(self): """Equal counts defaults to comma (tab_count not > comma_count).""" assert _detect_delimiter("a,b\tc") == "," # #endregion test_detect_delimiter_tie class TestEnforceDictionary: """_enforce_dictionary — post-process LLM output.""" # #region test_enforce_no_matches [C:1] [TYPE Function] def test_enforce_no_matches(self): """Empty dict_matches list → no change.""" vals = {"ru": "привет"} _enforce_dictionary("hello", vals, [], "b1", "r1") assert vals["ru"] == "привет" # #endregion test_enforce_no_matches # #region test_enforce_source_term_present [C:1] [TYPE Function] def test_enforce_source_term_present(self): """Source term found in text, target term missing in translation → replaced.""" vals = {"ru": "мир hello мир"} _enforce_dictionary( "hello world", vals, [{"source_term": "hello", "target_term": "привет"}], "b1", "r1", ) assert vals["ru"] == "мир привет мир" # #endregion test_enforce_source_term_present # #region test_enforce_target_already_present [C:1] [TYPE Function] def test_enforce_target_already_present(self): """Target term already in translation → no change.""" vals = {"ru": "привет мир"} _enforce_dictionary( "hello world", vals, [{"source_term": "hello", "target_term": "привет"}], "b1", "r1", ) assert vals["ru"] == "привет мир" # #endregion test_enforce_target_already_present # #region test_enforce_empty_src_or_tgt [C:1] [TYPE Function] def test_enforce_empty_src_or_tgt(self): """Empty source_term or target_term → entry skipped.""" vals = {"ru": "hello world"} _enforce_dictionary( "hello world", vals, [{"source_term": "", "target_term": "привет"}, # skipped: empty source {"source_term": "hello", "target_term": ""}, # skipped: empty target {"source_term": "hello", "target_term": "привет"}], # valid → replaces "b1", "r1", ) assert vals["ru"] == "привет world" # #endregion test_enforce_empty_src_or_tgt # #region test_enforce_empty_val [C:1] [TYPE Function] def test_enforce_empty_val(self): """Empty translation value for a language → skipped.""" vals = {"ru": ""} _enforce_dictionary( "hello", vals, [{"source_term": "hello", "target_term": "привет"}], "b1", "r1", ) assert vals["ru"] == "" # unchanged # #endregion test_enforce_empty_val # #region test_enforce_regex_match [C:1] [TYPE Function] def test_enforce_regex_match(self): """Regex-based dictionary match works.""" vals = {"ru": "цена 123"} _enforce_dictionary( "price is 123", vals, [{"source_term": r"\d+", "target_term": "NUMBER", "is_regex": True}], "b1", "r1", ) assert "NUMBER" in vals["ru"] # #endregion test_enforce_regex_match # #region test_enforce_regex_no_match_source [C:1] [TYPE Function] def test_enforce_regex_no_match_source(self): """Regex does not match source text → skip.""" vals = {"ru": "hello"} _enforce_dictionary( "hello world", vals, [{"source_term": r"\d+", "target_term": "NUMBER", "is_regex": True}], "b1", "r1", ) assert vals["ru"] == "hello" # unchanged # #endregion test_enforce_regex_no_match_source # #region test_enforce_bad_regex_skipped [C:1] [TYPE Function] def test_enforce_bad_regex_skipped(self): """Invalid regex pattern → entry skipped without crash.""" vals = {"ru": "hello"} _enforce_dictionary( "hello world", vals, [{"source_term": r"[invalid", "target_term": "NUMBER", "is_regex": True}], "b1", "r1", ) assert vals["ru"] == "hello" # #endregion test_enforce_bad_regex_skipped # #region test_enforce_no_source_text [C:1] [TYPE Function] def test_enforce_no_source_text(self): """Empty source_text → returns early.""" vals = {"ru": "test"} _enforce_dictionary("", vals, [{"source_term": "x", "target_term": "y"}], "b1", "r1") assert vals["ru"] == "test" # #endregion test_enforce_no_source_text # #region test_enforce_multi_lang [C:1] [TYPE Function] def test_enforce_multi_lang(self): """Enforcement applied to all languages that still contain the source term.""" vals = {"ru": "мир hello", "de": "hallo hello"} _enforce_dictionary( "hello world", vals, [{"source_term": "hello", "target_term": "привет"}], "b1", "r1", ) assert vals["ru"] == "мир привет" assert vals["de"] == "hallo привет" # #endregion test_enforce_multi_lang # #region test_enforce_source_not_in_text [C:1] [TYPE Function] def test_enforce_source_not_in_text(self): """Source term not in source text → skip (line 90 path).""" vals = {"ru": "test"} _enforce_dictionary( "hello world", vals, [{"source_term": "goodbye", "target_term": "до свидания"}], "b1", "r1", ) assert vals["ru"] == "test" # unchanged # #endregion test_enforce_source_not_in_text class TestComputeSourceHash: """_compute_source_hash — deterministic cache key.""" # #region test_hash_basic [C:1] [TYPE Function] def test_hash_basic(self): """Basic hash without context or config.""" h = _compute_source_hash("hello", {}, None, None) assert isinstance(h, str) assert len(h) == 64 # SHA256 hex # #endregion test_hash_basic # #region test_hash_deterministic [C:1] [TYPE Function] def test_hash_deterministic(self): """Same inputs → same hash.""" h1 = _compute_source_hash("hello", {"id": 1}, "dict-hash", "cfg-hash", ["id"]) h2 = _compute_source_hash("hello", {"id": 1}, "dict-hash", "cfg-hash", ["id"]) assert h1 == h2 # #endregion test_hash_deterministic # #region test_hash_with_context [C:1] [TYPE Function] def test_hash_with_context(self): """Context keys included in hash.""" h = _compute_source_hash("hello", {"key_col": "val1", "other": "val2"}, None, None, ["key_col"]) assert isinstance(h, str) # #endregion test_hash_with_context # #region test_hash_differs_without_context [C:1] [TYPE Function] def test_hash_differs_without_context(self): """Without context_keys, context fields are excluded.""" h1 = _compute_source_hash("hello", {"key": "a"}, None, None) h2 = _compute_source_hash("hello", {"key": "b"}, None, None) assert h1 == h2 # context excluded when no context_keys # #endregion test_hash_differs_without_context class TestCheckTranslationCache: """_check_translation_cache — DB cache lookup.""" # #region test_cache_miss [C:1] [TYPE Function] def test_cache_miss(self): """No cached record → returns None.""" db = MagicMock() db.query.return_value.options.return_value.filter.return_value.order_by.return_value.first.return_value = None result = _check_translation_cache(db, "hash-123") assert result is None # #endregion test_cache_miss # #region test_cache_hit [C:1] [TYPE Function] def test_cache_hit(self): """Cached record with languages → returns lang dict.""" db = MagicMock() mock_lang_ru = MagicMock() mock_lang_ru.language_code = "ru" mock_lang_ru.status = "translated" mock_lang_ru.final_value = "привет" mock_lang_de = MagicMock() mock_lang_de.language_code = "de" mock_lang_de.status = "translated" mock_lang_de.final_value = "hallo" mock_record = MagicMock() mock_record.languages = [mock_lang_ru, mock_lang_de] db.query.return_value.options.return_value.filter.return_value.order_by.return_value.first.return_value = mock_record result = _check_translation_cache(db, "hash-123") assert result == {"ru": "привет", "de": "hallo"} # #endregion test_cache_hit # #region test_cache_hit_partial [C:1] [TYPE Function] def test_cache_hit_partial(self): """Only 'translated' status languages are included.""" db = MagicMock() mock_lang_translated = MagicMock() mock_lang_translated.language_code = "ru" mock_lang_translated.status = "translated" mock_lang_translated.final_value = "привет" mock_lang_pending = MagicMock() mock_lang_pending.language_code = "de" mock_lang_pending.status = "pending" mock_lang_pending.final_value = None mock_record = MagicMock() mock_record.languages = [mock_lang_translated, mock_lang_pending] db.query.return_value.options.return_value.filter.return_value.order_by.return_value.first.return_value = mock_record result = _check_translation_cache(db, "hash-123") assert result == {"ru": "привет"} # #endregion test_cache_hit_partial # #region test_cache_no_values [C:1] [TYPE Function] def test_cache_no_values(self): """No translated languages → returns None.""" db = MagicMock() mock_lang = MagicMock() mock_lang.status = "pending" mock_lang.final_value = None mock_record = MagicMock() mock_record.languages = [mock_lang] db.query.return_value.options.return_value.filter.return_value.order_by.return_value.first.return_value = mock_record result = _check_translation_cache(db, "hash-123") assert result is None # #endregion test_cache_no_values class TestComputeKeyHash: """_compute_key_hash — stable hash from dict.""" # #region test_key_hash_stable [C:1] [TYPE Function] def test_key_hash_stable(self): """Same dict → same hash.""" h1 = _compute_key_hash({"a": 1, "b": 2}) h2 = _compute_key_hash({"b": 2, "a": 1}) assert h1 == h2 # #endregion test_key_hash_stable # #region test_key_hash_different [C:1] [TYPE Function] def test_key_hash_different(self): """Different dicts → different hashes.""" h1 = _compute_key_hash({"a": 1}) h2 = _compute_key_hash({"a": 2}) assert h1 != h2 # #endregion test_key_hash_different class TestEstimateRowTokens: """estimate_row_tokens — token estimation.""" # #region test_estimate_basic [C:1] [TYPE Function] def test_estimate_basic(self): """Basic text with no context. estimate_row_tokens calls _estimate_tokens_for_text twice (once for source, once for empty context).""" job = MagicMock() job.context_columns = [] with patch("src.plugins.translate._token_budget._estimate_tokens_for_text", side_effect=[10, 3]): tokens = estimate_row_tokens("hello", {}, job) assert tokens == 13 # #endregion test_estimate_basic # #region test_estimate_with_context [C:1] [TYPE Function] def test_estimate_with_context(self): """Context columns included in estimate.""" job = MagicMock() job.context_columns = ["desc"] with patch("src.plugins.translate._token_budget._estimate_tokens_for_text", side_effect=[10, 5]): tokens = estimate_row_tokens("hello", {"desc": "long description"}, job) assert tokens == 15 # #endregion test_estimate_with_context # #region test_estimate_no_source_data [C:1] [TYPE Function] def test_estimate_no_source_data(self): """None source_data still works.""" job = MagicMock() job.context_columns = [] with patch("src.plugins.translate._token_budget._estimate_tokens_for_text", side_effect=[10, 3]): tokens = estimate_row_tokens("hello", None, job) assert tokens == 13 # #endregion test_estimate_no_source_data # #endregion Test.Translate.Utils