chore: remainder — backend test infra, agent config, docker, i18n, frontend ui
- Backend: alembic env, config manager/models, dependencies, translate plugin - Backend tests: async_sync_regression, integration tests, git services, test_agent - Docker: docker-compose.yml updates - Agent: qa-tester.md update, semantics-testing SKILL.md update - Frontend: TopNavbar, sidebarNavigation, FeaturesSettings, FeatureGate - i18n: assistant.json en/ru locale updates - New: frontend/src/lib/components/agent/ directory
This commit is contained in:
383
backend/tests/plugins/translate/test_utils.py
Normal file
383
backend/tests/plugins/translate/test_utils.py
Normal file
@@ -0,0 +1,383 @@
|
||||
# #region Test.Translate.Utils [C:3] [TYPE Module] [SEMANTICS test,translate,utils,hash,dictionary]
|
||||
# @BRIEF Tests for _utils.py — term normalization, delimiter detection, dictionary enforcement,
|
||||
# source hashing, cache lookup, key hashing, token estimation.
|
||||
# @RELATION BINDS_TO -> [TranslationUtils]
|
||||
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "src"))
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from src.plugins.translate._utils import (
|
||||
_normalize_term,
|
||||
_detect_delimiter,
|
||||
_enforce_dictionary,
|
||||
_compute_source_hash,
|
||||
_check_translation_cache,
|
||||
_compute_key_hash,
|
||||
estimate_row_tokens,
|
||||
)
|
||||
|
||||
|
||||
class TestNormalizeTerm:
|
||||
"""_normalize_term — NFC normalization, lowercasing, whitespace."""
|
||||
|
||||
# #region test_normalize_term_basic [C:1] [TYPE Function]
|
||||
def test_normalize_term_basic(self):
|
||||
"""Basic string is lowercased and stripped."""
|
||||
assert _normalize_term(" Hello World ") == "hello world"
|
||||
# #endregion test_normalize_term_basic
|
||||
|
||||
# #region test_normalize_term_nfc [C:1] [TYPE Function]
|
||||
def test_normalize_term_nfc(self):
|
||||
"""NFC normalization is applied."""
|
||||
assert _normalize_term("café") == "café"
|
||||
# #endregion test_normalize_term_nfc
|
||||
|
||||
# #region test_normalize_term_empty [C:1] [TYPE Function]
|
||||
def test_normalize_term_empty(self):
|
||||
"""Empty string returns empty string."""
|
||||
assert _normalize_term("") == ""
|
||||
# #endregion test_normalize_term_empty
|
||||
|
||||
# #region test_normalize_term_none [C:1] [TYPE Function]
|
||||
def test_normalize_term_none(self):
|
||||
"""None or whitespace-only returns empty."""
|
||||
assert _normalize_term(" ") == ""
|
||||
# #endregion test_normalize_term_none
|
||||
|
||||
|
||||
class TestDetectDelimiter:
|
||||
"""_detect_delimiter — CSV vs TSV detection."""
|
||||
|
||||
# #region test_detect_delimiter_csv [C:1] [TYPE Function]
|
||||
def test_detect_delimiter_csv(self):
|
||||
"""More commas than tabs → comma."""
|
||||
assert _detect_delimiter("a,b,c") == ","
|
||||
# #endregion test_detect_delimiter_csv
|
||||
|
||||
# #region test_detect_delimiter_tsv [C:1] [TYPE Function]
|
||||
def test_detect_delimiter_tsv(self):
|
||||
"""More tabs than commas → tab."""
|
||||
assert _detect_delimiter("a\tb\tc") == "\t"
|
||||
# #endregion test_detect_delimiter_tsv
|
||||
|
||||
# #region test_detect_delimiter_empty [C:1] [TYPE Function]
|
||||
def test_detect_delimiter_empty(self):
|
||||
"""Empty header defaults to comma."""
|
||||
assert _detect_delimiter("") == ","
|
||||
# #endregion test_detect_delimiter_empty
|
||||
|
||||
# #region test_detect_delimiter_tie [C:1] [TYPE Function]
|
||||
def test_detect_delimiter_tie(self):
|
||||
"""Equal counts defaults to comma (tab_count not > comma_count)."""
|
||||
assert _detect_delimiter("a,b\tc") == ","
|
||||
# #endregion test_detect_delimiter_tie
|
||||
|
||||
|
||||
class TestEnforceDictionary:
|
||||
"""_enforce_dictionary — post-process LLM output."""
|
||||
|
||||
# #region test_enforce_no_matches [C:1] [TYPE Function]
|
||||
def test_enforce_no_matches(self):
|
||||
"""Empty dict_matches list → no change."""
|
||||
vals = {"ru": "привет"}
|
||||
_enforce_dictionary("hello", vals, [], "b1", "r1")
|
||||
assert vals["ru"] == "привет"
|
||||
# #endregion test_enforce_no_matches
|
||||
|
||||
# #region test_enforce_source_term_present [C:1] [TYPE Function]
|
||||
def test_enforce_source_term_present(self):
|
||||
"""Source term found in text, target term missing in translation → replaced."""
|
||||
vals = {"ru": "мир hello мир"}
|
||||
_enforce_dictionary(
|
||||
"hello world",
|
||||
vals,
|
||||
[{"source_term": "hello", "target_term": "привет"}],
|
||||
"b1", "r1",
|
||||
)
|
||||
assert vals["ru"] == "мир привет мир"
|
||||
# #endregion test_enforce_source_term_present
|
||||
|
||||
# #region test_enforce_target_already_present [C:1] [TYPE Function]
|
||||
def test_enforce_target_already_present(self):
|
||||
"""Target term already in translation → no change."""
|
||||
vals = {"ru": "привет мир"}
|
||||
_enforce_dictionary(
|
||||
"hello world",
|
||||
vals,
|
||||
[{"source_term": "hello", "target_term": "привет"}],
|
||||
"b1", "r1",
|
||||
)
|
||||
assert vals["ru"] == "привет мир"
|
||||
# #endregion test_enforce_target_already_present
|
||||
|
||||
# #region test_enforce_empty_src_or_tgt [C:1] [TYPE Function]
|
||||
def test_enforce_empty_src_or_tgt(self):
|
||||
"""Empty source_term or target_term → entry skipped."""
|
||||
vals = {"ru": "hello world"}
|
||||
_enforce_dictionary(
|
||||
"hello world",
|
||||
vals,
|
||||
[{"source_term": "", "target_term": "привет"}, # skipped: empty source
|
||||
{"source_term": "hello", "target_term": ""}, # skipped: empty target
|
||||
{"source_term": "hello", "target_term": "привет"}], # valid → replaces
|
||||
"b1", "r1",
|
||||
)
|
||||
assert vals["ru"] == "привет world"
|
||||
# #endregion test_enforce_empty_src_or_tgt
|
||||
|
||||
# #region test_enforce_empty_val [C:1] [TYPE Function]
|
||||
def test_enforce_empty_val(self):
|
||||
"""Empty translation value for a language → skipped."""
|
||||
vals = {"ru": ""}
|
||||
_enforce_dictionary(
|
||||
"hello",
|
||||
vals,
|
||||
[{"source_term": "hello", "target_term": "привет"}],
|
||||
"b1", "r1",
|
||||
)
|
||||
assert vals["ru"] == "" # unchanged
|
||||
# #endregion test_enforce_empty_val
|
||||
|
||||
# #region test_enforce_regex_match [C:1] [TYPE Function]
|
||||
def test_enforce_regex_match(self):
|
||||
"""Regex-based dictionary match works."""
|
||||
vals = {"ru": "цена 123"}
|
||||
_enforce_dictionary(
|
||||
"price is 123",
|
||||
vals,
|
||||
[{"source_term": r"\d+", "target_term": "NUMBER", "is_regex": True}],
|
||||
"b1", "r1",
|
||||
)
|
||||
assert "NUMBER" in vals["ru"]
|
||||
# #endregion test_enforce_regex_match
|
||||
|
||||
# #region test_enforce_regex_no_match_source [C:1] [TYPE Function]
|
||||
def test_enforce_regex_no_match_source(self):
|
||||
"""Regex does not match source text → skip."""
|
||||
vals = {"ru": "hello"}
|
||||
_enforce_dictionary(
|
||||
"hello world",
|
||||
vals,
|
||||
[{"source_term": r"\d+", "target_term": "NUMBER", "is_regex": True}],
|
||||
"b1", "r1",
|
||||
)
|
||||
assert vals["ru"] == "hello" # unchanged
|
||||
# #endregion test_enforce_regex_no_match_source
|
||||
|
||||
# #region test_enforce_bad_regex_skipped [C:1] [TYPE Function]
|
||||
def test_enforce_bad_regex_skipped(self):
|
||||
"""Invalid regex pattern → entry skipped without crash."""
|
||||
vals = {"ru": "hello"}
|
||||
_enforce_dictionary(
|
||||
"hello world",
|
||||
vals,
|
||||
[{"source_term": r"[invalid", "target_term": "NUMBER", "is_regex": True}],
|
||||
"b1", "r1",
|
||||
)
|
||||
assert vals["ru"] == "hello"
|
||||
# #endregion test_enforce_bad_regex_skipped
|
||||
|
||||
# #region test_enforce_no_source_text [C:1] [TYPE Function]
|
||||
def test_enforce_no_source_text(self):
|
||||
"""Empty source_text → returns early."""
|
||||
vals = {"ru": "test"}
|
||||
_enforce_dictionary("", vals, [{"source_term": "x", "target_term": "y"}], "b1", "r1")
|
||||
assert vals["ru"] == "test"
|
||||
# #endregion test_enforce_no_source_text
|
||||
|
||||
# #region test_enforce_multi_lang [C:1] [TYPE Function]
|
||||
def test_enforce_multi_lang(self):
|
||||
"""Enforcement applied to all languages that still contain the source term."""
|
||||
vals = {"ru": "мир hello", "de": "hallo hello"}
|
||||
_enforce_dictionary(
|
||||
"hello world",
|
||||
vals,
|
||||
[{"source_term": "hello", "target_term": "привет"}],
|
||||
"b1", "r1",
|
||||
)
|
||||
assert vals["ru"] == "мир привет"
|
||||
assert vals["de"] == "hallo привет"
|
||||
# #endregion test_enforce_multi_lang
|
||||
|
||||
|
||||
# #region test_enforce_source_not_in_text [C:1] [TYPE Function]
|
||||
def test_enforce_source_not_in_text(self):
|
||||
"""Source term not in source text → skip (line 90 path)."""
|
||||
vals = {"ru": "test"}
|
||||
_enforce_dictionary(
|
||||
"hello world",
|
||||
vals,
|
||||
[{"source_term": "goodbye", "target_term": "до свидания"}],
|
||||
"b1", "r1",
|
||||
)
|
||||
assert vals["ru"] == "test" # unchanged
|
||||
# #endregion test_enforce_source_not_in_text
|
||||
|
||||
|
||||
|
||||
|
||||
class TestComputeSourceHash:
|
||||
"""_compute_source_hash — deterministic cache key."""
|
||||
|
||||
# #region test_hash_basic [C:1] [TYPE Function]
|
||||
def test_hash_basic(self):
|
||||
"""Basic hash without context or config."""
|
||||
h = _compute_source_hash("hello", {}, None, None)
|
||||
assert isinstance(h, str)
|
||||
assert len(h) == 64 # SHA256 hex
|
||||
# #endregion test_hash_basic
|
||||
|
||||
# #region test_hash_deterministic [C:1] [TYPE Function]
|
||||
def test_hash_deterministic(self):
|
||||
"""Same inputs → same hash."""
|
||||
h1 = _compute_source_hash("hello", {"id": 1}, "dict-hash", "cfg-hash", ["id"])
|
||||
h2 = _compute_source_hash("hello", {"id": 1}, "dict-hash", "cfg-hash", ["id"])
|
||||
assert h1 == h2
|
||||
# #endregion test_hash_deterministic
|
||||
|
||||
# #region test_hash_with_context [C:1] [TYPE Function]
|
||||
def test_hash_with_context(self):
|
||||
"""Context keys included in hash."""
|
||||
h = _compute_source_hash("hello", {"key_col": "val1", "other": "val2"}, None, None, ["key_col"])
|
||||
assert isinstance(h, str)
|
||||
# #endregion test_hash_with_context
|
||||
|
||||
# #region test_hash_differs_without_context [C:1] [TYPE Function]
|
||||
def test_hash_differs_without_context(self):
|
||||
"""Without context_keys, context fields are excluded."""
|
||||
h1 = _compute_source_hash("hello", {"key": "a"}, None, None)
|
||||
h2 = _compute_source_hash("hello", {"key": "b"}, None, None)
|
||||
assert h1 == h2 # context excluded when no context_keys
|
||||
# #endregion test_hash_differs_without_context
|
||||
|
||||
|
||||
class TestCheckTranslationCache:
|
||||
"""_check_translation_cache — DB cache lookup."""
|
||||
|
||||
# #region test_cache_miss [C:1] [TYPE Function]
|
||||
def test_cache_miss(self):
|
||||
"""No cached record → returns None."""
|
||||
db = MagicMock()
|
||||
db.query.return_value.options.return_value.filter.return_value.order_by.return_value.first.return_value = None
|
||||
result = _check_translation_cache(db, "hash-123")
|
||||
assert result is None
|
||||
# #endregion test_cache_miss
|
||||
|
||||
# #region test_cache_hit [C:1] [TYPE Function]
|
||||
def test_cache_hit(self):
|
||||
"""Cached record with languages → returns lang dict."""
|
||||
db = MagicMock()
|
||||
mock_lang_ru = MagicMock()
|
||||
mock_lang_ru.language_code = "ru"
|
||||
mock_lang_ru.status = "translated"
|
||||
mock_lang_ru.final_value = "привет"
|
||||
mock_lang_de = MagicMock()
|
||||
mock_lang_de.language_code = "de"
|
||||
mock_lang_de.status = "translated"
|
||||
mock_lang_de.final_value = "hallo"
|
||||
|
||||
mock_record = MagicMock()
|
||||
mock_record.languages = [mock_lang_ru, mock_lang_de]
|
||||
db.query.return_value.options.return_value.filter.return_value.order_by.return_value.first.return_value = mock_record
|
||||
|
||||
result = _check_translation_cache(db, "hash-123")
|
||||
assert result == {"ru": "привет", "de": "hallo"}
|
||||
# #endregion test_cache_hit
|
||||
|
||||
# #region test_cache_hit_partial [C:1] [TYPE Function]
|
||||
def test_cache_hit_partial(self):
|
||||
"""Only 'translated' status languages are included."""
|
||||
db = MagicMock()
|
||||
mock_lang_translated = MagicMock()
|
||||
mock_lang_translated.language_code = "ru"
|
||||
mock_lang_translated.status = "translated"
|
||||
mock_lang_translated.final_value = "привет"
|
||||
mock_lang_pending = MagicMock()
|
||||
mock_lang_pending.language_code = "de"
|
||||
mock_lang_pending.status = "pending"
|
||||
mock_lang_pending.final_value = None
|
||||
|
||||
mock_record = MagicMock()
|
||||
mock_record.languages = [mock_lang_translated, mock_lang_pending]
|
||||
db.query.return_value.options.return_value.filter.return_value.order_by.return_value.first.return_value = mock_record
|
||||
|
||||
result = _check_translation_cache(db, "hash-123")
|
||||
assert result == {"ru": "привет"}
|
||||
# #endregion test_cache_hit_partial
|
||||
|
||||
# #region test_cache_no_values [C:1] [TYPE Function]
|
||||
def test_cache_no_values(self):
|
||||
"""No translated languages → returns None."""
|
||||
db = MagicMock()
|
||||
mock_lang = MagicMock()
|
||||
mock_lang.status = "pending"
|
||||
mock_lang.final_value = None
|
||||
|
||||
mock_record = MagicMock()
|
||||
mock_record.languages = [mock_lang]
|
||||
db.query.return_value.options.return_value.filter.return_value.order_by.return_value.first.return_value = mock_record
|
||||
|
||||
result = _check_translation_cache(db, "hash-123")
|
||||
assert result is None
|
||||
# #endregion test_cache_no_values
|
||||
|
||||
|
||||
class TestComputeKeyHash:
|
||||
"""_compute_key_hash — stable hash from dict."""
|
||||
|
||||
# #region test_key_hash_stable [C:1] [TYPE Function]
|
||||
def test_key_hash_stable(self):
|
||||
"""Same dict → same hash."""
|
||||
h1 = _compute_key_hash({"a": 1, "b": 2})
|
||||
h2 = _compute_key_hash({"b": 2, "a": 1})
|
||||
assert h1 == h2
|
||||
# #endregion test_key_hash_stable
|
||||
|
||||
# #region test_key_hash_different [C:1] [TYPE Function]
|
||||
def test_key_hash_different(self):
|
||||
"""Different dicts → different hashes."""
|
||||
h1 = _compute_key_hash({"a": 1})
|
||||
h2 = _compute_key_hash({"a": 2})
|
||||
assert h1 != h2
|
||||
# #endregion test_key_hash_different
|
||||
|
||||
|
||||
class TestEstimateRowTokens:
|
||||
"""estimate_row_tokens — token estimation."""
|
||||
|
||||
# #region test_estimate_basic [C:1] [TYPE Function]
|
||||
def test_estimate_basic(self):
|
||||
"""Basic text with no context. estimate_row_tokens calls _estimate_tokens_for_text twice
|
||||
(once for source, once for empty context)."""
|
||||
job = MagicMock()
|
||||
job.context_columns = []
|
||||
with patch("src.plugins.translate._token_budget._estimate_tokens_for_text", side_effect=[10, 3]):
|
||||
tokens = estimate_row_tokens("hello", {}, job)
|
||||
assert tokens == 13
|
||||
# #endregion test_estimate_basic
|
||||
|
||||
# #region test_estimate_with_context [C:1] [TYPE Function]
|
||||
def test_estimate_with_context(self):
|
||||
"""Context columns included in estimate."""
|
||||
job = MagicMock()
|
||||
job.context_columns = ["desc"]
|
||||
with patch("src.plugins.translate._token_budget._estimate_tokens_for_text", side_effect=[10, 5]):
|
||||
tokens = estimate_row_tokens("hello", {"desc": "long description"}, job)
|
||||
assert tokens == 15
|
||||
# #endregion test_estimate_with_context
|
||||
|
||||
# #region test_estimate_no_source_data [C:1] [TYPE Function]
|
||||
def test_estimate_no_source_data(self):
|
||||
"""None source_data still works."""
|
||||
job = MagicMock()
|
||||
job.context_columns = []
|
||||
with patch("src.plugins.translate._token_budget._estimate_tokens_for_text", side_effect=[10, 3]):
|
||||
tokens = estimate_row_tokens("hello", None, job)
|
||||
assert tokens == 13
|
||||
# #endregion test_estimate_no_source_data
|
||||
# #endregion Test.Translate.Utils
|
||||
Reference in New Issue
Block a user