fix(translate): deduplicate bulk replace buttons with distinct labels
- Rename page-level button to 'Массовая замена по всем запускам' (bulk_replace_all key in i18n) to distinguish from run-level button - Remove duplicate Bulk Replace button from records table header in TranslationRunResult (kept only in result header) - Fix i18n path for bulk_replace_all (was incorrectly in run namespace)
This commit is contained in:
@@ -123,11 +123,11 @@ class TestClassify:
|
||||
assert len(pre_rows) == 1
|
||||
|
||||
def test_cache_partial_targets_llm_rows(self):
|
||||
"""Cache has only 'ru', but targets are [ru, en] → goes to LLM."""
|
||||
"""Cache has only 'en', targets [en, fr, zh], detected 'ru' → missing fr/zh → LLM."""
|
||||
svc = _make_service()
|
||||
rows = [_make_row(detected_lang="en",
|
||||
cached_lang_values={"ru": "текст"})]
|
||||
tls = _make_tls(["ru", "en"])
|
||||
rows = [_make_row(detected_lang="ru",
|
||||
cached_lang_values={"en": "translated"})]
|
||||
tls = _make_tls(["en", "fr", "zh"])
|
||||
|
||||
llm_rows, pre_rows = svc._classify(rows, None, tls)
|
||||
|
||||
@@ -206,6 +206,194 @@ class TestClassify:
|
||||
assert len(llm_rows) == 1
|
||||
|
||||
|
||||
# ── Cache Source-Language Exclusion Tests ───────────────────────────
|
||||
# Verifies the fix: tls may include the source language (e.g.
|
||||
# ["ru","en","fr","zh"]) but cache only contains translations
|
||||
# (en/fr/zh). _classify now filters detected source language from
|
||||
# cache-completeness check.
|
||||
|
||||
|
||||
# #region TestClassifyCacheSourceLang [C:3] [TYPE Class] [SEMANTICS test,classify,cache,source-lang,exclusion]
|
||||
# @BRIEF Verifies _classify correctly excludes detected source language
|
||||
# from cache-completeness check, preventing false-negative
|
||||
# routing to LLM when tls includes the source language.
|
||||
class TestClassifyCacheSourceLang:
|
||||
"""Tests for the non-source-tls filter in _classify()."""
|
||||
|
||||
# #region test_source_lang_in_tls_cache_all_translations [C:3] [TYPE Function]
|
||||
# @BRIEF Source lang in tls, cache has ALL translations → pre_rows.
|
||||
# @RATIONALE This is the exact production bug scenario: detected_lang="ru",
|
||||
# tls=["ru","en","fr","zh"], cache={"en":...,"fr":...,"zh":...}.
|
||||
# Old code: all(lc in cl for lc in tls) → "ru" not in cl → False → LLM.
|
||||
# New code: non_source_tls=["en","fr","zh"] → all present → pre_rows.
|
||||
def test_source_lang_in_tls_cache_all_translations(self):
|
||||
svc = _make_service()
|
||||
rows = [_make_row(
|
||||
detected_lang="ru",
|
||||
cached_lang_values={"en": "Hello", "fr": "Bonjour", "zh": "你好"},
|
||||
)]
|
||||
tls = _make_tls(["ru", "en", "fr", "zh"])
|
||||
|
||||
llm_rows, pre_rows = svc._classify(rows, None, tls)
|
||||
|
||||
assert len(llm_rows) == 0, (
|
||||
f"Expected 0 llm_rows (all cached), got {len(llm_rows)}"
|
||||
)
|
||||
assert len(pre_rows) == 1
|
||||
assert rows[0].get("_same_language") is True # ru in tls_lower
|
||||
# #endregion test_source_lang_in_tls_cache_all_translations
|
||||
|
||||
# #region test_source_lang_in_tls_cache_missing_one_translation [C:3] [TYPE Function]
|
||||
# @BRIEF Source lang in tls, cache MISSING one non-source lang → LLM.
|
||||
def test_source_lang_in_tls_cache_missing_one_translation(self):
|
||||
svc = _make_service()
|
||||
rows = [_make_row(
|
||||
detected_lang="ru",
|
||||
cached_lang_values={"en": "Hello", "fr": "Bonjour"}, # zh missing
|
||||
)]
|
||||
tls = _make_tls(["ru", "en", "fr", "zh"])
|
||||
|
||||
llm_rows, pre_rows = svc._classify(rows, None, tls)
|
||||
|
||||
assert len(llm_rows) == 1, (
|
||||
f"Expected 1 llm_row (zh not cached), got {len(llm_rows)}"
|
||||
)
|
||||
assert len(pre_rows) == 0
|
||||
# #endregion test_source_lang_in_tls_cache_missing_one_translation
|
||||
|
||||
# #region test_detected_und_tls_includes_und [C:2] [TYPE Function]
|
||||
# @BRIEF Detected lang "und": excluded from cache check like any detected source.
|
||||
# Cache has ALL non-und langs → row goes to pre_rows.
|
||||
def test_detected_und_tls_includes_und(self):
|
||||
svc = _make_service()
|
||||
rows = [_make_row(
|
||||
detected_lang="und",
|
||||
cached_lang_values={"en": "Hello", "fr": "Bonjour"},
|
||||
)]
|
||||
tls = _make_tls(["und", "en", "fr"])
|
||||
|
||||
llm_rows, pre_rows = svc._classify(rows, None, tls)
|
||||
|
||||
# non_source_tls = ["en","fr"] (und excluded)
|
||||
# both present in cache → pre_rows (correct: und is just "unknown")
|
||||
assert len(llm_rows) == 0, (
|
||||
f"Expected 0 llm_rows (und excluded, en/fr in cache), got {len(llm_rows)}"
|
||||
)
|
||||
assert len(pre_rows) == 1
|
||||
# #endregion test_detected_und_tls_includes_und
|
||||
|
||||
# #region test_empty_detected_lang_no_exclusion [C:2] [TYPE Function]
|
||||
# @BRIEF Empty detected language → non_source_tls equals full tls.
|
||||
def test_empty_detected_lang_no_exclusion(self):
|
||||
svc = _make_service()
|
||||
rows = [_make_row(
|
||||
detected_lang="",
|
||||
cached_lang_values={"en": "Hello", "ru": "Привет"},
|
||||
)]
|
||||
tls = _make_tls(["ru", "en"])
|
||||
|
||||
llm_rows, pre_rows = svc._classify(rows, None, tls)
|
||||
|
||||
# non_source_tls = ["ru","en"] (no filtering)
|
||||
# all present in cache → pre_rows
|
||||
assert len(llm_rows) == 0
|
||||
assert len(pre_rows) == 1
|
||||
# #endregion test_empty_detected_lang_no_exclusion
|
||||
|
||||
# #region test_source_lang_not_in_tls [C:2] [TYPE Function]
|
||||
# @BRIEF Detected source language is NOT in tls — no filtering occurs but
|
||||
# cache completeness still checked against full tls.
|
||||
def test_source_lang_not_in_tls(self):
|
||||
svc = _make_service()
|
||||
rows = [_make_row(
|
||||
detected_lang="ru", # ru not in target languages
|
||||
cached_lang_values={"en": "Hello", "fr": "Bonjour", "zh": "你好"},
|
||||
)]
|
||||
tls = _make_tls(["en", "fr", "zh"])
|
||||
|
||||
llm_rows, pre_rows = svc._classify(rows, None, tls)
|
||||
|
||||
# non_source_tls = ["en","fr","zh"] (ru not in tls, so all remain)
|
||||
# all present in cache → pre_rows
|
||||
assert len(llm_rows) == 0
|
||||
assert len(pre_rows) == 1
|
||||
# #endregion test_source_lang_not_in_tls
|
||||
|
||||
# #region test_multi_row_mixed_source_lang_cache [C:3] [TYPE Function]
|
||||
# @BRIEF Multiple rows with different detected languages and cache states.
|
||||
def test_multi_row_mixed_source_lang_cache(self):
|
||||
svc = _make_service()
|
||||
rows = [
|
||||
# Row 0: ru source, full cache → pre
|
||||
_make_row(detected_lang="ru",
|
||||
cached_lang_values={"en": "Hello", "fr": "Bonjour", "zh": "你好"}),
|
||||
# Row 1: fr source, partial cache → LLM (zh missing)
|
||||
_make_row(detected_lang="fr",
|
||||
cached_lang_values={"en": "Hello"}),
|
||||
# Row 2: no cache, ru source → LLM
|
||||
_make_row(detected_lang="ru", cached_lang_values=None),
|
||||
# Row 3: en source (not in tls), full cache → pre
|
||||
_make_row(detected_lang="en",
|
||||
cached_lang_values={"ru": "привет", "fr": "Bonjour", "zh": "你好"}),
|
||||
]
|
||||
# Give each row a unique index for verification
|
||||
for i, row in enumerate(rows):
|
||||
row["row_index"] = str(i)
|
||||
|
||||
tls = _make_tls(["ru", "en", "fr", "zh"])
|
||||
|
||||
llm_rows, pre_rows = svc._classify(rows, None, tls)
|
||||
|
||||
assert len(pre_rows) == 2, (
|
||||
f"Expected 2 pre_rows (rows 0 and 3), got {len(pre_rows)}"
|
||||
)
|
||||
assert len(llm_rows) == 2, (
|
||||
f"Expected 2 llm_rows (rows 1 and 2), got {len(llm_rows)}"
|
||||
)
|
||||
|
||||
# Verify specific rows
|
||||
pre_indices = [r.get("row_index") for r in pre_rows]
|
||||
assert "0" in pre_indices, "Row 0 should be in pre_rows (full cache)"
|
||||
assert "3" in pre_indices, "Row 3 should be in pre_rows (full cache)"
|
||||
|
||||
llm_indices = [r.get("row_index") for r in llm_rows]
|
||||
assert "1" in llm_indices, "Row 1 should be in llm_rows (partial cache)"
|
||||
assert "2" in llm_indices, "Row 2 should be in llm_rows (no cache)"
|
||||
# #endregion test_multi_row_mixed_source_lang_cache
|
||||
|
||||
# #region test_all_same_lang_short_circuit_with_source_in_tls [C:2] [TYPE Function]
|
||||
# @BRIEF Single target language matching detected source → short-circuit to pre.
|
||||
def test_all_same_lang_short_circuit_with_source_in_tls(self):
|
||||
svc = _make_service()
|
||||
rows = [_make_row(detected_lang="ru", cached_lang_values=None)]
|
||||
tls = _make_tls(["ru"]) # only ru, which matches detected
|
||||
|
||||
llm_rows, pre_rows = svc._classify(rows, None, tls)
|
||||
|
||||
assert len(llm_rows) == 0
|
||||
assert len(pre_rows) == 1
|
||||
assert rows[0].get("_same_language") is True
|
||||
# #endregion test_all_same_lang_short_circuit_with_source_in_tls
|
||||
|
||||
# #region test_case_insensitive_detected_lang_matching [C:2] [TYPE Function]
|
||||
# @BRIEF Detected language "RU" vs tls "ru" — case-insensitive exclusion.
|
||||
def test_case_insensitive_detected_lang_matching(self):
|
||||
svc = _make_service()
|
||||
rows = [_make_row(
|
||||
detected_lang="RU", # uppercase
|
||||
cached_lang_values={"en": "Hello", "fr": "Bonjour"},
|
||||
)]
|
||||
tls = _make_tls(["ru", "en", "fr"])
|
||||
|
||||
llm_rows, pre_rows = svc._classify(rows, None, tls)
|
||||
|
||||
# non_source_tls should exclude "ru" (case-insensitive match with "RU")
|
||||
# remaining ["en","fr"] present in cache → pre_rows
|
||||
assert len(llm_rows) == 0
|
||||
assert len(pre_rows) == 1
|
||||
# #endregion test_case_insensitive_detected_lang_matching
|
||||
|
||||
|
||||
# ── _persist_pre() Tests ───────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -436,4 +624,185 @@ class TestCacheHitLogging:
|
||||
# #endregion test_cache_hit_zero_no_log
|
||||
|
||||
|
||||
# ── E2E Batch Pipeline Integration Tests ───────────────────────────
|
||||
# Full pipeline: _check_cache → _classify → _persist_pre with
|
||||
# production-like data shapes. Verifies warm cache skips LLM entirely.
|
||||
|
||||
|
||||
# #region TestBatchPipelineE2E [C:4] [TYPE Class] [SEMANTICS test,integration,pipeline,cache,e2e]
|
||||
# @BRIEF End-to-end batch processing pipeline: cache lookup → classify
|
||||
# → persist. Verifies that warm-cache batches require ZERO LLM calls.
|
||||
class TestBatchPipelineE2E:
|
||||
"""Integration tests for the full batch processing pipeline."""
|
||||
|
||||
# #region test_warm_cache_full_pipeline_no_llm [C:4] [TYPE Function] [SEMANTICS test,integration,cache,warm,e2e]
|
||||
# @BRIEF Production scenario: source lang in tls, warm cache covers all
|
||||
# non-source targets → entire batch should go to pre_rows (0 LLM).
|
||||
# @RATIONALE This is the exact scenario from production logs where
|
||||
# cache_hits=14-17 but pre=0. The fix should make pre=17, llm=0.
|
||||
def test_warm_cache_full_pipeline_no_llm(self):
|
||||
"""Full pipeline: all rows cached, source lang in tls → 0 LLM rows."""
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
svc = _make_service()
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.context_columns = None
|
||||
|
||||
# 17 rows — production batch size
|
||||
rows = []
|
||||
for i in range(17):
|
||||
row = _make_row(
|
||||
detected_lang="ru",
|
||||
source_text=f"Source text {i}",
|
||||
cached_lang_values={"en": f"English {i}", "fr": f"Francais {i}", "zh": f"Chinese {i}"},
|
||||
source_data={"id": str(i)},
|
||||
)
|
||||
row.pop("_source_hash", None) # let _check_cache compute it
|
||||
row["row_index"] = str(i)
|
||||
rows.append(row)
|
||||
|
||||
tls = ["ru", "en", "fr", "zh"] # ru=source, others=targets
|
||||
|
||||
# Step 1: _check_cache — all should be found
|
||||
with patch("src.plugins.translate._batch_proc._check_translation_cache",
|
||||
return_value={"en": "yes", "fr": "oui", "zh": "是"}):
|
||||
svc._check_cache(job, rows, "dict_hash", "cfg_hash")
|
||||
|
||||
# Verify all rows have _cached_lang_values set
|
||||
for row in rows:
|
||||
assert row.get("_cached_lang_values") is not None, (
|
||||
f"Row {row['row_index']} missing cached values"
|
||||
)
|
||||
|
||||
# Step 2: _classify — all should go to pre_rows
|
||||
llm_rows, pre_rows = svc._classify(rows, None, tls)
|
||||
|
||||
assert len(llm_rows) == 0, (
|
||||
f"Expected 0 llm_rows (all cached), got {len(llm_rows)}"
|
||||
)
|
||||
assert len(pre_rows) == 17, (
|
||||
f"Expected 17 pre_rows, got {len(pre_rows)}"
|
||||
)
|
||||
|
||||
# Step 3: _persist_pre — should create all records
|
||||
bid = str(uuid.uuid4())
|
||||
rid = str(uuid.uuid4())
|
||||
count = svc._persist_pre(pre_rows, bid, rid, tls)
|
||||
|
||||
assert count == 17
|
||||
# 1 TranslationRecord + 3 TranslationLanguage per row = 4 DB adds per row
|
||||
# 17 * 4 = 68 DB operations
|
||||
expected_db_calls = 17 * (1 + len([t for t in tls if str(t).lower() != "ru"]))
|
||||
# Actually _persist_pre creates TranslationLanguage for each target: ru→source_text, en/fr/zh→cached
|
||||
assert svc.db.add.call_count >= 17, (
|
||||
f"Expected at least 17 TranslationRecords, got {svc.db.add.call_count} calls"
|
||||
)
|
||||
# #endregion test_warm_cache_full_pipeline_no_llm
|
||||
|
||||
# #region test_cold_cache_pipeline_all_llm [C:3] [TYPE Function] [SEMANTICS test,integration,cold-cache,llm,e2e]
|
||||
# @BRIEF Cold cache (no hits) → ALL rows go to llm_rows.
|
||||
def test_cold_cache_pipeline_all_llm(self):
|
||||
"""Full pipeline: no cache hits → all rows to LLM."""
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
svc = _make_service()
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.context_columns = None
|
||||
|
||||
rows = []
|
||||
for i in range(5):
|
||||
row = _make_row(
|
||||
detected_lang="ru",
|
||||
source_text=f"Untranslated text {i}",
|
||||
cached_lang_values=None,
|
||||
source_data={"id": str(i)},
|
||||
)
|
||||
row.pop("_source_hash", None)
|
||||
row["row_index"] = str(i)
|
||||
rows.append(row)
|
||||
|
||||
tls = ["ru", "en", "fr", "zh"]
|
||||
|
||||
# _check_cache — nothing found
|
||||
with patch("src.plugins.translate._batch_proc._check_translation_cache",
|
||||
return_value=None):
|
||||
svc._check_cache(job, rows, "dict_hash", "cfg_hash")
|
||||
|
||||
for row in rows:
|
||||
assert row.get("_cached_lang_values") is None
|
||||
|
||||
# _classify — all to LLM
|
||||
llm_rows, pre_rows = svc._classify(rows, None, tls)
|
||||
|
||||
assert len(llm_rows) == 5, f"Expected 5 llm_rows, got {len(llm_rows)}"
|
||||
# ru-ru same-language rows might go to pre if all targets match source
|
||||
# With tls=["ru","en","fr","zh"], only ru isn't all targets → partial match
|
||||
# But _same_language=True is set, and since not all targets match ru,
|
||||
# the usual flow continues to cache check (which has None) → LLM
|
||||
assert len(pre_rows) == 0, f"Expected 0 pre_rows, got {len(pre_rows)}"
|
||||
# #endregion test_cold_cache_pipeline_all_llm
|
||||
|
||||
# #region test_partial_cache_pipeline_split [C:3] [TYPE Function] [SEMANTICS test,integration,partial-cache,split,e2e]
|
||||
# @BRIEF Mixed: some rows cached, some not → correct split.
|
||||
def test_partial_cache_pipeline_split(self):
|
||||
"""Half cached, half not → split into llm and pre."""
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
svc = _make_service()
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.context_columns = None
|
||||
|
||||
rows = []
|
||||
for i in range(10):
|
||||
cached = i < 5 # first 5 cached
|
||||
row = _make_row(
|
||||
detected_lang="ru",
|
||||
source_text=f"Text {i}",
|
||||
cached_lang_values={"en": f"EN{i}", "fr": f"FR{i}", "zh": f"ZH{i}"} if cached else None,
|
||||
source_data={"id": str(i)},
|
||||
)
|
||||
row.pop("_source_hash", None)
|
||||
row["row_index"] = str(i)
|
||||
rows.append(row)
|
||||
|
||||
tls = ["ru", "en", "fr", "zh"]
|
||||
|
||||
# _check_cache — find first 5
|
||||
cache_call_count = [0]
|
||||
|
||||
def mock_check_cache(db, h):
|
||||
cache_call_count[0] += 1
|
||||
if cache_call_count[0] <= 5:
|
||||
return {"en": "cached", "fr": "cached", "zh": "cached"}
|
||||
return None
|
||||
|
||||
with patch("src.plugins.translate._batch_proc._check_translation_cache",
|
||||
side_effect=mock_check_cache):
|
||||
svc._check_cache(job, rows, "dict_hash", "cfg_hash")
|
||||
|
||||
# First 5 have cache, last 5 don't
|
||||
for i, row in enumerate(rows):
|
||||
if i < 5:
|
||||
assert row.get("_cached_lang_values") is not None, f"Row {i} missing cache"
|
||||
else:
|
||||
assert row.get("_cached_lang_values") is None, f"Row {i} should have no cache"
|
||||
|
||||
# _classify → correct split
|
||||
llm_rows, pre_rows = svc._classify(rows, None, tls)
|
||||
|
||||
assert len(pre_rows) == 5, f"Expected 5 pre_rows, got {len(pre_rows)}"
|
||||
assert len(llm_rows) == 5, f"Expected 5 llm_rows, got {len(llm_rows)}"
|
||||
|
||||
# Verify rows 0-4 in pre, 5-9 in llm
|
||||
pre_indices = {r["row_index"] for r in pre_rows}
|
||||
llm_indices = {r["row_index"] for r in llm_rows}
|
||||
for i in range(5):
|
||||
assert str(i) in pre_indices, f"Row {i} should be in pre_rows"
|
||||
for i in range(5, 10):
|
||||
assert str(i) in llm_indices, f"Row {i} should be in llm_rows"
|
||||
# #endregion test_partial_cache_pipeline_split
|
||||
|
||||
# #endregion TestBatchPipelineE2E
|
||||
|
||||
|
||||
# #endregion BatchClassifyPersistTests
|
||||
|
||||
@@ -379,4 +379,87 @@ class TestOutputPerRow:
|
||||
# endregion test_output_per_row_lang_200_multi
|
||||
|
||||
# endregion TestOutputPerRow
|
||||
|
||||
|
||||
# #region TestOutputSafetyFactor [C:3] [TYPE Class] [SEMANTICS test,token,budget,safety-factor,regression]
|
||||
# @BRIEF Regression tests for OUTPUT_SAFETY_FACTOR=0.55 — verifies it prevents
|
||||
# finish_reason=length by being conservative (≤0.70) while not falling
|
||||
# below the realistic minimum for qwen-flash 4-language batch sizing.
|
||||
# @RATIONALE The factor was lowered from 0.70→0.55 to be MORE conservative,
|
||||
# reducing max_rows_by_output from an overoptimistic 36 to a realistic 17.
|
||||
# The actual batch size was always ~17 due to INPUT budget constraints;
|
||||
# this fix just made the output estimate match reality, preventing
|
||||
# finish_reason=length truncation cascades.
|
||||
class TestOutputSafetyFactor:
|
||||
|
||||
# #region test_output_safety_factor_not_above_070 [C:3] [TYPE Function]
|
||||
# @BRIEF OUTPUT_SAFETY_FACTOR must be ≤ 0.70 — the conservative guard.
|
||||
# @RATIONALE Higher values (e.g. 0.75) would overestimate output capacity,
|
||||
# causing finish_reason=length and LLM retry cascades.
|
||||
def test_output_safety_factor_not_above_070(self):
|
||||
from src.plugins.translate._token_budget import OUTPUT_SAFETY_FACTOR
|
||||
assert OUTPUT_SAFETY_FACTOR <= 0.70, (
|
||||
f"OUTPUT_SAFETY_FACTOR={OUTPUT_SAFETY_FACTOR}, must be ≤ 0.70"
|
||||
)
|
||||
assert OUTPUT_SAFETY_FACTOR >= 0.40, (
|
||||
f"OUTPUT_SAFETY_FACTOR={OUTPUT_SAFETY_FACTOR}, too conservative"
|
||||
)
|
||||
# #endregion test_output_safety_factor_not_above_070
|
||||
|
||||
# #region test_max_rows_by_output_qwen_flash_4langs [C:3] [TYPE Function]
|
||||
# @BRIEF qwen-flash + 4 target languages: max_rows_by_output ≥ 14 and ≤ 24.
|
||||
# @RATIONALE Production: input-budget limits batches to ~17 rows anyway.
|
||||
# Output budget estimate must be realistic (14-24 range).
|
||||
def test_max_rows_by_output_qwen_flash_4langs(self):
|
||||
from src.plugins.translate._token_budget import _compute_max_rows_by_output
|
||||
|
||||
max_rows = _compute_max_rows_by_output(
|
||||
max_output_tokens=32768, # qwen-flash
|
||||
num_languages=4,
|
||||
)
|
||||
|
||||
assert 14 <= max_rows <= 24, (
|
||||
f"max_rows_by_output={max_rows}, expected 14-24 range."
|
||||
)
|
||||
# #endregion test_max_rows_by_output_qwen_flash_4langs
|
||||
|
||||
# #region test_output_safety_factor_consistent_with_per_row [C:2] [TYPE Function]
|
||||
# @BRIEF max_rows_by_output is consistent with OUTPUT_PER_ROW_PER_LANG.
|
||||
def test_output_safety_factor_consistent_with_per_row(self):
|
||||
from src.plugins.translate._token_budget import (
|
||||
_compute_max_rows_by_output,
|
||||
OUTPUT_SAFETY_FACTOR,
|
||||
OUTPUT_PER_ROW_PER_LANG,
|
||||
REASONING_OVERHEAD,
|
||||
MAX_OUTPUT_HEADROOM,
|
||||
JSON_OVERHEAD_PER_ROW,
|
||||
)
|
||||
|
||||
# Manual computation must match
|
||||
max_tok = 32768
|
||||
n_lang = 4
|
||||
overhead = REASONING_OVERHEAD + MAX_OUTPUT_HEADROOM
|
||||
per_row = n_lang * OUTPUT_PER_ROW_PER_LANG + JSON_OVERHEAD_PER_ROW
|
||||
available = int((max_tok - overhead) * OUTPUT_SAFETY_FACTOR)
|
||||
expected = max(available // per_row, 1)
|
||||
|
||||
actual = _compute_max_rows_by_output(max_tok, n_lang)
|
||||
assert actual == expected, (
|
||||
f"actual={actual} != expected={expected}"
|
||||
)
|
||||
# #endregion test_output_safety_factor_consistent_with_per_row
|
||||
|
||||
# #region test_single_lang_output_rows_above_20 [C:2] [TYPE Function]
|
||||
# @BRIEF Single target language with 0.55: max rows ≥ 20.
|
||||
def test_single_lang_output_rows_above_20(self):
|
||||
from src.plugins.translate._token_budget import _compute_max_rows_by_output
|
||||
|
||||
max_rows = _compute_max_rows_by_output(
|
||||
max_output_tokens=16384,
|
||||
num_languages=1,
|
||||
)
|
||||
assert max_rows >= 20, f"Expected ≥20 rows for 1 lang, got {max_rows}"
|
||||
# #endregion test_single_lang_output_rows_above_20
|
||||
|
||||
# #endregion TestOutputSafetyFactor
|
||||
# #endregion TestTokenBudget
|
||||
|
||||
@@ -94,13 +94,22 @@ class BatchProcessingService:
|
||||
return {**result, "batch_id": bid}
|
||||
# #endregion process_batch
|
||||
|
||||
# #region _create_batch [C:2] [TYPE Function] [SEMANTICS translate,batch,create]
|
||||
# @ingroup Translate
|
||||
# @BRIEF Create a TranslationBatch DB record and flush to get its ID.
|
||||
def _create_batch(self, run_id, batch_index, batch_rows):
|
||||
b = TranslationBatch(id=str(uuid.uuid4()), run_id=run_id, batch_index=batch_index,
|
||||
status="RUNNING", total_records=len(batch_rows), started_at=datetime.now(UTC))
|
||||
self.db.add(b)
|
||||
self.db.flush()
|
||||
return b
|
||||
# #endregion _create_batch
|
||||
|
||||
# #region _check_cache [C:3] [TYPE Function] [SEMANTICS translate,cache,hash,lookup]
|
||||
# @ingroup Translate
|
||||
# @BRIEF Check translation cache for each batch row, attach _cached_lang_values.
|
||||
# @POST Rows with cache hits have _cached_lang_values dict set.
|
||||
# @SIDE_EFFECT Modifies batch_rows in-place. Emits aggregated log.
|
||||
def _check_cache(self, job, batch_rows, dict_snapshot_hash, config_hash):
|
||||
cache_hits = 0
|
||||
for row in batch_rows:
|
||||
@@ -121,8 +130,11 @@ class BatchProcessingService:
|
||||
"Translation cache hits",
|
||||
{"batch_rows": len(batch_rows), "cache_hits": cache_hits},
|
||||
)
|
||||
# #endregion _check_cache
|
||||
|
||||
# ★ Local language detection — replaces LLM-based detection
|
||||
# #region _detect_languages [C:2] [TYPE Function] [SEMANTICS translate,language,detect,lingua]
|
||||
# @ingroup Translate
|
||||
# @BRIEF Run local language detection on all batch rows (no LLM).
|
||||
def _detect_languages(self, batch_rows: list[dict], target_languages: list[str]) -> None:
|
||||
"""Run local language detection on all batch rows (no LLM).
|
||||
|
||||
@@ -133,7 +145,17 @@ class BatchProcessingService:
|
||||
results = batch_detect(texts, target_languages)
|
||||
for row, lang in zip(batch_rows, results):
|
||||
row["_detected_lang"] = lang
|
||||
# #endregion _detect_languages
|
||||
|
||||
# #region _classify [C:4] [TYPE Function] [SEMANTICS translate,classify,cache,same-lang]
|
||||
# @ingroup Translate
|
||||
# @BRIEF Classify batch rows into pre_rows (no LLM needed) and llm_rows.
|
||||
# @POST Returns (llm_rows, pre_rows). Cached rows go to pre_rows only when
|
||||
# all non-source target languages are present in cache.
|
||||
# @RATIONALE Non-source tls filtering: tls may include the source language
|
||||
# (e.g. ["ru","en","fr","zh"]) but cache only has translations
|
||||
# (en/fr/zh). We exclude detected source language from the
|
||||
# cache-completeness check via non_source_tls.
|
||||
def _classify(self, batch_rows, preview_edits_cache, tls):
|
||||
llm_rows, pre_rows = [], []
|
||||
tls_lower = [str(t).lower() for t in tls]
|
||||
@@ -156,9 +178,15 @@ class BatchProcessingService:
|
||||
pre_rows.append(row)
|
||||
continue
|
||||
cl = row.get("_cached_lang_values")
|
||||
if cl and all(lc in cl for lc in tls):
|
||||
pre_rows.append(row)
|
||||
continue
|
||||
if cl:
|
||||
# Exclude source language from cache-completeness check:
|
||||
# tls may include the source language (e.g. ["ru","en","fr","zh"])
|
||||
# but cache only contains translations (en/fr/zh), not the source.
|
||||
dl = row.get("_detected_lang") or ""
|
||||
non_source_tls = [lc for lc in tls if str(lc).lower() != str(dl).lower()]
|
||||
if non_source_tls and all(lc in cl for lc in non_source_tls):
|
||||
pre_rows.append(row)
|
||||
continue
|
||||
if preview_edits_cache:
|
||||
sd = row.get("source_data") or {}
|
||||
if sd:
|
||||
@@ -175,7 +203,11 @@ class BatchProcessingService:
|
||||
{"tls": tls, "pre": len(pre_rows), "llm": len(llm_rows),
|
||||
"total": len(pre_rows) + len(llm_rows)})
|
||||
return llm_rows, pre_rows
|
||||
# #endregion _classify
|
||||
|
||||
# #region _persist_pre [C:3] [TYPE Function] [SEMANTICS translate,persist,record,language]
|
||||
# @ingroup Translate
|
||||
# @BRIEF Persist pre-classified rows as TranslationRecord + TranslationLanguage.
|
||||
def _persist_pre(self, pre_rows, bid, run_id, tls):
|
||||
count = 0
|
||||
for row in pre_rows:
|
||||
@@ -213,7 +245,11 @@ class BatchProcessingService:
|
||||
))
|
||||
count += 1
|
||||
return count
|
||||
# #endregion _persist_pre
|
||||
|
||||
# #region _process_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call,batch]
|
||||
# @ingroup Translate
|
||||
# @BRIEF Process LLM translation for a batch: estimate budget, call LLM, return results.
|
||||
async def _process_llm(self, job, run_id, rows_for_llm, dict_matches, bid, tls):
|
||||
# Resolve provider token config (DB values take priority over PROVIDER_DEFAULTS)
|
||||
token_config = {"model": None, "context_window": None, "max_output_tokens": None}
|
||||
@@ -260,9 +296,13 @@ class BatchProcessingService:
|
||||
},
|
||||
)
|
||||
return result
|
||||
# #endregion _process_llm
|
||||
|
||||
# -- Batch insert (delegation) --
|
||||
# #region insert_batch_to_target [C:4] [TYPE Function] [SEMANTICS translate,batch,insert,target]
|
||||
# @ingroup Translate
|
||||
# @BRIEF Insert batch records into the target table via SQL Lab or direct DB.
|
||||
async def insert_batch_to_target(self, job: TranslationJob, batch_id: str, run_id: str) -> None:
|
||||
await insert_batch_to_target(self.db, self.config_manager, job, batch_id, run_id)
|
||||
# #endregion insert_batch_to_target
|
||||
# #endregion BatchProcessingService
|
||||
# #endregion BatchProcessingService
|
||||
|
||||
Reference in New Issue
Block a user