diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 0d967c46..b6a3e9c2 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -17,6 +17,7 @@ include = ["src*"] [tool.pytest.ini_options] pythonpath = ["."] asyncio_mode = "auto" +testpaths = ["tests", "src/plugins"] markers = [ "integration: Integration tests requiring external services (Docker, Testcontainers, Superset). Use --run-integration to enable.", ] diff --git a/backend/src/plugins/translate/__tests__/test_batch_classify_persist.py b/backend/src/plugins/translate/__tests__/test_batch_classify_persist.py index 470ac882..192ce6f2 100644 --- a/backend/src/plugins/translate/__tests__/test_batch_classify_persist.py +++ b/backend/src/plugins/translate/__tests__/test_batch_classify_persist.py @@ -123,11 +123,11 @@ class TestClassify: assert len(pre_rows) == 1 def test_cache_partial_targets_llm_rows(self): - """Cache has only 'ru', but targets are [ru, en] → goes to LLM.""" + """Cache has only 'en', targets [en, fr, zh], detected 'ru' → missing fr/zh → LLM.""" svc = _make_service() - rows = [_make_row(detected_lang="en", - cached_lang_values={"ru": "текст"})] - tls = _make_tls(["ru", "en"]) + rows = [_make_row(detected_lang="ru", + cached_lang_values={"en": "translated"})] + tls = _make_tls(["en", "fr", "zh"]) llm_rows, pre_rows = svc._classify(rows, None, tls) @@ -206,6 +206,194 @@ class TestClassify: assert len(llm_rows) == 1 +# ── Cache Source-Language Exclusion Tests ─────────────────────────── +# Verifies the fix: tls may include the source language (e.g. +# ["ru","en","fr","zh"]) but cache only contains translations +# (en/fr/zh). _classify now filters detected source language from +# cache-completeness check. + + +# #region TestClassifyCacheSourceLang [C:3] [TYPE Class] [SEMANTICS test,classify,cache,source-lang,exclusion] +# @BRIEF Verifies _classify correctly excludes detected source language +# from cache-completeness check, preventing false-negative +# routing to LLM when tls includes the source language. +class TestClassifyCacheSourceLang: + """Tests for the non-source-tls filter in _classify().""" + + # #region test_source_lang_in_tls_cache_all_translations [C:3] [TYPE Function] + # @BRIEF Source lang in tls, cache has ALL translations → pre_rows. + # @RATIONALE This is the exact production bug scenario: detected_lang="ru", + # tls=["ru","en","fr","zh"], cache={"en":...,"fr":...,"zh":...}. + # Old code: all(lc in cl for lc in tls) → "ru" not in cl → False → LLM. + # New code: non_source_tls=["en","fr","zh"] → all present → pre_rows. + def test_source_lang_in_tls_cache_all_translations(self): + svc = _make_service() + rows = [_make_row( + detected_lang="ru", + cached_lang_values={"en": "Hello", "fr": "Bonjour", "zh": "你好"}, + )] + tls = _make_tls(["ru", "en", "fr", "zh"]) + + llm_rows, pre_rows = svc._classify(rows, None, tls) + + assert len(llm_rows) == 0, ( + f"Expected 0 llm_rows (all cached), got {len(llm_rows)}" + ) + assert len(pre_rows) == 1 + assert rows[0].get("_same_language") is True # ru in tls_lower + # #endregion test_source_lang_in_tls_cache_all_translations + + # #region test_source_lang_in_tls_cache_missing_one_translation [C:3] [TYPE Function] + # @BRIEF Source lang in tls, cache MISSING one non-source lang → LLM. + def test_source_lang_in_tls_cache_missing_one_translation(self): + svc = _make_service() + rows = [_make_row( + detected_lang="ru", + cached_lang_values={"en": "Hello", "fr": "Bonjour"}, # zh missing + )] + tls = _make_tls(["ru", "en", "fr", "zh"]) + + llm_rows, pre_rows = svc._classify(rows, None, tls) + + assert len(llm_rows) == 1, ( + f"Expected 1 llm_row (zh not cached), got {len(llm_rows)}" + ) + assert len(pre_rows) == 0 + # #endregion test_source_lang_in_tls_cache_missing_one_translation + + # #region test_detected_und_tls_includes_und [C:2] [TYPE Function] + # @BRIEF Detected lang "und": excluded from cache check like any detected source. + # Cache has ALL non-und langs → row goes to pre_rows. + def test_detected_und_tls_includes_und(self): + svc = _make_service() + rows = [_make_row( + detected_lang="und", + cached_lang_values={"en": "Hello", "fr": "Bonjour"}, + )] + tls = _make_tls(["und", "en", "fr"]) + + llm_rows, pre_rows = svc._classify(rows, None, tls) + + # non_source_tls = ["en","fr"] (und excluded) + # both present in cache → pre_rows (correct: und is just "unknown") + assert len(llm_rows) == 0, ( + f"Expected 0 llm_rows (und excluded, en/fr in cache), got {len(llm_rows)}" + ) + assert len(pre_rows) == 1 + # #endregion test_detected_und_tls_includes_und + + # #region test_empty_detected_lang_no_exclusion [C:2] [TYPE Function] + # @BRIEF Empty detected language → non_source_tls equals full tls. + def test_empty_detected_lang_no_exclusion(self): + svc = _make_service() + rows = [_make_row( + detected_lang="", + cached_lang_values={"en": "Hello", "ru": "Привет"}, + )] + tls = _make_tls(["ru", "en"]) + + llm_rows, pre_rows = svc._classify(rows, None, tls) + + # non_source_tls = ["ru","en"] (no filtering) + # all present in cache → pre_rows + assert len(llm_rows) == 0 + assert len(pre_rows) == 1 + # #endregion test_empty_detected_lang_no_exclusion + + # #region test_source_lang_not_in_tls [C:2] [TYPE Function] + # @BRIEF Detected source language is NOT in tls — no filtering occurs but + # cache completeness still checked against full tls. + def test_source_lang_not_in_tls(self): + svc = _make_service() + rows = [_make_row( + detected_lang="ru", # ru not in target languages + cached_lang_values={"en": "Hello", "fr": "Bonjour", "zh": "你好"}, + )] + tls = _make_tls(["en", "fr", "zh"]) + + llm_rows, pre_rows = svc._classify(rows, None, tls) + + # non_source_tls = ["en","fr","zh"] (ru not in tls, so all remain) + # all present in cache → pre_rows + assert len(llm_rows) == 0 + assert len(pre_rows) == 1 + # #endregion test_source_lang_not_in_tls + + # #region test_multi_row_mixed_source_lang_cache [C:3] [TYPE Function] + # @BRIEF Multiple rows with different detected languages and cache states. + def test_multi_row_mixed_source_lang_cache(self): + svc = _make_service() + rows = [ + # Row 0: ru source, full cache → pre + _make_row(detected_lang="ru", + cached_lang_values={"en": "Hello", "fr": "Bonjour", "zh": "你好"}), + # Row 1: fr source, partial cache → LLM (zh missing) + _make_row(detected_lang="fr", + cached_lang_values={"en": "Hello"}), + # Row 2: no cache, ru source → LLM + _make_row(detected_lang="ru", cached_lang_values=None), + # Row 3: en source (not in tls), full cache → pre + _make_row(detected_lang="en", + cached_lang_values={"ru": "привет", "fr": "Bonjour", "zh": "你好"}), + ] + # Give each row a unique index for verification + for i, row in enumerate(rows): + row["row_index"] = str(i) + + tls = _make_tls(["ru", "en", "fr", "zh"]) + + llm_rows, pre_rows = svc._classify(rows, None, tls) + + assert len(pre_rows) == 2, ( + f"Expected 2 pre_rows (rows 0 and 3), got {len(pre_rows)}" + ) + assert len(llm_rows) == 2, ( + f"Expected 2 llm_rows (rows 1 and 2), got {len(llm_rows)}" + ) + + # Verify specific rows + pre_indices = [r.get("row_index") for r in pre_rows] + assert "0" in pre_indices, "Row 0 should be in pre_rows (full cache)" + assert "3" in pre_indices, "Row 3 should be in pre_rows (full cache)" + + llm_indices = [r.get("row_index") for r in llm_rows] + assert "1" in llm_indices, "Row 1 should be in llm_rows (partial cache)" + assert "2" in llm_indices, "Row 2 should be in llm_rows (no cache)" + # #endregion test_multi_row_mixed_source_lang_cache + + # #region test_all_same_lang_short_circuit_with_source_in_tls [C:2] [TYPE Function] + # @BRIEF Single target language matching detected source → short-circuit to pre. + def test_all_same_lang_short_circuit_with_source_in_tls(self): + svc = _make_service() + rows = [_make_row(detected_lang="ru", cached_lang_values=None)] + tls = _make_tls(["ru"]) # only ru, which matches detected + + llm_rows, pre_rows = svc._classify(rows, None, tls) + + assert len(llm_rows) == 0 + assert len(pre_rows) == 1 + assert rows[0].get("_same_language") is True + # #endregion test_all_same_lang_short_circuit_with_source_in_tls + + # #region test_case_insensitive_detected_lang_matching [C:2] [TYPE Function] + # @BRIEF Detected language "RU" vs tls "ru" — case-insensitive exclusion. + def test_case_insensitive_detected_lang_matching(self): + svc = _make_service() + rows = [_make_row( + detected_lang="RU", # uppercase + cached_lang_values={"en": "Hello", "fr": "Bonjour"}, + )] + tls = _make_tls(["ru", "en", "fr"]) + + llm_rows, pre_rows = svc._classify(rows, None, tls) + + # non_source_tls should exclude "ru" (case-insensitive match with "RU") + # remaining ["en","fr"] present in cache → pre_rows + assert len(llm_rows) == 0 + assert len(pre_rows) == 1 + # #endregion test_case_insensitive_detected_lang_matching + + # ── _persist_pre() Tests ─────────────────────────────────────────── @@ -436,4 +624,185 @@ class TestCacheHitLogging: # #endregion test_cache_hit_zero_no_log +# ── E2E Batch Pipeline Integration Tests ─────────────────────────── +# Full pipeline: _check_cache → _classify → _persist_pre with +# production-like data shapes. Verifies warm cache skips LLM entirely. + + +# #region TestBatchPipelineE2E [C:4] [TYPE Class] [SEMANTICS test,integration,pipeline,cache,e2e] +# @BRIEF End-to-end batch processing pipeline: cache lookup → classify +# → persist. Verifies that warm-cache batches require ZERO LLM calls. +class TestBatchPipelineE2E: + """Integration tests for the full batch processing pipeline.""" + + # #region test_warm_cache_full_pipeline_no_llm [C:4] [TYPE Function] [SEMANTICS test,integration,cache,warm,e2e] + # @BRIEF Production scenario: source lang in tls, warm cache covers all + # non-source targets → entire batch should go to pre_rows (0 LLM). + # @RATIONALE This is the exact scenario from production logs where + # cache_hits=14-17 but pre=0. The fix should make pre=17, llm=0. + def test_warm_cache_full_pipeline_no_llm(self): + """Full pipeline: all rows cached, source lang in tls → 0 LLM rows.""" + from unittest.mock import patch, MagicMock + + svc = _make_service() + job = MagicMock(spec=TranslationJob) + job.context_columns = None + + # 17 rows — production batch size + rows = [] + for i in range(17): + row = _make_row( + detected_lang="ru", + source_text=f"Source text {i}", + cached_lang_values={"en": f"English {i}", "fr": f"Francais {i}", "zh": f"Chinese {i}"}, + source_data={"id": str(i)}, + ) + row.pop("_source_hash", None) # let _check_cache compute it + row["row_index"] = str(i) + rows.append(row) + + tls = ["ru", "en", "fr", "zh"] # ru=source, others=targets + + # Step 1: _check_cache — all should be found + with patch("src.plugins.translate._batch_proc._check_translation_cache", + return_value={"en": "yes", "fr": "oui", "zh": "是"}): + svc._check_cache(job, rows, "dict_hash", "cfg_hash") + + # Verify all rows have _cached_lang_values set + for row in rows: + assert row.get("_cached_lang_values") is not None, ( + f"Row {row['row_index']} missing cached values" + ) + + # Step 2: _classify — all should go to pre_rows + llm_rows, pre_rows = svc._classify(rows, None, tls) + + assert len(llm_rows) == 0, ( + f"Expected 0 llm_rows (all cached), got {len(llm_rows)}" + ) + assert len(pre_rows) == 17, ( + f"Expected 17 pre_rows, got {len(pre_rows)}" + ) + + # Step 3: _persist_pre — should create all records + bid = str(uuid.uuid4()) + rid = str(uuid.uuid4()) + count = svc._persist_pre(pre_rows, bid, rid, tls) + + assert count == 17 + # 1 TranslationRecord + 3 TranslationLanguage per row = 4 DB adds per row + # 17 * 4 = 68 DB operations + expected_db_calls = 17 * (1 + len([t for t in tls if str(t).lower() != "ru"])) + # Actually _persist_pre creates TranslationLanguage for each target: ru→source_text, en/fr/zh→cached + assert svc.db.add.call_count >= 17, ( + f"Expected at least 17 TranslationRecords, got {svc.db.add.call_count} calls" + ) + # #endregion test_warm_cache_full_pipeline_no_llm + + # #region test_cold_cache_pipeline_all_llm [C:3] [TYPE Function] [SEMANTICS test,integration,cold-cache,llm,e2e] + # @BRIEF Cold cache (no hits) → ALL rows go to llm_rows. + def test_cold_cache_pipeline_all_llm(self): + """Full pipeline: no cache hits → all rows to LLM.""" + from unittest.mock import patch, MagicMock + + svc = _make_service() + job = MagicMock(spec=TranslationJob) + job.context_columns = None + + rows = [] + for i in range(5): + row = _make_row( + detected_lang="ru", + source_text=f"Untranslated text {i}", + cached_lang_values=None, + source_data={"id": str(i)}, + ) + row.pop("_source_hash", None) + row["row_index"] = str(i) + rows.append(row) + + tls = ["ru", "en", "fr", "zh"] + + # _check_cache — nothing found + with patch("src.plugins.translate._batch_proc._check_translation_cache", + return_value=None): + svc._check_cache(job, rows, "dict_hash", "cfg_hash") + + for row in rows: + assert row.get("_cached_lang_values") is None + + # _classify — all to LLM + llm_rows, pre_rows = svc._classify(rows, None, tls) + + assert len(llm_rows) == 5, f"Expected 5 llm_rows, got {len(llm_rows)}" + # ru-ru same-language rows might go to pre if all targets match source + # With tls=["ru","en","fr","zh"], only ru isn't all targets → partial match + # But _same_language=True is set, and since not all targets match ru, + # the usual flow continues to cache check (which has None) → LLM + assert len(pre_rows) == 0, f"Expected 0 pre_rows, got {len(pre_rows)}" + # #endregion test_cold_cache_pipeline_all_llm + + # #region test_partial_cache_pipeline_split [C:3] [TYPE Function] [SEMANTICS test,integration,partial-cache,split,e2e] + # @BRIEF Mixed: some rows cached, some not → correct split. + def test_partial_cache_pipeline_split(self): + """Half cached, half not → split into llm and pre.""" + from unittest.mock import patch, MagicMock + + svc = _make_service() + job = MagicMock(spec=TranslationJob) + job.context_columns = None + + rows = [] + for i in range(10): + cached = i < 5 # first 5 cached + row = _make_row( + detected_lang="ru", + source_text=f"Text {i}", + cached_lang_values={"en": f"EN{i}", "fr": f"FR{i}", "zh": f"ZH{i}"} if cached else None, + source_data={"id": str(i)}, + ) + row.pop("_source_hash", None) + row["row_index"] = str(i) + rows.append(row) + + tls = ["ru", "en", "fr", "zh"] + + # _check_cache — find first 5 + cache_call_count = [0] + + def mock_check_cache(db, h): + cache_call_count[0] += 1 + if cache_call_count[0] <= 5: + return {"en": "cached", "fr": "cached", "zh": "cached"} + return None + + with patch("src.plugins.translate._batch_proc._check_translation_cache", + side_effect=mock_check_cache): + svc._check_cache(job, rows, "dict_hash", "cfg_hash") + + # First 5 have cache, last 5 don't + for i, row in enumerate(rows): + if i < 5: + assert row.get("_cached_lang_values") is not None, f"Row {i} missing cache" + else: + assert row.get("_cached_lang_values") is None, f"Row {i} should have no cache" + + # _classify → correct split + llm_rows, pre_rows = svc._classify(rows, None, tls) + + assert len(pre_rows) == 5, f"Expected 5 pre_rows, got {len(pre_rows)}" + assert len(llm_rows) == 5, f"Expected 5 llm_rows, got {len(llm_rows)}" + + # Verify rows 0-4 in pre, 5-9 in llm + pre_indices = {r["row_index"] for r in pre_rows} + llm_indices = {r["row_index"] for r in llm_rows} + for i in range(5): + assert str(i) in pre_indices, f"Row {i} should be in pre_rows" + for i in range(5, 10): + assert str(i) in llm_indices, f"Row {i} should be in llm_rows" + # #endregion test_partial_cache_pipeline_split + +# #endregion TestBatchPipelineE2E + + # #endregion BatchClassifyPersistTests diff --git a/backend/src/plugins/translate/__tests__/test_token_budget.py b/backend/src/plugins/translate/__tests__/test_token_budget.py index 8d10ad26..60063db2 100644 --- a/backend/src/plugins/translate/__tests__/test_token_budget.py +++ b/backend/src/plugins/translate/__tests__/test_token_budget.py @@ -379,4 +379,87 @@ class TestOutputPerRow: # endregion test_output_per_row_lang_200_multi # endregion TestOutputPerRow + + +# #region TestOutputSafetyFactor [C:3] [TYPE Class] [SEMANTICS test,token,budget,safety-factor,regression] +# @BRIEF Regression tests for OUTPUT_SAFETY_FACTOR=0.55 — verifies it prevents +# finish_reason=length by being conservative (≤0.70) while not falling +# below the realistic minimum for qwen-flash 4-language batch sizing. +# @RATIONALE The factor was lowered from 0.70→0.55 to be MORE conservative, +# reducing max_rows_by_output from an overoptimistic 36 to a realistic 17. +# The actual batch size was always ~17 due to INPUT budget constraints; +# this fix just made the output estimate match reality, preventing +# finish_reason=length truncation cascades. +class TestOutputSafetyFactor: + + # #region test_output_safety_factor_not_above_070 [C:3] [TYPE Function] + # @BRIEF OUTPUT_SAFETY_FACTOR must be ≤ 0.70 — the conservative guard. + # @RATIONALE Higher values (e.g. 0.75) would overestimate output capacity, + # causing finish_reason=length and LLM retry cascades. + def test_output_safety_factor_not_above_070(self): + from src.plugins.translate._token_budget import OUTPUT_SAFETY_FACTOR + assert OUTPUT_SAFETY_FACTOR <= 0.70, ( + f"OUTPUT_SAFETY_FACTOR={OUTPUT_SAFETY_FACTOR}, must be ≤ 0.70" + ) + assert OUTPUT_SAFETY_FACTOR >= 0.40, ( + f"OUTPUT_SAFETY_FACTOR={OUTPUT_SAFETY_FACTOR}, too conservative" + ) + # #endregion test_output_safety_factor_not_above_070 + + # #region test_max_rows_by_output_qwen_flash_4langs [C:3] [TYPE Function] + # @BRIEF qwen-flash + 4 target languages: max_rows_by_output ≥ 14 and ≤ 24. + # @RATIONALE Production: input-budget limits batches to ~17 rows anyway. + # Output budget estimate must be realistic (14-24 range). + def test_max_rows_by_output_qwen_flash_4langs(self): + from src.plugins.translate._token_budget import _compute_max_rows_by_output + + max_rows = _compute_max_rows_by_output( + max_output_tokens=32768, # qwen-flash + num_languages=4, + ) + + assert 14 <= max_rows <= 24, ( + f"max_rows_by_output={max_rows}, expected 14-24 range." + ) + # #endregion test_max_rows_by_output_qwen_flash_4langs + + # #region test_output_safety_factor_consistent_with_per_row [C:2] [TYPE Function] + # @BRIEF max_rows_by_output is consistent with OUTPUT_PER_ROW_PER_LANG. + def test_output_safety_factor_consistent_with_per_row(self): + from src.plugins.translate._token_budget import ( + _compute_max_rows_by_output, + OUTPUT_SAFETY_FACTOR, + OUTPUT_PER_ROW_PER_LANG, + REASONING_OVERHEAD, + MAX_OUTPUT_HEADROOM, + JSON_OVERHEAD_PER_ROW, + ) + + # Manual computation must match + max_tok = 32768 + n_lang = 4 + overhead = REASONING_OVERHEAD + MAX_OUTPUT_HEADROOM + per_row = n_lang * OUTPUT_PER_ROW_PER_LANG + JSON_OVERHEAD_PER_ROW + available = int((max_tok - overhead) * OUTPUT_SAFETY_FACTOR) + expected = max(available // per_row, 1) + + actual = _compute_max_rows_by_output(max_tok, n_lang) + assert actual == expected, ( + f"actual={actual} != expected={expected}" + ) + # #endregion test_output_safety_factor_consistent_with_per_row + + # #region test_single_lang_output_rows_above_20 [C:2] [TYPE Function] + # @BRIEF Single target language with 0.55: max rows ≥ 20. + def test_single_lang_output_rows_above_20(self): + from src.plugins.translate._token_budget import _compute_max_rows_by_output + + max_rows = _compute_max_rows_by_output( + max_output_tokens=16384, + num_languages=1, + ) + assert max_rows >= 20, f"Expected ≥20 rows for 1 lang, got {max_rows}" + # #endregion test_single_lang_output_rows_above_20 + +# #endregion TestOutputSafetyFactor # #endregion TestTokenBudget diff --git a/backend/src/plugins/translate/_batch_proc.py b/backend/src/plugins/translate/_batch_proc.py index 93b81933..e53efe66 100644 --- a/backend/src/plugins/translate/_batch_proc.py +++ b/backend/src/plugins/translate/_batch_proc.py @@ -94,13 +94,22 @@ class BatchProcessingService: return {**result, "batch_id": bid} # #endregion process_batch + # #region _create_batch [C:2] [TYPE Function] [SEMANTICS translate,batch,create] + # @ingroup Translate + # @BRIEF Create a TranslationBatch DB record and flush to get its ID. def _create_batch(self, run_id, batch_index, batch_rows): b = TranslationBatch(id=str(uuid.uuid4()), run_id=run_id, batch_index=batch_index, status="RUNNING", total_records=len(batch_rows), started_at=datetime.now(UTC)) self.db.add(b) self.db.flush() return b + # #endregion _create_batch + # #region _check_cache [C:3] [TYPE Function] [SEMANTICS translate,cache,hash,lookup] + # @ingroup Translate + # @BRIEF Check translation cache for each batch row, attach _cached_lang_values. + # @POST Rows with cache hits have _cached_lang_values dict set. + # @SIDE_EFFECT Modifies batch_rows in-place. Emits aggregated log. def _check_cache(self, job, batch_rows, dict_snapshot_hash, config_hash): cache_hits = 0 for row in batch_rows: @@ -121,8 +130,11 @@ class BatchProcessingService: "Translation cache hits", {"batch_rows": len(batch_rows), "cache_hits": cache_hits}, ) + # #endregion _check_cache - # ★ Local language detection — replaces LLM-based detection + # #region _detect_languages [C:2] [TYPE Function] [SEMANTICS translate,language,detect,lingua] + # @ingroup Translate + # @BRIEF Run local language detection on all batch rows (no LLM). def _detect_languages(self, batch_rows: list[dict], target_languages: list[str]) -> None: """Run local language detection on all batch rows (no LLM). @@ -133,7 +145,17 @@ class BatchProcessingService: results = batch_detect(texts, target_languages) for row, lang in zip(batch_rows, results): row["_detected_lang"] = lang + # #endregion _detect_languages + # #region _classify [C:4] [TYPE Function] [SEMANTICS translate,classify,cache,same-lang] + # @ingroup Translate + # @BRIEF Classify batch rows into pre_rows (no LLM needed) and llm_rows. + # @POST Returns (llm_rows, pre_rows). Cached rows go to pre_rows only when + # all non-source target languages are present in cache. + # @RATIONALE Non-source tls filtering: tls may include the source language + # (e.g. ["ru","en","fr","zh"]) but cache only has translations + # (en/fr/zh). We exclude detected source language from the + # cache-completeness check via non_source_tls. def _classify(self, batch_rows, preview_edits_cache, tls): llm_rows, pre_rows = [], [] tls_lower = [str(t).lower() for t in tls] @@ -156,9 +178,15 @@ class BatchProcessingService: pre_rows.append(row) continue cl = row.get("_cached_lang_values") - if cl and all(lc in cl for lc in tls): - pre_rows.append(row) - continue + if cl: + # Exclude source language from cache-completeness check: + # tls may include the source language (e.g. ["ru","en","fr","zh"]) + # but cache only contains translations (en/fr/zh), not the source. + dl = row.get("_detected_lang") or "" + non_source_tls = [lc for lc in tls if str(lc).lower() != str(dl).lower()] + if non_source_tls and all(lc in cl for lc in non_source_tls): + pre_rows.append(row) + continue if preview_edits_cache: sd = row.get("source_data") or {} if sd: @@ -175,7 +203,11 @@ class BatchProcessingService: {"tls": tls, "pre": len(pre_rows), "llm": len(llm_rows), "total": len(pre_rows) + len(llm_rows)}) return llm_rows, pre_rows + # #endregion _classify + # #region _persist_pre [C:3] [TYPE Function] [SEMANTICS translate,persist,record,language] + # @ingroup Translate + # @BRIEF Persist pre-classified rows as TranslationRecord + TranslationLanguage. def _persist_pre(self, pre_rows, bid, run_id, tls): count = 0 for row in pre_rows: @@ -213,7 +245,11 @@ class BatchProcessingService: )) count += 1 return count + # #endregion _persist_pre + # #region _process_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call,batch] + # @ingroup Translate + # @BRIEF Process LLM translation for a batch: estimate budget, call LLM, return results. async def _process_llm(self, job, run_id, rows_for_llm, dict_matches, bid, tls): # Resolve provider token config (DB values take priority over PROVIDER_DEFAULTS) token_config = {"model": None, "context_window": None, "max_output_tokens": None} @@ -260,9 +296,13 @@ class BatchProcessingService: }, ) return result + # #endregion _process_llm - # -- Batch insert (delegation) -- + # #region insert_batch_to_target [C:4] [TYPE Function] [SEMANTICS translate,batch,insert,target] + # @ingroup Translate + # @BRIEF Insert batch records into the target table via SQL Lab or direct DB. async def insert_batch_to_target(self, job: TranslationJob, batch_id: str, run_id: str) -> None: await insert_batch_to_target(self.db, self.config_manager, job, batch_id, run_id) + # #endregion insert_batch_to_target # #endregion BatchProcessingService # #endregion BatchProcessingService diff --git a/frontend/src/lib/components/translate/RunTabContent.svelte b/frontend/src/lib/components/translate/RunTabContent.svelte index 7907d417..ef4e9b67 100644 --- a/frontend/src/lib/components/translate/RunTabContent.svelte +++ b/frontend/src/lib/components/translate/RunTabContent.svelte @@ -376,7 +376,7 @@ onclick={() => showPageBulkReplace = true} class="px-3 py-1.5 text-xs bg-primary text-white rounded hover:bg-primary-hover transition-colors" > - {_t.translate?.run?.bulk_replace || 'Bulk Replace'} + {_t.translate?.bulk_replace?.bulk_replace_all || _t.translate?.run?.bulk_replace || 'Bulk Replace'}
diff --git a/frontend/src/lib/components/translate/TranslationRunResult.svelte b/frontend/src/lib/components/translate/TranslationRunResult.svelte index 24b5b226..2c649fa5 100644 --- a/frontend/src/lib/components/translate/TranslationRunResult.svelte +++ b/frontend/src/lib/components/translate/TranslationRunResult.svelte @@ -476,12 +476,6 @@ import { SvelteSet } from "svelte/reactivity"; : (_t.translate?.run?.load_more_records || 'Load more') + ' (' + (recordsTotal - records.length) + ')'} {/if} -
diff --git a/frontend/src/lib/i18n/locales/en/translate.json b/frontend/src/lib/i18n/locales/en/translate.json index d5d52407..edf8a178 100644 --- a/frontend/src/lib/i18n/locales/en/translate.json +++ b/frontend/src/lib/i18n/locales/en/translate.json @@ -637,6 +637,7 @@ }, "bulk_replace": { "title": "Bulk Find & Replace", + "bulk_replace_all": "Bulk Replace Across All Runs", "close": "Close", "find_pattern": "Find pattern", "find_placeholder": "Enter text or regex pattern...", diff --git a/frontend/src/lib/i18n/locales/ru/translate.json b/frontend/src/lib/i18n/locales/ru/translate.json index 5e4e6f40..84451c21 100644 --- a/frontend/src/lib/i18n/locales/ru/translate.json +++ b/frontend/src/lib/i18n/locales/ru/translate.json @@ -638,6 +638,7 @@ }, "bulk_replace": { "title": "Массовая замена", + "bulk_replace_all": "Массовая замена по всем запускам", "close": "Закрыть", "find_pattern": "Шаблон поиска", "find_placeholder": "Введите текст или регулярное выражение...",