feat(translate): multi-language optimization (Phase 11)

- Auto-detection of source language per row via LLM (US6)
- Multi-target translation — one LLM call for N languages (US1-US3)
- Language-aware storage: TranslationLanguage, per-language stats
- Multilingual dictionaries with language-pair-aware filtering (US7)
- Inline correction on any run result + submit-to-dictionary (US8)
- Context-aware dictionary: auto-capture row context, usage notes,
  Jaccard similarity, priority flagging in LLM prompts (US8b)
- Configurable preview sample size 1-100, cost warning at >30
- Per-language history & metrics with MetricSnapshot preservation
- 36 files, +5022/-373, all specs GRACE-Poly v2.6 compliant
This commit is contained in:
2026-05-14 17:12:41 +03:00
parent 5e741a4332
commit bb0fbfdafd
36 changed files with 5025 additions and 376 deletions

View File

@@ -1,29 +1,37 @@
# region DictionaryTests [TYPE Module]
# @SEMANTICS: tests, dictionary, crud, import, filter
# @PURPOSE: Validate DictionaryManager CRUD, import, deletion guards, and batch filtering.
# @SEMANTICS: tests, dictionary, crud, import, filter, language_pair
# @PURPOSE: Validate DictionaryManager CRUD, import, deletion guards, batch filtering, and language-pair support.
# @RELATION: BINDS_TO -> [DictionaryManager:Class]
#
# @TEST_CONTRACT: [DictionaryManager] -> {
# invariants: [
# "Create/Read/Update/Delete dictionaries works",
# "Entry CRUD enforces unique source_term_normalized per dictionary",
# "Entry CRUD enforces unique (dictionary_id, source_term_normalized, source_language, target_language)",
# "Import CSV/TSV handles overwrite/keep_existing/cancel conflict modes",
# "Delete blocked when dictionary attached to active/scheduled jobs",
# "filter_for_batch returns matched entries with word-boundary awareness"
# "filter_for_batch returns matched entries with word-boundary awareness",
# "filter_for_batch supports language-pair filtering",
# "Language-pair-aware duplicate detection: same term with different lang pair is allowed"
# ]
# }
# @TEST_EDGE: duplicate_entry -> 409-style ValueError on repeated source_term
# @TEST_EDGE: duplicate_entry -> 409-style ValueError on repeated (dictionary_id, source_term_norm, source_lang, target_lang)
# @TEST_EDGE: delete_active_job -> ValueError with active/scheduled message
# @TEST_EDGE: import_invalid_format -> ValueError for missing columns
# @TEST_INVARIANT: unique_normalized -> verifies: [duplicate_entry]
# @TEST_EDGE: same_term_different_lang_pair -> allowed (not duplicate)
# @TEST_INVARIANT: unique_normalized -> verifies: [duplicate_entry, same_term_different_lang_pair allowed]
import csv
import io
import pytest
from sqlalchemy import create_engine, event
from sqlalchemy.orm import Session, sessionmaker
from src.models.translate import (
Base,
DictionaryEntry,
TerminologyDictionary,
TranslationJob,
TranslationJobDictionary,
)
@@ -112,8 +120,8 @@ def test_delete_dictionary(db_session: Session):
db_session, name="To Delete",
source_dialect="a", target_dialect="b",
)
# Add an entry
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
# Add an entry with language pair
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
assert entry.id is not None
DictionaryManager.delete_dictionary(db_session, d.id)
@@ -144,15 +152,15 @@ def test_add_entry_duplicate(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "Hello", "Hola")
DictionaryManager.add_entry(db_session, d.id, "Hello", "Hola", source_language="en", target_language="es")
# Same normalized term should raise
# Same normalized term with same language pair should raise
with pytest.raises(ValueError, match="already exists"):
DictionaryManager.add_entry(db_session, d.id, "hello", "Bonjour")
DictionaryManager.add_entry(db_session, d.id, "hello", "Bonjour", source_language="en", target_language="es")
# Different case, same normalized should also raise
# Different case, same normalized, same lang pair should also raise
with pytest.raises(ValueError, match="already exists"):
DictionaryManager.add_entry(db_session, d.id, "HELLO", "Ciao")
DictionaryManager.add_entry(db_session, d.id, "HELLO", "Ciao", source_language="en", target_language="es")
# endregion test_add_entry_duplicate
@@ -167,9 +175,9 @@ def test_add_entry_duplicate_per_dictionary(db_session: Session):
db_session, name="Dict2",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d1.id, "hello", "hola")
DictionaryManager.add_entry(db_session, d1.id, "hello", "hola", source_language="en", target_language="es")
# Same term in different dictionary should work
entry = DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour")
entry = DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour", source_language="en", target_language="es")
assert entry.id is not None
# endregion test_add_entry_duplicate_per_dictionary
@@ -181,13 +189,13 @@ def test_edit_entry(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
updated = DictionaryManager.edit_entry(db_session, entry.id, target_term="HOLA!")
assert updated.target_term == "HOLA!"
# Edit source term to a value that's already taken should fail
DictionaryManager.add_entry(db_session, d.id, "world", "mundo")
DictionaryManager.add_entry(db_session, d.id, "world", "mundo", source_language="en", target_language="es")
with pytest.raises(ValueError, match="already exists"):
DictionaryManager.edit_entry(db_session, entry.id, source_term="WORLD")
# endregion test_edit_entry
@@ -200,7 +208,7 @@ def test_delete_entry(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
DictionaryManager.delete_entry(db_session, entry.id)
entries, total = DictionaryManager.list_entries(db_session, d.id)
@@ -215,10 +223,10 @@ def test_import_csv_overwrite(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
# Pre-add an entry
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
# Pre-add an entry with language pair
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
csv_content = "source_term,target_term,context_notes\nhello,HELLO,\nworld,mundo,"
csv_content = "source_term,target_term,context_notes,source_language,target_language\nhello,HELLO,,en,es\nworld,mundo,,en,es"
result = DictionaryManager.import_entries(
db_session, d.id, csv_content,
delimiter=",", on_conflict="overwrite",
@@ -242,9 +250,9 @@ def test_import_csv_keep_existing(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo"
csv_content = "source_term,target_term,source_language,target_language\nhello,HELLO,en,es\nworld,mundo,en,es"
result = DictionaryManager.import_entries(
db_session, d.id, csv_content,
delimiter=",", on_conflict="keep_existing",
@@ -267,9 +275,9 @@ def test_import_csv_cancel_on_conflict(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo"
csv_content = "source_term,target_term,source_language,target_language\nhello,HELLO,en,es\nworld,mundo,en,es"
result = DictionaryManager.import_entries(
db_session, d.id, csv_content,
delimiter=",", on_conflict="cancel",
@@ -289,7 +297,7 @@ def test_import_tsv(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
tsv_content = "source_term\ttarget_term\nhello\thola\nworld\tmundo"
tsv_content = "source_term\ttarget_term\tsource_language\ttarget_language\nhello\thola\ten\tes\nworld\tmundo\ten\tes"
result = DictionaryManager.import_entries(
db_session, d.id, tsv_content,
delimiter="\t", on_conflict="overwrite",
@@ -339,9 +347,9 @@ def test_import_preview(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo"
csv_content = "source_term,target_term,source_language,target_language\nhello,HELLO,en,es\nworld,mundo,en,es"
result = DictionaryManager.import_entries(
db_session, d.id, csv_content,
delimiter=",", on_conflict="overwrite",
@@ -446,9 +454,9 @@ def test_filter_for_batch_matches(db_session: Session):
db_session, name="Test Dict",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
DictionaryManager.add_entry(db_session, d.id, "world", "mundo")
DictionaryManager.add_entry(db_session, d.id, "foo", "bar")
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
DictionaryManager.add_entry(db_session, d.id, "world", "mundo", source_language="en", target_language="es")
DictionaryManager.add_entry(db_session, d.id, "foo", "bar", source_language="en", target_language="es")
job = TranslationJob(
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
@@ -492,7 +500,7 @@ def test_filter_for_batch_case_insensitive(db_session: Session):
db_session, name="Test Dict",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "Hello World", "Hola Mundo")
DictionaryManager.add_entry(db_session, d.id, "Hello World", "Hola Mundo", source_language="en", target_language="es")
job = TranslationJob(
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
@@ -523,7 +531,7 @@ def test_filter_for_batch_word_boundary(db_session: Session):
db_session, name="Test Dict",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "cat", "gato")
DictionaryManager.add_entry(db_session, d.id, "cat", "gato", source_language="en", target_language="es")
job = TranslationJob(
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
@@ -562,8 +570,8 @@ def test_filter_for_batch_multi_dictionary_priority(db_session: Session):
d2 = DictionaryManager.create_dictionary(
db_session, name="Priority2", source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d1.id, "hello", "hola")
DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour")
DictionaryManager.add_entry(db_session, d1.id, "hello", "hola", source_language="en", target_language="es")
DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour", source_language="en", target_language="es")
job = TranslationJob(
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
@@ -625,8 +633,8 @@ def test_clear_entries(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test", source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
DictionaryManager.add_entry(db_session, d.id, "world", "mundo")
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
DictionaryManager.add_entry(db_session, d.id, "world", "mundo", source_language="en", target_language="es")
deleted = DictionaryManager.clear_entries(db_session, d.id)
assert deleted == 2
@@ -634,4 +642,558 @@ def test_clear_entries(db_session: Session):
entries, total = DictionaryManager.list_entries(db_session, d.id)
assert total == 0
# endregion test_clear_entries
# region test_add_entry_with_language_pair [TYPE Function]
# @PURPOSE: Verify creating entry with source_language and target_language stores correctly.
def test_add_entry_with_language_pair(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Lang Test",
source_dialect="a", target_dialect="b",
)
entry = DictionaryManager.add_entry(
db_session, d.id, "hello", "привет",
source_language="en", target_language="ru",
)
assert entry.source_language == "en"
assert entry.target_language == "ru"
assert entry.source_term == "hello"
# Read back
entries, total = DictionaryManager.list_entries(db_session, d.id)
assert total == 1
assert entries[0].source_language == "en"
assert entries[0].target_language == "ru"
# endregion test_add_entry_with_language_pair
# region test_duplicate_same_language_pair [TYPE Function]
# @PURPOSE: Verify duplicate entry with same (dictionary_id, source_term_norm, source_lang, target_lang) raises conflict.
def test_duplicate_same_language_pair(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Dup Test",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(
db_session, d.id, "hello", "привет",
source_language="en", target_language="ru",
)
# Same term with same language pair should raise
with pytest.raises(ValueError, match="already exists"):
DictionaryManager.add_entry(
db_session, d.id, "hello", "hallo",
source_language="en", target_language="ru",
)
# endregion test_duplicate_same_language_pair
# region test_same_term_different_language_pair [TYPE Function]
# @PURPOSE: Verify same source_term with different language pair is allowed (not duplicate).
def test_same_term_different_language_pair(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Multi Lang",
source_dialect="a", target_dialect="b",
)
entry1 = DictionaryManager.add_entry(
db_session, d.id, "hello", "привет",
source_language="en", target_language="ru",
)
entry2 = DictionaryManager.add_entry(
db_session, d.id, "hello", "hallo",
source_language="en", target_language="de",
)
assert entry1.id != entry2.id
assert entry1.target_language == "ru"
assert entry2.target_language == "de"
entries, total = DictionaryManager.list_entries(db_session, d.id)
assert total == 2
# endregion test_same_term_different_language_pair
# region test_filter_for_batch_with_language_pair [TYPE Function]
# @PURPOSE: Verify filter_for_batch with source_language and target_language filters entries correctly.
def test_filter_for_batch_with_language_pair(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Lang Filter",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")
DictionaryManager.add_entry(db_session, d.id, "world", "мир", source_language="en", target_language="ru")
DictionaryManager.add_entry(db_session, d.id, "hello", "hallo", source_language="en", target_language="de")
job = TranslationJob(
name="Lang Job", source_dialect="a", target_dialect="b", status="DRAFT",
)
db_session.add(job)
db_session.flush()
link = TranslationJobDictionary(job_id=job.id, dictionary_id=d.id)
db_session.add(link)
db_session.commit()
# Filter for en → ru: should get 2 entries (hello/privet, world/mir)
result = DictionaryManager.filter_for_batch(
db_session, ["hello world"], job.id,
source_language="en", target_language="ru",
)
assert len(result) == 2
assert all(m["target_language"] == "ru" for m in result)
# Filter for en → de: should get only the hello/hallo entry
result2 = DictionaryManager.filter_for_batch(
db_session, ["hello"], job.id,
source_language="en", target_language="de",
)
assert len(result2) == 1
assert result2[0]["target_language"] == "de"
assert result2[0]["target_term"] == "hallo"
# endregion test_filter_for_batch_with_language_pair
# region test_filter_for_batch_target_language_only [TYPE Function]
# @PURPOSE: Verify filter_for_batch with only target_language filters by target language only.
def test_filter_for_batch_target_language_only(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Target Only",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")
DictionaryManager.add_entry(db_session, d.id, "hello", "hallo", source_language="en", target_language="de")
DictionaryManager.add_entry(db_session, d.id, "bonjour", "hallo", source_language="fr", target_language="de")
job = TranslationJob(
name="Tgt Job", source_dialect="a", target_dialect="b", status="DRAFT",
)
db_session.add(job)
db_session.flush()
link = TranslationJobDictionary(job_id=job.id, dictionary_id=d.id)
db_session.add(link)
db_session.commit()
# Filter for target=de only: should match both hello→hallo and bonjour→hallo (from any source lang)
result = DictionaryManager.filter_for_batch(
db_session, ["hello bonjour"], job.id,
target_language="de",
)
assert len(result) == 2
assert all(m["target_language"] == "de" for m in result)
# Filter for target=ru only: should match hello→privet
result2 = DictionaryManager.filter_for_batch(
db_session, ["hello"], job.id,
target_language="ru",
)
assert len(result2) == 1
assert result2[0]["target_language"] == "ru"
# endregion test_filter_for_batch_target_language_only
# region test_migrate_old_entries [TYPE Function]
# @PURPOSE: Verify migration populates language pair fields for old-style entries.
def test_migrate_old_entries(db_session: Session):
# Create a dictionary with deprecated target_language set
d = TerminologyDictionary(
name="Old Dict",
source_dialect="a",
target_dialect="b",
target_language="ru",
)
db_session.add(d)
db_session.flush()
# Create old-style entries without explicit language pair (source_language=und, target_language=und)
entry1 = DictionaryEntry(
dictionary_id=d.id,
source_term="hello",
source_term_normalized="hello",
target_term="привет",
source_language="und",
target_language="und",
)
entry2 = DictionaryEntry(
dictionary_id=d.id,
source_term="world",
source_term_normalized="world",
target_term="мир",
source_language="und",
target_language="und",
origin_source_language="en",
)
db_session.add(entry1)
db_session.add(entry2)
db_session.commit()
# Run migration
result = DictionaryManager.migrate_old_entries(db_session)
# Verify migration counts
assert result["total_processed"] >= 2
assert result["migrated_source"] >= 1 # entry2 had origin_source_language
assert result["migrated_target"] >= 2 # both should get target_language from dictionary
# Verify entry1 got target_language from dictionary
db_session.refresh(entry1)
assert entry1.target_language == "ru"
# source_language should remain "und" (no origin_source_language)
assert entry1.source_language == "und"
# Verify entry2 got source from origin and target from dictionary
db_session.refresh(entry2)
assert entry2.source_language == "en"
assert entry2.target_language == "ru"
# endregion test_migrate_old_entries
# region test_export_entries [TYPE Function]
# @PURPOSE: Verify export_entries includes language columns.
def test_export_entries(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Export Test",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")
DictionaryManager.add_entry(db_session, d.id, "world", "мир", source_language="en", target_language="ru")
csv_output = DictionaryManager.export_entries(db_session, d.id)
assert "source_language" in csv_output
assert "target_language" in csv_output
assert "context_data" in csv_output
assert "usage_notes" in csv_output
assert "en" in csv_output
assert "ru" in csv_output
# Verify it's valid CSV
reader = csv.DictReader(io.StringIO(csv_output))
rows = list(reader)
assert len(rows) == 2
assert rows[0]["source_language"] == "en"
assert rows[0]["target_language"] == "ru"
# endregion test_export_entries
# region test_import_with_default_language [TYPE Function]
# @PURPOSE: Verify import uses default_source_language and default_target_language when columns missing.
def test_import_with_default_language(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Default Lang Import",
source_dialect="a", target_dialect="b",
)
# CSV without language columns
csv_content = "source_term,target_term\nhello,привет\nworld,мир"
result = DictionaryManager.import_entries(
db_session, d.id, csv_content,
delimiter=",", on_conflict="overwrite",
default_source_language="en",
default_target_language="ru",
)
assert result["created"] == 2
entries, _ = DictionaryManager.list_entries(db_session, d.id)
for e in entries:
assert e.source_language == "en"
assert e.target_language == "ru"
# endregion test_import_with_default_language
# region test_context_capture_in_correction [TYPE Function]
# @PURPOSE: Verify context_data is auto-captured when submitting a correction with source row context.
def test_context_capture_in_correction(db_session: Session):
"""Test T132: Context auto-capture when submitting a correction."""
from src.plugins.translate.service import InlineCorrectionService
from src.models.translate import TranslationRecord, TranslationRun, TranslationBatch, TranslationLanguage
d = DictionaryManager.create_dictionary(
db_session, name="Test Dict",
source_dialect="a", target_dialect="b",
)
job = TranslationJob(
name="Ctx Job", source_dialect="a", target_dialect="b",
status="DRAFT",
)
db_session.add(job)
db_session.flush()
run = TranslationRun(
job_id=job.id, status="COMPLETED",
)
db_session.add(run)
db_session.flush()
batch = TranslationBatch(
run_id=run.id, batch_index=0, status="COMPLETED",
)
db_session.add(batch)
db_session.flush()
# Create a TranslationRecord with source_data (context columns)
record = TranslationRecord(
batch_id=batch.id,
run_id=run.id,
source_sql="hello world",
source_object_type="table_row",
source_object_name="Row 0",
source_object_id="0",
source_data={"schema": "public", "table": "users", "column": "name"},
status="SUCCESS",
)
db_session.add(record)
db_session.flush()
# Create language entries for test languages
for lang_code in ("ru", "de", "fr", "it"):
tl = TranslationLanguage(
record_id=record.id,
language_code=lang_code,
source_language_detected="en",
translated_value="translated",
final_value="translated",
status="translated",
)
db_session.add(tl)
db_session.commit()
# Submit correction to dictionary
result = InlineCorrectionService.submit_correction_to_dict(
db=db_session,
record_id=record.id,
language_code="ru",
dictionary_id=d.id,
corrected_value="привет мир (исправлено)",
current_user="test_user",
)
assert result["action"] == "created"
assert result["entry_id"] is not None
# Verify context was auto-captured
entry = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result["entry_id"]).first()
assert entry is not None
assert entry.has_context is True
assert entry.context_source == "auto"
assert entry.context_data is not None
# source_data should contain the row's context columns
assert "source_data" in entry.context_data
assert entry.context_data["source_object_type"] == "table_row"
assert entry.context_data["source_object_name"] == "Row 0"
# Test context_data_override (use different language to avoid conflict)
result2 = InlineCorrectionService.submit_correction_to_dict(
db=db_session,
record_id=record.id,
language_code="de", # different target language avoids conflict
dictionary_id=d.id,
corrected_value="another correction",
current_user="test_user",
context_data_override={"custom_key": "custom_value"},
)
assert result2["action"] == "created"
entry2 = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result2["entry_id"]).first()
assert entry2.context_data == {"custom_key": "custom_value"}
assert entry2.context_source == "manual"
# Test keep_context=False
result3 = InlineCorrectionService.submit_correction_to_dict(
db=db_session,
record_id=record.id,
language_code="fr", # different target language avoids conflict
dictionary_id=d.id,
corrected_value="no context",
current_user="test_user",
keep_context=False,
)
assert result3["action"] == "created"
entry3 = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result3["entry_id"]).first()
assert entry3.context_data is None
assert entry3.has_context is False
assert entry3.context_source == "manual"
# Test usage_notes
result4 = InlineCorrectionService.submit_correction_to_dict(
db=db_session,
record_id=record.id,
language_code="it", # different target language avoids conflict
dictionary_id=d.id,
corrected_value="with notes",
current_user="test_user",
usage_notes="Use this for user-facing columns only",
)
assert result4["action"] == "created"
entry4 = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result4["entry_id"]).first()
assert entry4.usage_notes == "Use this for user-facing columns only"
# endregion test_context_capture_in_correction
# region test_jaccard_similarity [TYPE Function]
# @PURPOSE: Verify Jaccard similarity computation in ContextAwarePromptBuilder.
def test_jaccard_similarity():
"""Test T132: Jaccard similarity = 1.0 for identical, 0.0 for disjoint."""
from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder
builder = ContextAwarePromptBuilder()
# Identical contexts
ctx1 = {"schema": "public", "table": "users"}
ctx2 = {"schema": "public", "table": "users"}
sim = builder.compute_context_similarity(ctx1, ctx2)
assert sim == 1.0, f"Expected 1.0, got {sim}"
# Disjoint contexts
ctx3 = {"schema": "private"}
sim = builder.compute_context_similarity(ctx1, ctx3)
assert sim == 0.0, f"Expected 0.0, got {sim}"
# Partial overlap
ctx4 = {"schema": "public", "table": "orders"}
sim = builder.compute_context_similarity(ctx1, ctx4)
# intersection: {"public"} union: {"public", "users", "orders"} => 1/3 ≈ 0.33
assert 0.33 <= sim <= 0.34, f"Expected ~0.33, got {sim}"
# Empty/missing contexts
assert builder.compute_context_similarity(None, ctx1) == 0.0
assert builder.compute_context_similarity(ctx1, None) == 0.0
assert builder.compute_context_similarity({}, ctx1) == 0.0
# Case-insensitive matching (same keys, different case)
ctx5 = {"schema": "PUBLIC", "table": "USERS"}
sim = builder.compute_context_similarity(ctx1, ctx5)
assert sim == 1.0, f"Expected 1.0 for case-insensitive, got {sim}"
# endregion test_jaccard_similarity
# region test_context_truncation [TYPE Function]
# @PURPOSE: Verify context string truncation at ~2000 chars.
def test_context_truncation():
"""Test T132: Context string truncation in render_entry."""
from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder
builder = ContextAwarePromptBuilder()
# Build context data that would produce a very long context string
long_val = "x" * 500
long_context = {f"key{i}": long_val for i in range(10)}
# Create an entry-like dict with this context
entry = {
"source_term": "hello",
"target_term": "world",
"has_context": True,
"context_data": long_context,
"usage_notes": None,
}
rendered = builder.render_entry(entry, priority=False)
# The context part should be truncated
assert len(rendered) < 2200, f"Rendered length {len(rendered)} exceeds expected max"
assert "...[truncated]" in rendered, "Expected truncation marker in output"
# Short context should not be truncated
short_entry = {
"source_term": "hello",
"target_term": "world",
"has_context": True,
"context_data": {"table": "users"},
"usage_notes": None,
}
short_rendered = builder.render_entry(short_entry, priority=False)
assert "...[truncated]" not in short_rendered
assert "(context: table=users)" in short_rendered
# Priority prefix
priority_rendered = builder.render_entry(short_entry, priority=True)
assert priority_rendered.startswith("# PRIORITY (context match)")
# Usage notes
notes_entry = {
"source_term": "hello",
"target_term": "world",
"has_context": False,
"context_data": None,
"usage_notes": "Use for admin panels",
}
notes_rendered = builder.render_entry(notes_entry, priority=False)
assert "# Usage: Use for admin panels" in notes_rendered
# endregion test_context_truncation
# region test_render_entry_dict_and_object [TYPE Function]
# @PURPOSE: Verify render_entry handles both dicts and objects.
def test_render_entry_dict_and_object():
"""Test T132: render_entry accepts dict or object with same results."""
from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder
builder = ContextAwarePromptBuilder()
# Dict entry
dict_entry = {
"source_term": "hello",
"target_term": "hola",
"has_context": False,
"context_data": None,
"usage_notes": None,
}
dict_result = builder.render_entry(dict_entry)
# Object-like entry using a simple class
class FakeEntry:
source_term = "hello"
target_term = "hola"
has_context = False
context_data = None
usage_notes = None
obj_result = builder.render_entry(FakeEntry())
assert dict_result == obj_result
assert dict_result == '"hello" -> "hola"'
# endregion test_render_entry_dict_and_object
# region test_build_context_entries_prioritization [TYPE Function]
# @PURPOSE: Verify build_context_entries sorts priority entries first.
def test_build_context_entries_prioritization():
"""Test T132: build_context_entries sorts priority entries before non-priority."""
from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder
builder = ContextAwarePromptBuilder()
entries = [
{
"source_term": "high_match",
"target_term": "alta_coincidencia",
"has_context": True,
"context_data": {"schema": "public", "table": "users"},
"usage_notes": None,
},
{
"source_term": "no_match",
"target_term": "sin_coincidencia",
"has_context": True,
"context_data": {"schema": "private", "table": "config"},
"usage_notes": None,
},
{
"source_term": "no_context",
"target_term": "sin_contexto",
"has_context": False,
"context_data": None,
"usage_notes": None,
},
]
# Row context that matches the first entry
row_context = {"schema": "public", "table": "users"}
rendered = builder.build_context_entries(entries, row_context)
# The first entry should be priority (similarity=1.0 >= 0.5)
assert len(rendered) == 3
assert rendered[0].startswith("# PRIORITY (context match)"), f"Expected priority first, got: {rendered[0]}"
assert '"high_match"' in rendered[0]
# The non-priority entries should not have priority prefix
assert not rendered[1].startswith("# PRIORITY")
assert not rendered[2].startswith("# PRIORITY")
# endregion test_build_context_entries_prioritization
# endregion DictionaryTests

View File

@@ -11,6 +11,7 @@
# @TEST_EDGE: invalid_run_status -> raises ValueError
# @TEST_EDGE: executor_failure -> run is marked FAILED
import json
from datetime import UTC, datetime
from unittest.mock import MagicMock, patch
@@ -18,11 +19,14 @@ import pytest
from src.models.translate import (
TranslationJob,
TranslationLanguage,
TranslationPreviewSession,
TranslationRecord,
TranslationRun,
TranslationRunLanguageStats,
)
from src.plugins.translate.events import TranslationEventLog
from src.plugins.translate.executor import TranslationExecutor
from src.plugins.translate.orchestrator import TranslationOrchestrator
@@ -598,4 +602,249 @@ class TestTranslationOrchestrator:
# endregion test_get_run_records
# endregion test_get_run_history
# endregion TestTranslationOrchestrator
# region TestTranslationExecutorMultiLang [TYPE Class]
# @PURPOSE: Tests for multi-language LLM translation execution: parsing, per-language entries, source-as-reference, backward compat.
class TestTranslationExecutorMultiLang:
# region test_parse_multi_language_response [TYPE Function]
# @PURPOSE: _parse_llm_response handles multi-language JSON format correctly.
def test_parse_multi_language_response(self) -> None:
response = json.dumps({
"rows": [
{"row_id": 0, "detected_source_language": "fr", "ru": "текст", "en": "text", "de": "Text"},
{"row_id": 1, "detected_source_language": "en", "ru": "дашборд", "en": "dashboard", "de": "Dashboard"},
]
})
result = TranslationExecutor._parse_llm_response(
response, expected_count=2, target_languages=["ru", "en", "de"]
)
assert len(result) == 2
assert "0" in result
assert result["0"]["detected_source_language"] == "fr"
assert result["0"]["ru"] == "текст"
assert result["0"]["en"] == "text"
assert result["0"]["de"] == "Text"
assert "1" in result
assert result["1"]["en"] == "dashboard"
assert result["1"]["ru"] == "дашборд"
# endregion test_parse_multi_language_response
# region test_parse_legacy_format_backward_compat [TYPE Function]
# @PURPOSE: _parse_llm_response still handles legacy single "translation" key format.
def test_parse_legacy_format_backward_compat(self) -> None:
response = json.dumps({
"rows": [
{"row_id": 0, "translation": "текст", "detected_source_language": "fr"},
{"row_id": 1, "translation": "дашборд", "detected_source_language": "en"},
]
})
result = TranslationExecutor._parse_llm_response(
response, expected_count=2, target_languages=["ru"]
)
assert len(result) == 2
assert result["0"]["translation"] == "текст"
assert result["1"]["detected_source_language"] == "en"
# endregion test_parse_legacy_format_backward_compat
# region test_multi_language_translation_language_entries [TYPE Function]
# @PURPOSE: Multi-language LLM response creates per-language TranslationLanguage entries.
def test_multi_language_translation_language_entries(self) -> None:
db = MagicMock()
config_manager = MagicMock()
# Mock job with multi-language target_languages
job = MagicMock(spec=TranslationJob)
job.id = "job-ml-1"
job.target_languages = ["ru", "en"]
job.target_language = None
job.target_dialect = "postgresql"
job.source_dialect = "postgresql"
job.translation_column = "name"
job.provider_id = "provider-1"
job.batch_size = 50
executor = TranslationExecutor(db, config_manager, "test-user")
# Patch _call_llm to return multi-language response
multi_lang_response = json.dumps({
"rows": [
{"row_id": "0", "detected_source_language": "fr", "ru": "текст", "en": "text"},
{"row_id": "1", "detected_source_language": "en", "ru": "дашборд", "en": "dashboard"},
]
})
batch_rows = [
{"row_index": "0", "source_text": "texte", "source_object_name": "Row 0"},
{"row_index": "1", "source_text": "dashboard", "source_object_name": "Row 1"},
]
with patch.object(executor, '_call_llm', return_value=multi_lang_response):
result = executor._call_llm_for_batch(
job=job,
run_id="run-ml-1",
batch_rows=batch_rows,
dict_matches=[],
batch_id="batch-ml-1",
)
assert result["successful"] == 2
assert result["failed"] == 0
assert result["skipped"] == 0
# Check that TranslationLanguage entries were created for each row per language
# Each row should have 2 language entries (ru, en) = 4 total
lang_added_calls = [
call for call in db.add.call_args_list
if isinstance(call[0][0], TranslationLanguage)
]
# Should have 4 language entries (2 rows x 2 languages)
assert len(lang_added_calls) >= 4, f"Expected at least 4 TranslationLanguage entries, got {len(lang_added_calls)}"
# Group by language code
lang_codes: list[str] = []
for call in lang_added_calls:
lang_entry = call[0][0]
lang_codes.append(lang_entry.language_code)
assert "ru" in lang_codes
assert "en" in lang_codes
# Each language should appear twice (once per row)
assert lang_codes.count("ru") == 2
assert lang_codes.count("en") == 2
# endregion test_multi_language_translation_language_entries
# region test_source_as_reference [TYPE Function]
# @PURPOSE: When detected source language matches a target language, store original text verbatim.
def test_source_as_reference(self) -> None:
db = MagicMock()
config_manager = MagicMock()
job = MagicMock(spec=TranslationJob)
job.id = "job-sar-1"
job.target_languages = ["fr", "en"]
job.target_language = None
job.target_dialect = "postgresql"
job.source_dialect = "postgresql"
job.translation_column = "name"
job.provider_id = "provider-1"
job.batch_size = 50
executor = TranslationExecutor(db, config_manager, "test-user")
# Source is French, target_languages include French
response = json.dumps({
"rows": [
{"row_id": "0", "detected_source_language": "fr", "fr": "texte français", "en": "french text"},
]
})
batch_rows = [
{"row_index": "0", "source_text": "texte français", "source_object_name": "Row 0"},
]
with patch.object(executor, '_call_llm', return_value=response):
result = executor._call_llm_for_batch(
job=job,
run_id="run-sar-1",
batch_rows=batch_rows,
dict_matches=[],
batch_id="batch-sar-1",
)
assert result["successful"] == 1
# Find the TranslationLanguage for French — should use source text (original)
lang_calls = [
call for call in db.add.call_args_list
if isinstance(call[0][0], TranslationLanguage)
]
fr_entry = None
en_entry = None
for call in lang_calls:
le = call[0][0]
if le.language_code == "fr":
fr_entry = le
elif le.language_code == "en":
en_entry = le
assert fr_entry is not None, "FR language entry should exist"
# Source-as-reference: fr translation should be the original source text
assert fr_entry.translated_value == "texte français", (
f"Expected source text 'texte français' for fr, got '{fr_entry.translated_value}'"
)
assert en_entry is not None, "EN language entry should exist"
# EN should get the LLM-translated value
assert en_entry.translated_value == "french text"
# endregion test_source_as_reference
# region test_per_language_stats_on_execute_run [TYPE Function]
# @PURPOSE: execute_run with multi-language job creates and populates TranslationRunLanguageStats.
def test_per_language_stats_on_execute_run(self, mock_job: MagicMock) -> None:
db = MagicMock()
config_manager = MagicMock()
# Configure mock_job for multi-language
mock_job.target_languages = ["ru", "en"]
mock_job.target_language = None
mock_job.target_table = "target_tbl"
# Mock run
run = MagicMock()
run.id = "run-stats-1"
run.job_id = "job-123"
run.status = "PENDING"
# Mock completed run returned by executor
completed_run = MagicMock()
completed_run.id = "run-stats-1"
completed_run.job_id = "job-123"
completed_run.status = "COMPLETED"
completed_run.total_records = 5
completed_run.successful_records = 5
completed_run.failed_records = 0
completed_run.skipped_records = 0
completed_run.completed_at = datetime.now(UTC)
completed_run.error_message = None
completed_run.insert_status = "success"
completed_run.superset_execution_id = "q-1"
completed_run.superset_execution_log = {}
# Mock queries: job -> mock_job, run -> completed_run
db.query.return_value.filter.return_value.first.side_effect = [
mock_job,
completed_run,
]
orch = TranslationOrchestrator(db, config_manager, "test-user")
with patch.object(orch, 'event_log'), \
patch.object(orch, '_generate_and_insert_sql',
return_value={"status": "success", "query_id": "q-1", "rows_affected": 5}), \
patch.object(orch, '_update_language_stats') as mock_update_stats, \
patch('src.plugins.translate.orchestrator.TranslationExecutor') as MockExecutor:
mock_executor_instance = MagicMock()
mock_executor_instance.execute_run.return_value = completed_run
MockExecutor.return_value = mock_executor_instance
result = orch.execute_run(run)
assert result.status == "COMPLETED"
# Verify that TranslationRunLanguageStats entries were created
stats_added = [
call for call in db.add.call_args_list
if isinstance(call[0][0], TranslationRunLanguageStats)
]
# 2 language stats entries (ru, en)
assert len(stats_added) >= 2, (
f"Expected at least 2 TranslationRunLanguageStats entries, got {len(stats_added)}"
)
codes = [call[0][0].language_code for call in stats_added]
assert "ru" in codes
assert "en" in codes
# Verify _update_language_stats was called
mock_update_stats.assert_called_once()
# endregion test_per_language_stats_on_execute_run
# endregion TestTranslationExecutorMultiLang
# endregion OrchestratorTests

View File

@@ -420,8 +420,13 @@ class TestTranslationPreview:
assert tokens > 0
assert tokens < len(prompt) # tokens typically fewer than chars
output_tokens = TokenEstimator.estimate_output_tokens(10)
assert output_tokens == 500
# Single language
output_tokens = TokenEstimator.estimate_output_tokens(10, num_languages=1)
assert output_tokens == 600 # 10 * 1 * 50 * 1.2
# Multi-language
output_tokens_multi = TokenEstimator.estimate_output_tokens(10, num_languages=3)
assert output_tokens_multi == 1800 # 10 * 3 * 50 * 1.2
cost = TokenEstimator.estimate_cost(1000, 0.002)
assert cost == 0.002
@@ -431,28 +436,65 @@ class TestTranslationPreview:
# endregion test_cost_estimation
# region test_preview_parse_llm_response [TYPE Function]
# @PURPOSE: Verify LLM JSON response parsing.
# @PURPOSE: Verify LLM JSON response parsing includes detected_source_language.
def test_preview_parse_llm_response(self):
"""Parse LLM response should extract translations keyed by row_id."""
"""Parse LLM response should extract translations with detected_source_language per row."""
response = json.dumps({
"rows": [
{"row_id": "0", "translation": "Привет"},
{"row_id": "0", "translation": "Привет", "detected_source_language": "en"},
{"row_id": "1", "translation": "Мир"},
]
})
result = TranslationPreview._parse_llm_response(response, 2)
assert result == {"0": "Привет", "1": "Мир"}
assert result == {
"0": {"translation": "Привет", "detected_source_language": "en"},
"1": {"translation": "Мир", "detected_source_language": "und"},
}
# endregion test_preview_parse_llm_response
# region test_preview_parse_llm_response_multilang [TYPE Function]
# @PURPOSE: Verify multi-language LLM response parsing.
def test_preview_parse_llm_response_multilang(self):
"""Parse multi-language LLM response should extract per-language translations."""
response = json.dumps({
"rows": [
{"row_id": "0", "detected_source_language": "fr", "ru": "текст", "en": "text", "de": "Text"},
{"row_id": "1", "detected_source_language": "en", "ru": "привет", "en": "hello"},
]
})
result = TranslationPreview._parse_llm_response(response, 2, target_languages=["ru", "en", "de"])
assert result == {
"0": {"detected_source_language": "fr", "ru": "текст", "en": "text", "de": "Text"},
"1": {"detected_source_language": "en", "ru": "привет", "en": "hello"},
}
# Verify per-language extraction
assert result["0"]["ru"] == "текст"
assert result["0"]["en"] == "text"
assert result["1"]["ru"] == "привет"
# endregion test_preview_parse_llm_response_multilang
# region test_preview_parse_llm_response_with_code_block [TYPE Function]
# @PURPOSE: Verify LLM response parsing handles markdown code blocks.
def test_preview_parse_llm_response_with_code_block(self):
"""Parse LLM response should handle markdown code block wrapping."""
response = "```json\n{\n \"rows\": [\n {\"row_id\": \"0\", \"translation\": \"Test\"}\n ]\n}\n```"
result = TranslationPreview._parse_llm_response(response, 1)
assert result == {"0": "Test"}
assert result == {"0": {"translation": "Test", "detected_source_language": "und"}}
# endregion test_preview_parse_llm_response_with_code_block
# region test_cost_warning [TYPE Function]
# @PURPOSE: Verify cost warning for large sample sizes.
def test_cost_warning(self):
"""Cost warning should appear for sample_size > 30."""
warning = TokenEstimator.check_cost_warning(31, 2, 5000, 0.01)
assert warning is not None
assert "Large preview" in warning
assert "5000" in warning
no_warning = TokenEstimator.check_cost_warning(10, 2, 1000, 0.002)
assert no_warning is None
# endregion test_cost_warning
# region test_preview_compute_config_hash [TYPE Function]
# @PURPOSE: Verify config hash computation is deterministic.
def test_preview_compute_config_hash(self):

View File

@@ -14,7 +14,7 @@ import io
import re
from typing import Any
from sqlalchemy import func
from sqlalchemy import func, or_
from sqlalchemy.orm import Session
from ...core.logger import belief_scope, logger
@@ -27,6 +27,23 @@ from ...models.translate import (
from ._utils import _detect_delimiter, _normalize_term
# #region _validate_bcp47 [C:2] [TYPE Function] [SEMANTICS validation, bcp47, language]
# @BRIEF Validate that a language tag is a non-empty BCP-47 string.
def _validate_bcp47(tag: str, field_name: str) -> None:
"""Validate that tag is a non-empty BCP-47 string (basic check)."""
if not tag or not tag.strip():
raise ValueError(f"{field_name} must be a non-empty BCP-47 language tag")
tag = tag.strip()
# Basic BCP-47 validation: must match lang[-script][-region][-variant]* pattern
import re as _re
if not _re.match(r'^[a-zA-Z]{2,8}(-[a-zA-Z0-9]{1,8})*$', tag):
raise ValueError(
f"{field_name} is not a valid BCP-47 tag: '{tag}'. "
"Expected format like 'en', 'ru', 'zh-CN', 'pt-BR'."
)
# #endregion _validate_bcp47
# #region DictionaryManager [C:4] [TYPE Class]
# @BRIEF Manages terminology dictionaries and their entries with referential integrity.
# @PRE: Database session is open and valid.
@@ -34,12 +51,14 @@ from ._utils import _detect_delimiter, _normalize_term
# @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
class DictionaryManager:
# region DictionaryManager.create_dictionary [TYPE Function]
# @PURPOSE: Create a new terminology dictionary.
# @PRE: payload contains name, source_dialect, target_dialect.
# @PURPOSE: Create a new terminology dictionary (language independent at dictionary level; language pairs are per-entry).
# @PRE: name is provided.
# @POST: New TerminologyDictionary row is created and returned.
@staticmethod
def create_dictionary(
db: Session, name: str, source_dialect: str, target_dialect: str,
db: Session, name: str,
source_dialect: str = "",
target_dialect: str = "",
created_by: str | None = None, description: str | None = None,
is_active: bool = True,
) -> TerminologyDictionary:
@@ -48,8 +67,8 @@ class DictionaryManager:
dictionary = TerminologyDictionary(
name=name,
description=description,
source_dialect=source_dialect,
target_dialect=target_dialect,
source_dialect=source_dialect or "",
target_dialect=target_dialect or "",
is_active=is_active,
created_by=created_by,
)
@@ -166,53 +185,66 @@ class DictionaryManager:
# endregion DictionaryManager.list_dictionaries
# region DictionaryManager.add_entry [TYPE Function]
# @PURPOSE: Add an entry to a dictionary, enforcing unique source_term_normalized.
# @PRE: dict_id exists. source_term and target_term are non-empty.
# @PURPOSE: Add an entry to a dictionary with language-pair-aware uniqueness validation.
# @PRE: dict_id exists. source_term, target_term are non-empty. source_language, target_language are valid BCP-47.
# @POST: New DictionaryEntry row is created or raises on duplicate.
@staticmethod
def add_entry(
db: Session, dict_id: str, source_term: str, target_term: str,
context_notes: str | None = None,
source_language: str = "und",
target_language: str = "und",
) -> DictionaryEntry:
with belief_scope("DictionaryManager.add_entry"):
_validate_bcp47(source_language, "source_language")
_validate_bcp47(target_language, "target_language")
normalized = _normalize_term(source_term)
# Uniqueness is now (dictionary_id, source_term_normalized, source_language, target_language)
existing = (
db.query(DictionaryEntry)
.filter(
DictionaryEntry.dictionary_id == dict_id,
DictionaryEntry.source_term_normalized == normalized,
DictionaryEntry.source_language == source_language,
DictionaryEntry.target_language == target_language,
)
.first()
)
if existing:
raise ValueError(
f"Duplicate entry: '{source_term}' already exists in this dictionary "
f"(id={existing.id}). Use overwrite or keep_existing conflict mode."
f"Duplicate entry: '{source_term}' (lang: {source_language}{target_language}) "
f"already exists in this dictionary (id={existing.id}). "
"Use overwrite or keep_existing conflict mode."
)
logger.reason("Adding dictionary entry", {"dict_id": dict_id, "term": source_term})
logger.reason("Adding dictionary entry", {"dict_id": dict_id, "term": source_term, "src_lang": source_language, "tgt_lang": target_language})
entry = DictionaryEntry(
dictionary_id=dict_id,
source_term=source_term.strip(),
source_term_normalized=normalized,
target_term=target_term.strip(),
context_notes=context_notes.strip() if context_notes else None,
source_language=source_language.strip(),
target_language=target_language.strip(),
)
db.add(entry)
db.commit()
db.refresh(entry)
logger.reflect("Entry added", {"entry_id": entry.id})
logger.reflect("Entry added", {"entry_id": entry.id, "src_lang": source_language, "tgt_lang": target_language})
return entry
# endregion DictionaryManager.add_entry
# region DictionaryManager.edit_entry [TYPE Function]
# @PURPOSE: Edit an existing dictionary entry with duplicate-aware normalization.
# @PURPOSE: Edit an existing dictionary entry with language-pair-aware duplicate check.
# @PRE: entry_id exists.
# @POST: Entry fields are updated.
@staticmethod
def edit_entry(
db: Session, entry_id: str, source_term: str | None = None,
target_term: str | None = None, context_notes: str | None = None,
source_language: str | None = None,
target_language: str | None = None,
) -> DictionaryEntry:
with belief_scope("DictionaryManager.edit_entry"):
entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()
@@ -220,14 +252,24 @@ class DictionaryManager:
raise ValueError(f"Entry not found: {entry_id}")
logger.reason("Editing dictionary entry", {"entry_id": entry_id})
if source_language is not None:
_validate_bcp47(source_language, "source_language")
entry.source_language = source_language.strip()
if target_language is not None:
_validate_bcp47(target_language, "target_language")
entry.target_language = target_language.strip()
if source_term is not None:
normalized = _normalize_term(source_term)
# Check uniqueness within the same dictionary
# Check uniqueness within the same dictionary + language pair
existing = (
db.query(DictionaryEntry)
.filter(
DictionaryEntry.dictionary_id == entry.dictionary_id,
DictionaryEntry.source_term_normalized == normalized,
DictionaryEntry.source_language == entry.source_language,
DictionaryEntry.target_language == entry.target_language,
DictionaryEntry.id != entry_id,
)
.first()
@@ -318,6 +360,8 @@ class DictionaryManager:
delimiter: str | None = None,
on_conflict: str = "overwrite",
preview_only: bool = False,
default_source_language: str | None = None,
default_target_language: str | None = None,
) -> dict[str, Any]:
with belief_scope("DictionaryManager.import_entries"):
# Validate dictionary
@@ -359,6 +403,8 @@ class DictionaryManager:
source_term = row.get("source_term", "").strip()
target_term = row.get("target_term", "").strip()
context_notes = row.get("context_notes", "").strip() or None
source_language = row.get("source_language", "").strip() or default_source_language or "und"
target_language = row.get("target_language", "").strip() or default_target_language or "und"
if not source_term or not target_term:
result["errors"].append({
@@ -376,6 +422,8 @@ class DictionaryManager:
.filter(
DictionaryEntry.dictionary_id == dict_id,
DictionaryEntry.source_term_normalized == normalized,
DictionaryEntry.source_language == source_language,
DictionaryEntry.target_language == target_language,
)
.first()
)
@@ -384,6 +432,8 @@ class DictionaryManager:
"source_term": source_term,
"target_term": target_term,
"context_notes": context_notes,
"source_language": source_language,
"target_language": target_language,
"is_conflict": existing is not None,
"existing_target_term": existing.target_term if existing else None,
}
@@ -395,6 +445,8 @@ class DictionaryManager:
.filter(
DictionaryEntry.dictionary_id == dict_id,
DictionaryEntry.source_term_normalized == normalized,
DictionaryEntry.source_language == source_language,
DictionaryEntry.target_language == target_language,
)
.first()
)
@@ -404,6 +456,8 @@ class DictionaryManager:
existing.source_term = source_term
existing.target_term = target_term
existing.context_notes = context_notes
existing.source_language = source_language
existing.target_language = target_language
result["updated"] += 1
elif on_conflict == "keep_existing":
result["skipped"] += 1
@@ -421,6 +475,8 @@ class DictionaryManager:
source_term_normalized=normalized,
target_term=target_term,
context_notes=context_notes,
source_language=source_language,
target_language=target_language,
)
db.add(entry)
result["created"] += 1
@@ -446,14 +502,122 @@ class DictionaryManager:
return result
# endregion DictionaryManager.import_entries
# region DictionaryManager.export_entries [TYPE Function]
# @PURPOSE: Export all entries as CSV string with language columns.
# @PRE: dict_id exists.
# @POST: Returns CSV string with header: source_term, target_term, source_language, target_language, context_notes, context_data, usage_notes.
@staticmethod
def export_entries(
db: Session, dict_id: str, delimiter: str = ",",
) -> str:
with belief_scope("DictionaryManager.export_entries"):
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
if not dictionary:
raise ValueError(f"Dictionary not found: {dict_id}")
entries = (
db.query(DictionaryEntry)
.filter(DictionaryEntry.dictionary_id == dict_id)
.order_by(DictionaryEntry.source_term.asc())
.all()
)
output = io.StringIO()
fieldnames = [
"source_term", "target_term",
"source_language", "target_language",
"context_notes", "context_data", "usage_notes",
]
writer = csv.DictWriter(output, fieldnames=fieldnames, delimiter=delimiter)
writer.writeheader()
for entry in entries:
writer.writerow({
"source_term": entry.source_term,
"target_term": entry.target_term,
"source_language": entry.source_language,
"target_language": entry.target_language,
"context_notes": entry.context_notes or "",
"context_data": entry.context_data,
"usage_notes": entry.usage_notes or "",
})
result = output.getvalue()
logger.reflect("Entries exported", {"dict_id": dict_id, "count": len(entries), "delimiter": repr(delimiter)})
return result
# endregion DictionaryManager.export_entries
# region DictionaryManager.migrate_old_entries [TYPE Function]
# @PURPOSE: Migrate existing single-language dictionary entries to populate source_language and target_language.
# @PRE: db session is open.
# @POST: Entries with "und" source_language or target_language are updated with inferred values.
# @SIDE_EFFECT: Updates DictionaryEntry rows in bulk.
@staticmethod
def migrate_old_entries(db: Session) -> dict[str, int]:
with belief_scope("DictionaryManager.migrate_old_entries"):
logger.reason("Starting migration of old dictionary entries")
migrated_source = 0
migrated_target = 0
skipped = 0
# Find all entries that may need migration (source_language is "und" or target_language is "und")
entries = (
db.query(DictionaryEntry)
.filter(
(DictionaryEntry.source_language == "und") |
(DictionaryEntry.target_language == "und")
)
.all()
)
for entry in entries:
# Try to infer source_language from origin_source_language
if entry.source_language == "und":
if entry.origin_source_language:
entry.source_language = entry.origin_source_language
migrated_source += 1
# Try to infer target_language from the parent dictionary's deprecated target_language
if entry.target_language == "und":
dictionary = (
db.query(TerminologyDictionary)
.filter(TerminologyDictionary.id == entry.dictionary_id)
.first()
)
if dictionary and dictionary.target_language:
entry.target_language = dictionary.target_language
migrated_target += 1
if entry.source_language == "und" and entry.target_language == "und":
skipped += 1
db.commit()
logger.reflect("Migration complete", {
"migrated_source": migrated_source,
"migrated_target": migrated_target,
"skipped": skipped,
"total_processed": len(entries),
})
return {
"migrated_source": migrated_source,
"migrated_target": migrated_target,
"skipped": skipped,
"total_processed": len(entries),
}
# endregion DictionaryManager.migrate_old_entries
# region DictionaryManager.filter_for_batch [TYPE Function]
# @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.
# @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job,
# optionally filtered by language pair.
# @PRE: job_id exists and source_texts is a list of strings.
# @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries.
# @SIDE_EFFECT: Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.
@staticmethod
def filter_for_batch(
db: Session, source_texts: list[str], job_id: str,
source_language: str | None = None,
target_language: str | None = None,
) -> list[dict[str, Any]]:
with belief_scope("DictionaryManager.filter_for_batch"):
# Get dictionaries attached to this job
@@ -508,6 +672,18 @@ class DictionaryManager:
if not norm:
continue
# Apply language pair filtering
if source_language is not None:
src = source_language.strip().lower()
entry_src = entry.source_language.strip().lower()
if entry_src != src and entry_src != "und":
continue
if target_language is not None:
tgt = target_language.strip().lower()
entry_tgt = entry.target_language.strip().lower()
if entry_tgt != tgt:
continue
# Word-boundary-aware matching
# Build pattern: \bterm\b (case-insensitive)
# Escape regex special chars in the search term
@@ -528,6 +704,12 @@ class DictionaryManager:
"dictionary_id": entry.dictionary_id,
"dictionary_name": dict_map.get(entry.dictionary_id, ""),
"context_notes": entry.context_notes,
"context_data": entry.context_data,
"has_context": entry.has_context,
"context_source": entry.context_source,
"usage_notes": entry.usage_notes,
"source_language": entry.source_language,
"target_language": entry.target_language,
})
# Sort by dictionary link priority (order of dict_ids from link order)
@@ -539,6 +721,8 @@ class DictionaryManager:
"source_texts": len(source_texts),
"matches": len(matched),
"dictionaries_used": len(dictionaries),
"source_language": source_language,
"target_language": target_language,
})
return matched
# endregion DictionaryManager.filter_for_batch
@@ -560,6 +744,8 @@ class DictionaryManager:
origin_row_key: str | None = None,
origin_user_id: str | None = None,
on_conflict: str = "overwrite",
context_data: dict[str, Any] | None = None,
usage_notes: str | None = None,
) -> dict[str, Any]:
with belief_scope("DictionaryManager.submit_correction"):
# Validate dictionary exists
@@ -568,11 +754,22 @@ class DictionaryManager:
raise ValueError(f"Dictionary '{dict_id}' not found")
normalized = _normalize_term(source_term)
entry_src_lang = dictionary.source_dialect or "und"
entry_tgt_lang = dictionary.target_dialect or "und"
# Find existing entry: match exact language pair OR old-style "und"/"und" entries
existing = (
db.query(DictionaryEntry)
.filter(
DictionaryEntry.dictionary_id == dict_id,
DictionaryEntry.source_term_normalized == normalized,
or_(
# Exact language pair match
(DictionaryEntry.source_language == entry_src_lang) &
(DictionaryEntry.target_language == entry_tgt_lang),
# Old-style entries without language pair
(DictionaryEntry.source_language == "und") &
(DictionaryEntry.target_language == "und"),
),
)
.first()
)
@@ -606,6 +803,12 @@ class DictionaryManager:
existing.origin_row_key = origin_row_key
if origin_user_id:
existing.origin_user_id = origin_user_id
if context_data is not None:
existing.context_data = context_data
existing.has_context = bool(context_data)
existing.context_source = "auto"
if usage_notes is not None:
existing.usage_notes = usage_notes
db.flush()
result["action"] = "updated"
result["entry_id"] = existing.id
@@ -621,12 +824,18 @@ class DictionaryManager:
result["message"] = "Correction cancelled by conflict mode"
return result
else:
# Create new entry
# Create new entry — derive language pair from dictionary
entry = DictionaryEntry(
dictionary_id=dict_id,
source_term=source_term.strip(),
source_term_normalized=normalized,
target_term=corrected_target_term.strip(),
source_language=entry_src_lang,
target_language=entry_tgt_lang,
context_data=context_data,
usage_notes=usage_notes,
has_context=bool(context_data),
context_source="auto" if context_data else None,
origin_run_id=origin_run_id,
origin_row_key=origin_row_key,
origin_user_id=origin_user_id,
@@ -678,11 +887,19 @@ class DictionaryManager:
continue
normalized = _normalize_term(source_term)
bulk_src_lang = dictionary.source_dialect or "und"
bulk_tgt_lang = dictionary.target_dialect or "und"
existing = (
db.query(DictionaryEntry)
.filter(
DictionaryEntry.dictionary_id == dict_id,
DictionaryEntry.source_term_normalized == normalized,
or_(
(DictionaryEntry.source_language == bulk_src_lang) &
(DictionaryEntry.target_language == bulk_tgt_lang),
(DictionaryEntry.source_language == "und") &
(DictionaryEntry.target_language == "und"),
),
)
.first()
)
@@ -729,6 +946,8 @@ class DictionaryManager:
source_term=source_term,
source_term_normalized=normalized,
target_term=corrected_target,
source_language=bulk_src_lang,
target_language=bulk_tgt_lang,
origin_run_id=corr.get("origin_run_id"),
origin_row_key=corr.get("origin_row_key"),
origin_user_id=origin_user_id,

View File

@@ -191,6 +191,18 @@ class TranslationEventLog:
logger.reflect("No expired events to prune", {})
return {"pruned": 0, "snapshot_id": None}
# Compute per-language metrics from expired events before pruning
per_language: dict[str, dict[str, int | float]] = {}
expired_events = expired_query.all()
for evt in expired_events:
event_data = evt.event_data or {}
lang = event_data.get("language_code") or event_data.get("target_language") or "_unknown_"
if lang not in per_language:
per_language[lang] = {"cumulative_tokens": 0, "cumulative_cost": 0.0, "runs": 0}
per_language[lang]["cumulative_tokens"] += event_data.get("token_count", 0) or 0
per_language[lang]["cumulative_cost"] += event_data.get("cost", 0.0) or 0.0
per_language[lang]["runs"] += 1 if event_data.get("run_id") else 0
# Create MetricSnapshot before pruning
snapshot = MetricSnapshot(
id=str(uuid.uuid4()),
@@ -198,6 +210,7 @@ class TranslationEventLog:
key_hash=f"prune_{cutoff.timestamp():.0f}",
covers_events_before=cutoff,
total_records=total_expired,
per_language_metrics=per_language or None,
snapshot_date=datetime.now(UTC),
)
self.db.add(snapshot)

View File

@@ -27,6 +27,7 @@ from ...core.logger import belief_scope, logger
from ...models.translate import (
TranslationBatch,
TranslationJob,
TranslationLanguage,
TranslationPreviewRecord,
TranslationPreviewSession,
TranslationRecord,
@@ -36,6 +37,7 @@ from ...services.llm_prompt_templates import render_prompt
from ...services.llm_provider import LLMProviderService
from .dictionary import DictionaryManager
from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE
from .prompt_builder import ContextAwarePromptBuilder
# #region MAX_RETRIES_PER_BATCH [TYPE Constant]
# @BRIEF Maximum number of retries for a single batch before marking it failed.
@@ -62,6 +64,7 @@ class TranslationExecutor:
self.current_user = current_user
self.on_batch_progress = on_batch_progress
self._current_run_id: str | None = None
self._preview_edits_cache: dict[str, dict[str, str]] | None = None # key_hash -> {lang_code: edited_value}
# region execute_run [TYPE Function]
# @PURPOSE: Run full translation execution for a TranslationRun.
@@ -84,6 +87,9 @@ class TranslationExecutor:
"batch_size": job.batch_size,
})
# Load preview edits for carry-forward
self._load_preview_edits(job.id)
# Mark run as RUNNING
run.status = "RUNNING"
run.started_at = datetime.now(UTC)
@@ -410,6 +416,70 @@ class TranslationExecutor:
return []
# endregion _extract_chart_data_rows
# region _load_preview_edits [TYPE Function]
# @PURPOSE: Load user edits from accepted preview session for carry-forward.
# @PRE: job_id exists.
# @POST: Populates _preview_edits_cache with key_hash -> {lang_code: edited_value}.
# @SIDE_EFFECT: Queries TranslationPreviewLanguage and TranslationPreviewRecord.
def _load_preview_edits(self, job_id: str) -> None:
"""Load preview edits for carry-forward during execution."""
from ...models.translate import TranslationPreviewLanguage, TranslationPreviewRecord, TranslationPreviewSession
with belief_scope("TranslationExecutor._load_preview_edits"):
session = (
self.db.query(TranslationPreviewSession)
.filter(
TranslationPreviewSession.job_id == job_id,
TranslationPreviewSession.status == "APPLIED",
)
.order_by(TranslationPreviewSession.created_at.desc())
.first()
)
if not session:
logger.reason("No applied preview session found — no edits to carry forward", {
"job_id": job_id,
})
self._preview_edits_cache = {}
return
records = (
self.db.query(TranslationPreviewRecord)
.filter(TranslationPreviewRecord.session_id == session.id)
.all()
)
edits: dict[str, dict[str, str]] = {}
for rec in records:
if not rec.source_data:
continue
# Compute key hash from source_data
key_hash = self._compute_key_hash(rec.source_data)
edited_langs: dict[str, str] = {}
for lang_entry in (rec.languages or []):
if lang_entry.status in ("edited", "approved") and lang_entry.user_edit:
edited_langs[lang_entry.language_code] = lang_entry.final_value or lang_entry.user_edit
logger.reason("Carrying forward preview edit", {
"key_hash": key_hash,
"language_code": lang_entry.language_code,
})
if edited_langs:
edits[key_hash] = edited_langs
self._preview_edits_cache = edits
logger.reason(f"Loaded {len(edits)} preview edits for carry-forward", {
"job_id": job_id,
})
# endregion _load_preview_edits
# region _compute_key_hash [TYPE Function]
# @PURPOSE: Compute a stable hash from source_data dict for matching preview edits.
@staticmethod
def _compute_key_hash(source_data: dict) -> str:
import hashlib
stable = json.dumps(source_data, sort_keys=True)
return hashlib.sha256(stable.encode()).hexdigest()[:16]
# endregion _compute_key_hash
# region _process_batch [TYPE Function]
# @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records.
# @PRE: job and batch_rows are valid.
@@ -452,17 +522,39 @@ class TranslationExecutor:
self.db, source_texts, job.id
)
# For each row, determine if we need LLM translation or can use approved translation
# For each row, determine if we need LLM translation or can use approved/preview edit
rows_for_llm = []
pre_translated = []
for row in batch_rows:
if row.get("approved_translation"):
pre_translated.append(row)
else:
rows_for_llm.append(row)
continue
# Check for preview edits carry-forward
source_data = row.get("source_data") or {}
if source_data and self._preview_edits_cache:
key_hash = self._compute_key_hash(source_data)
preview_edit = self._preview_edits_cache.get(key_hash)
if preview_edit:
# Use the first edited language's value as the approved translation
first_edit = next(iter(preview_edit.values()), None)
if first_edit:
row["approved_translation"] = first_edit
logger.reason("Using preview edit carry-forward", {
"key_hash": key_hash,
"langs": list(preview_edit.keys()),
})
pre_translated.append(row)
continue
rows_for_llm.append(row)
# Handle pre-translated (approved) rows
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
for row in pre_translated:
record = TranslationRecord(
id=str(uuid.uuid4()),
@@ -477,6 +569,20 @@ class TranslationExecutor:
status="SUCCESS",
)
self.db.add(record)
# Create per-language entry for each target language (pre-approved)
for lang_code in target_languages:
lang_entry = TranslationLanguage(
id=str(uuid.uuid4()),
record_id=record.id,
language_code=lang_code,
source_language_detected="und",
translated_value=row.get("approved_translation"),
final_value=row.get("approved_translation"),
status="translated",
needs_review=False,
)
self.db.add(lang_entry)
result["successful"] += 1
# Process rows needing LLM translation
@@ -524,21 +630,31 @@ class TranslationExecutor:
batch_id: str,
) -> dict[str, int]:
with belief_scope("TranslationExecutor._call_llm_for_batch"):
# Build dictionary section
# Build dictionary section using ContextAwarePromptBuilder
dictionary_section = ""
if dict_matches:
# Get row_context from first batch row if available
row_context = batch_rows[0].get("source_data") if batch_rows else None
# Use ContextAwarePromptBuilder for context-aware annotations
annotated_entries = ContextAwarePromptBuilder.build_context_entries(
dict_matches, row_context
)
glossary_lines = []
for m in dict_matches:
glossary_lines.append(
f"- '{m['source_term']}' -> '{m['target_term']}'"
f"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}"
)
for annotated_line in annotated_entries:
glossary_lines.append(f"- {annotated_line}")
dictionary_section = (
"Terminology dictionary (use these translations when applicable):\n"
+ "\n".join(glossary_lines)
+ "\n\n"
)
# Resolve target languages
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
target_languages_str = ", ".join(target_languages)
# Build rows JSON for LLM
rows_json = json.dumps([
{
@@ -548,12 +664,13 @@ class TranslationExecutor:
for idx, row in enumerate(batch_rows)
], indent=2)
# Build prompt
# Build prompt (use multi-language format)
prompt = render_prompt(
DEFAULT_EXECUTION_PROMPT_TEMPLATE,
{
"source_language": job.source_dialect or "SQL",
"target_language": job.target_language or job.target_dialect or "en",
"target_language": target_languages_str,
"target_languages": target_languages_str,
"source_dialect": job.source_dialect or "",
"target_dialect": job.target_dialect or "",
"translation_column": job.translation_column or "",
@@ -607,9 +724,9 @@ class TranslationExecutor:
self.db.add(record)
return {"successful": 0, "failed": len(batch_rows), "skipped": 0, "retries": retries}
# Parse LLM response
# Parse LLM response (multi-language aware)
try:
translations = self._parse_llm_response(llm_response, len(batch_rows))
translations = self._parse_llm_response(llm_response, len(batch_rows), target_languages=target_languages)
except ValueError as e:
# Parse failure — mark all rows as SKIPPED
logger.explore("LLM response parse failed", {
@@ -646,16 +763,20 @@ class TranslationExecutor:
for row in batch_rows:
row_id = str(row.get("row_index", ""))
translation = translations.get(row_id)
translation_data = translations.get(row_id)
source_text = row.get("source_text", "")
detected_lang = "und"
if translation_data:
detected_lang = translation_data.get("detected_source_language", "und")
if translation is None:
if translation_data is None:
# NULL translation — skip
skipped += 1
record = TranslationRecord(
id=str(uuid.uuid4()),
batch_id=batch_id,
run_id=run_id,
source_sql=row.get("source_text", ""),
source_sql=source_text,
target_sql="",
source_object_type="table_row",
source_object_id=row.get("row_index"),
@@ -667,14 +788,32 @@ class TranslationExecutor:
self.db.add(record)
continue
if translation.strip() == "":
# Empty translation — skip
# Collect per-language translated values
per_lang_values: dict[str, str] = {}
has_any_translation = False
# First try multi-language format (per-language keys)
for lang_code in target_languages:
lang_val = translation_data.get(lang_code)
if lang_val is not None and str(lang_val).strip():
per_lang_values[lang_code] = str(lang_val)
has_any_translation = True
# Fallback to legacy single "translation" key format
if not has_any_translation:
translation_text = translation_data.get("translation", "")
if translation_text.strip():
per_lang_values[target_languages[0]] = translation_text
has_any_translation = True
if not has_any_translation:
# Empty/all-empty translations — skip
skipped += 1
record = TranslationRecord(
id=str(uuid.uuid4()),
batch_id=batch_id,
run_id=run_id,
source_sql=row.get("source_text", ""),
source_sql=source_text,
target_sql="",
source_object_type="table_row",
source_object_id=row.get("row_index"),
@@ -686,13 +825,16 @@ class TranslationExecutor:
self.db.add(record)
continue
# Use the first language's value as the primary target_sql for backward compat
primary_translation = next(iter(per_lang_values.values()), "")
successful += 1
record = TranslationRecord(
id=str(uuid.uuid4()),
batch_id=batch_id,
run_id=run_id,
source_sql=row.get("source_text", ""),
target_sql=translation,
source_sql=source_text,
target_sql=primary_translation,
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
@@ -701,6 +843,37 @@ class TranslationExecutor:
)
self.db.add(record)
# Create per-language entries — one TranslationLanguage per language_code
for lang_code in target_languages:
lang_translation = per_lang_values.get(lang_code, "")
# Source-as-reference: if detected source language matches this target,
# store the original text verbatim (no translation needed)
if detected_lang != "und" and str(lang_code).lower() == str(detected_lang).lower():
lang_translation = source_text
# Check for undetermined source language
lang_needs_review = (detected_lang == "und")
if lang_needs_review:
logger.explore("undetected language", {
"record_id": row_id,
"language_code": lang_code,
"source_text": source_text[:100],
})
lang_entry = TranslationLanguage(
id=str(uuid.uuid4()),
record_id=record.id,
language_code=lang_code,
source_language_detected=detected_lang,
translated_value=lang_translation or "",
final_value=lang_translation or "",
status="translated",
needs_review=lang_needs_review,
)
self.db.add(lang_entry)
return {
"successful": successful,
"failed": failed,
@@ -806,11 +979,13 @@ class TranslationExecutor:
# endregion _call_openai_compatible
# region _parse_llm_response [TYPE Function]
# @PURPOSE: Parse LLM JSON response into dict of row_id -> translation.
# @PURPOSE: Parse LLM JSON response into dict of row_id -> per-language translations.
# Supports multi-language format: {"row_id": 0, "detected_source_language": "fr", "ru": "текст", "en": "text"}
# Backward-compat: old format {"row_id": 0, "translation": "text", "detected_source_language": "fr"}
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
# @POST: Returns dict mapping row_id to translation text.
# @POST: Returns dict mapping row_id to dict with 'detected_source_language' and per-language codes.
@staticmethod
def _parse_llm_response(response_text: str, expected_count: int) -> dict[str, str]:
def _parse_llm_response(response_text: str, expected_count: int, target_languages: list[str] | None = None) -> dict[str, dict[str, str]]:
with belief_scope("TranslationExecutor._parse_llm_response"):
try:
data = json.loads(response_text)
@@ -830,15 +1005,41 @@ class TranslationExecutor:
if not isinstance(rows, list):
raise ValueError("LLM response missing 'rows' array")
translations: dict[str, str] = {}
translations: dict[str, dict[str, str]] = {}
for item in rows:
row_id = str(item.get("row_id", ""))
translation = item.get("translation")
if translation is None:
# Skip NULL translations — they'll be handled by caller
if not row_id:
continue
if row_id:
translations[row_id] = str(translation)
detected_lang = str(item.get("detected_source_language", "und")) if item.get("detected_source_language") else "und"
result: dict[str, str] = {
"detected_source_language": detected_lang,
}
# Multi-language format: extract per-language keys
has_language_data = False
if target_languages:
for lang_code in target_languages:
lang_val = item.get(lang_code)
if lang_val is not None and str(lang_val).strip():
result[lang_code] = str(lang_val)
has_language_data = True
# Fallback to legacy single "translation" key format
if not has_language_data:
translation = item.get("translation")
if translation is not None:
result["translation"] = str(translation)
has_language_data = True
if has_language_data:
translations[row_id] = result
if len(translations) < expected_count:
logger.explore(
f"LLM returned fewer translations expected={expected_count} "
f"got={len(translations)}"
)
return translations
# endregion _parse_llm_response

View File

@@ -21,6 +21,7 @@ from ...models.translate import (
MetricSnapshot,
TranslationEvent,
TranslationRun,
TranslationRunLanguageStats,
TranslationSchedule,
)
@@ -121,17 +122,37 @@ class TranslationMetrics:
# indicates cutoff
pass
# Live events (<90 days) for token/cost
cutoff = datetime.now(UTC)
live_events = (
self.db.query(TranslationEvent)
.filter(
TranslationEvent.job_id == job_id,
TranslationEvent.event_type.in_(["TRANSLATION_PHASE_COMPLETED", "RUN_COMPLETED"]),
TranslationEvent.created_at > cutoff, # events newer than snapshot
)
# Per-language metrics from TranslationRunLanguageStats (live runs)
per_language: dict[str, dict[str, int | float]] = {}
lang_stats_rows = (
self.db.query(TranslationRunLanguageStats)
.join(TranslationRun, TranslationRunLanguageStats.run_id == TranslationRun.id)
.filter(TranslationRun.job_id == job_id)
.all()
)
for ls in lang_stats_rows:
lang = ls.language_code
if lang not in per_language:
per_language[lang] = {"tokens": 0, "cost": 0.0, "runs": 0, "translated_rows": 0}
per_language[lang]["tokens"] += ls.token_count or 0
per_language[lang]["cost"] += ls.estimated_cost or 0.0
per_language[lang]["runs"] += 1
per_language[lang]["translated_rows"] += ls.translated_rows or 0
# Merge in per-language data from MetricSnapshot (pruned period)
latest_snapshot = (
self.db.query(MetricSnapshot)
.filter(MetricSnapshot.job_id == job_id)
.order_by(MetricSnapshot.snapshot_date.desc())
.first()
)
if latest_snapshot and latest_snapshot.per_language_metrics:
for lang, snap_data in latest_snapshot.per_language_metrics.items():
if lang not in per_language:
per_language[lang] = {"tokens": 0, "cost": 0.0, "runs": 0, "translated_rows": 0}
per_language[lang]["tokens"] += snap_data.get("cumulative_tokens", 0)
per_language[lang]["cost"] += snap_data.get("cumulative_cost", 0.0)
per_language[lang]["runs"] += snap_data.get("runs", 0)
return {
"job_id": job_id,
@@ -148,6 +169,7 @@ class TranslationMetrics:
"avg_duration_ms": int(avg_duration) if avg_duration else None,
"last_run_at": last_run.created_at.isoformat() if last_run else None,
"next_scheduled_run": next_schedule.last_run_at.isoformat() if next_schedule and next_schedule.last_run_at else None,
"per_language_metrics": per_language,
}
# endregion get_job_metrics

View File

@@ -30,9 +30,11 @@ from ...core.logger import belief_scope, logger
from ...models.translate import (
TranslationBatch,
TranslationJob,
TranslationLanguage,
TranslationPreviewSession,
TranslationRecord,
TranslationRun,
TranslationRunLanguageStats,
)
from .events import TranslationEventLog
from .executor import TranslationExecutor
@@ -201,6 +203,27 @@ class TranslationOrchestrator:
created_by=self.current_user,
)
# Initialize per-language stats
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
language_stats_map: dict[str, TranslationRunLanguageStats] = {}
for lang_code in target_languages:
lang_stat = TranslationRunLanguageStats(
id=str(uuid.uuid4()),
run_id=run.id,
language_code=lang_code,
total_rows=0,
translated_rows=0,
failed_rows=0,
skipped_rows=0,
token_count=0,
estimated_cost=0.0,
)
self.db.add(lang_stat)
language_stats_map[lang_code] = lang_stat
self.db.flush()
# Dispatch executor
executor = TranslationExecutor(
self.db, self.config_manager, self.current_user,
@@ -227,6 +250,9 @@ class TranslationOrchestrator:
self.db.commit()
return run
# Aggregate per-language statistics after executor completes
self._update_language_stats(run.id, language_stats_map)
# Record translation phase complete
self.event_log.log_event(
job_id=job.id,
@@ -745,6 +771,25 @@ class TranslationOrchestrator:
# Get event summary
event_summary = self.event_log.get_run_event_summary(run_id)
# Get language stats
language_stats_entries = (
self.db.query(TranslationRunLanguageStats)
.filter(TranslationRunLanguageStats.run_id == run_id)
.all()
)
language_stats = [
{
"language_code": ls.language_code,
"total_rows": ls.total_rows or 0,
"translated_rows": ls.translated_rows or 0,
"failed_rows": ls.failed_rows or 0,
"skipped_rows": ls.skipped_rows or 0,
"token_count": ls.token_count or 0,
"estimated_cost": ls.estimated_cost or 0.0,
}
for ls in language_stats_entries
]
return {
"id": run.id,
"job_id": run.job_id,
@@ -759,6 +804,7 @@ class TranslationOrchestrator:
"insert_status": run.insert_status,
"superset_execution_id": run.superset_execution_id,
"batch_count": batch_count,
"language_stats": language_stats,
"event_invariants": {
"has_run_started": event_summary["has_run_started"],
"terminal_event_count": event_summary["terminal_event_count"],
@@ -863,6 +909,80 @@ class TranslationOrchestrator:
]
# endregion get_run_history
# region _update_language_stats [TYPE Function]
# @PURPOSE: Aggregate TranslationLanguage entries by language_code and update TranslationRunLanguageStats.
# @PRE: run_id and language_stats_map are valid. DB session is available.
# @POST: Language stats are updated with row counts and estimated tokens/cost.
# @SIDE_EFFECT: DB writes on language_stats objects.
def _update_language_stats(
self,
run_id: str,
language_stats_map: dict[str, TranslationRunLanguageStats],
) -> None:
with belief_scope("TranslationOrchestrator._update_language_stats"):
# Get all records for this run to join with TranslationLanguage
records = (
self.db.query(TranslationRecord)
.filter(TranslationRecord.run_id == run_id)
.all()
)
record_ids = [r.id for r in records]
if not record_ids:
logger.reason("No records for language stats aggregation", {"run_id": run_id})
return
# Get all language entries for this run's records
lang_entries = (
self.db.query(TranslationLanguage)
.filter(TranslationLanguage.record_id.in_(record_ids))
.all()
)
# Aggregate by language_code
from collections import defaultdict
agg: dict[str, dict[str, int]] = defaultdict(lambda: {"total": 0, "translated": 0, "failed": 0, "skipped": 0})
for le in lang_entries:
code = le.language_code
agg[code]["total"] += 1
if le.status in ("translated", "approved", "edited"):
agg[code]["translated"] += 1
elif le.status == "failed":
agg[code]["failed"] += 1
elif le.status == "skipped":
agg[code]["skipped"] += 1
# Estimate tokens: heuristic based on character count of translated values
total_chars = sum(
len(le.translated_value or "") for le in lang_entries if le.translated_value
)
total_tokens = max(1, total_chars // 4) # ~4 chars per token
cost_per_token = 0.002 / 1000 # $0.002 per 1K tokens
# Update each language stat entry
for lang_code, lang_stat in language_stats_map.items():
data = agg.get(lang_code, {"total": 0, "translated": 0, "failed": 0, "skipped": 0})
lang_stat.total_rows = data["total"]
lang_stat.translated_rows = data["translated"]
lang_stat.failed_rows = data["failed"]
lang_stat.skipped_rows = data["skipped"]
# Proportional token split: share tokens across languages
num_langs = len(language_stats_map)
if num_langs > 0:
lang_stat.token_count = total_tokens // num_langs
lang_stat.estimated_cost = round((lang_stat.token_count / 1000) * cost_per_token, 6)
self.db.flush()
logger.reason("Language stats updated", {
"run_id": run_id,
"languages": list(language_stats_map.keys()),
"total_tokens_est": total_tokens,
})
# endregion _update_language_stats
# region _compute_config_hash [TYPE Function]
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
@staticmethod

View File

@@ -29,6 +29,7 @@ from ...core.superset_client import SupersetClient
from ...models.translate import (
TranslationJob,
TranslationJobDictionary,
TranslationPreviewLanguage,
TranslationPreviewRecord,
TranslationPreviewSession,
)
@@ -38,16 +39,19 @@ from .dictionary import DictionaryManager
# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [TYPE Constant]
# @BRIEF Default prompt template for batch LLM translation execution (no context columns — faster).
# Supports both single-language and multi-language modes via {target_languages} placeholder.
DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
"Translate the following database content from {source_language} to {target_language}.\n\n"
"Translate the following database content from {source_language} to the following language(s): {target_languages}.\n\n"
"Source dialect: {source_dialect}\n"
"Target dialect: {target_dialect}\n"
"Target dialect(s): {target_dialect}\n"
"Column to translate: {translation_column}\n\n"
"{dictionary_section}"
"For each row, provide an accurate translation of the text.\n\n"
"For each row, provide an accurate translation of the text into each target language.\n\n"
"Rows to translate:\n{rows_json}\n\n"
"Respond with a JSON object in this exact format:\n"
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
'{{"rows": [{{"row_id": "<row_index>", "detected_source_language": "<bcp47_or_und>", "<language_code_1>": "<translation_in_lang_1>", "<language_code_2>": "<translation_in_lang_2>"}}]}}\n'
"For each row, return the detected source language as a BCP-47 tag (e.g. 'en', 'ru', 'fr'), or 'und' if uncertain.\n"
"Include a separate key for EACH target language code with the translated text in that language.\n"
"Each row_id must match the index provided. Return exactly {row_count} entries."
)
# #endregion DEFAULT_EXECUTION_PROMPT_TEMPLATE
@@ -56,16 +60,18 @@ DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [TYPE Constant]
# @BRIEF Default prompt template for LLM translation preview.
DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
"Translate the following database content from {source_language} to {target_language}.\n\n"
"Translate the following database content.\n\n"
"Source dialect: {source_dialect}\n"
"Target dialect: {target_dialect}\n"
"Column to translate: {translation_column}\n\n"
"{dictionary_section}"
"For each row, provide an accurate translation of the '{translation_column}' value.\n"
"Translate to the following languages: {target_languages}\n\n"
"For each row, provide an accurate translation of the '{translation_column}' value into each language.\n"
"Consider the context columns when determining the meaning of the text.\n\n"
"Rows to translate:\n{rows_json}\n\n"
"Respond with a JSON object in this exact format:\n"
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
'{{"rows": [{{"row_id": "<row_index>", "detected_source_language": "<bcp47_or_und>", "language_code_1": "<translation_in_lang_1>", "language_code_2": "<translation_in_lang_2>"}}]}}\n'
"For each row, return the detected source language as a BCP-47 tag (e.g. 'en', 'ru', 'fr'), or 'und' if uncertain.\n"
"Include a separate key for EACH target language code with the translated text in that language.\n"
"Each row_id must match the index provided. Return exactly {row_count} entries."
)
# #endregion DEFAULT_PREVIEW_PROMPT_TEMPLATE
@@ -80,7 +86,9 @@ class TokenEstimator:
CHARS_PER_TOKEN_ESTIMATE: float = 4.0
OUTPUT_TOKENS_PER_ROW_ESTIMATE: int = 50
MULTI_LANG_FACTOR: float = 1.2 # Overhead for multi-language in one call
TOKEN_COST_PER_1K: float = 0.002 # Default cost per 1K tokens
COST_WARNING_THRESHOLD: int = 30 # Show warning above this sample size
# region estimate_prompt_tokens [TYPE Function]
# @PURPOSE: Estimate token count for a prompt string.
@@ -94,12 +102,12 @@ class TokenEstimator:
# endregion estimate_prompt_tokens
# region estimate_output_tokens [TYPE Function]
# @PURPOSE: Estimate output token count for translating N rows.
# @PRE: row_count >= 0.
# @PURPOSE: Estimate output token count for translating N rows across N languages.
# @PRE: row_count >= 0, num_languages >= 1.
# @POST: Returns estimated output token count.
@staticmethod
def estimate_output_tokens(row_count: int) -> int:
return row_count * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE
def estimate_output_tokens(row_count: int, num_languages: int = 1) -> int:
return int(row_count * num_languages * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE * TokenEstimator.MULTI_LANG_FACTOR)
# endregion estimate_output_tokens
# region estimate_cost [TYPE Function]
@@ -112,6 +120,20 @@ class TokenEstimator:
return round((total_tokens / 1000) * rate, 6)
# endregion estimate_cost
# region check_cost_warning [TYPE Function]
# @PURPOSE: Generate cost warning for large previews.
# @PRE: sample_size > 0, num_languages >= 1.
# @POST: Returns warning string or None.
@staticmethod
def check_cost_warning(sample_size: int, num_languages: int, estimated_tokens: int, estimated_cost: float) -> str | None:
if sample_size > TokenEstimator.COST_WARNING_THRESHOLD:
return (
f"Large preview — estimated {estimated_tokens} tokens, ~${estimated_cost:.4f} cost "
f"(across {num_languages} languages)"
)
return None
# endregion check_cost_warning
# #endregion TokenEstimator
@@ -134,10 +156,10 @@ class TranslationPreview:
self.current_user = current_user
# region preview_rows [TYPE Function]
# @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.
# @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for multi-language translation, create preview session with per-language records.
# @PRE: job_id exists and job has source_datasource_id, translation_column configured.
# @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session.
# @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.
# @POST: Returns TranslationPreviewResponse with per-language records, cost estimation, and persistent session.
# @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord + TranslationPreviewLanguage rows.
def preview_rows(
self,
job_id: str,
@@ -157,6 +179,12 @@ class TranslationPreview:
if not job.translation_column:
raise ValueError("Job must have a translation column configured for preview")
# Resolve target languages: prefer target_languages list, fallback to single target_language
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
num_languages = len(target_languages)
# 2. Compute config hash and dict snapshot hash
config_hash = self._compute_config_hash(job)
dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id)
@@ -224,40 +252,48 @@ class TranslationPreview:
+ "\n\n"
)
# 6. Build LLM prompt
# 6. Build LLM prompt with multi-language instructions
rows_json = json.dumps([
{"row_id": str(m["row_index"]), "text": m["source_text"], "context": m["context_data"]}
for m in row_meta
], indent=2)
target_languages_str = ", ".join(target_languages)
template = prompt_template or DEFAULT_PREVIEW_PROMPT_TEMPLATE
prompt = render_prompt(template, {
"source_language": job.source_dialect or "SQL",
"target_language": job.target_language or job.target_dialect or "en",
"target_language": target_languages_str,
"source_dialect": job.source_dialect or "",
"target_dialect": job.target_dialect or "",
"target_languages": target_languages_str,
"translation_column": job.translation_column or "",
"dictionary_section": dictionary_section,
"rows_json": rows_json,
"row_count": str(actual_row_count),
})
# 7. Estimate tokens/cost for sample and full dataset
# 7. Estimate tokens/cost for sample with multi-language factor
sample_prompt_tokens = TokenEstimator.estimate_prompt_tokens(prompt)
sample_output_tokens = TokenEstimator.estimate_output_tokens(actual_row_count)
sample_output_tokens = TokenEstimator.estimate_output_tokens(actual_row_count, num_languages)
sample_total_tokens = sample_prompt_tokens + sample_output_tokens
sample_cost = TokenEstimator.estimate_cost(sample_total_tokens)
# Estimate full dataset cost (if we knew total rows)
# Estimate full dataset cost (extrapolated)
total_est_rows = sample_size * 10 # rough extrapolation
total_est_tokens = TokenEstimator.estimate_prompt_tokens(
prompt.replace(str(actual_row_count), "{total}")
) + TokenEstimator.estimate_output_tokens(sample_size * 10) # rough extrapolation
prompt.replace(str(actual_row_count), str(total_est_rows))
) + TokenEstimator.estimate_output_tokens(total_est_rows, num_languages)
total_est_cost = TokenEstimator.estimate_cost(total_est_tokens)
# Cost warning for large previews
cost_warning = TokenEstimator.check_cost_warning(
sample_size, num_languages, sample_total_tokens, sample_cost
)
# 8. Call LLM
logger.reason("Calling LLM for preview translation", {
"provider_id": job.provider_id,
"row_count": actual_row_count,
"num_languages": num_languages,
"estimated_tokens": sample_total_tokens,
})
llm_response = self._call_llm(
@@ -265,8 +301,10 @@ class TranslationPreview:
prompt=prompt,
)
# 9. Parse LLM response
translations = self._parse_llm_response(llm_response, actual_row_count)
# 9. Parse LLM response (multi-language)
translations = self._parse_llm_response(
llm_response, actual_row_count, target_languages=target_languages
)
# 10. Create preview session
session = TranslationPreviewSession(
@@ -280,14 +318,50 @@ class TranslationPreview:
self.db.add(session)
self.db.flush()
# 11. Create preview records
# 11. Create preview records with per-language entries
records = []
for meta in row_meta:
idx = meta["row_index"]
translation = translations.get(str(idx), "")
is_rejected = False
status = "PENDING"
feedback = None
source_text = meta["source_text"]
translation_data = translations.get(str(idx), {})
detected_lang = translation_data.get("detected_source_language", "und") if isinstance(translation_data, dict) else "und"
# Extract per-language translations
lang_entries: list[TranslationPreviewLanguage] = []
for lang_code in target_languages:
# Get translation for this language from LLM response
lang_translation = translation_data.get(lang_code) if isinstance(translation_data, dict) else None
# Fallback: if no per-language data, use the single "translation" key
if not lang_translation:
if isinstance(translation_data, dict):
lang_translation = translation_data.get("translation", "")
elif isinstance(translation_data, str):
lang_translation = translation_data
else:
lang_translation = ""
# Source-as-reference: if this language code matches source, use original text
if str(lang_code).lower() == str(detected_lang).lower() and detected_lang != "und":
# Use source text as reference (no translation needed for source language)
lang_translation = source_text
lang_needs_review = detected_lang == "und"
lang_entry = TranslationPreviewLanguage(
id=str(uuid.uuid4()),
preview_record_id="", # Will set after record is created
language_code=lang_code,
source_language_detected=detected_lang,
translated_value=str(lang_translation),
final_value=str(lang_translation),
status="pending",
needs_review=lang_needs_review,
)
lang_entries.append(lang_entry)
# Determine overall status and needs_review for the record
overall_needs_review = any(le.needs_review for le in lang_entries)
# Extract source_data: store original row key columns for upsert matching
source_row = meta.get("source_row", {})
@@ -299,24 +373,41 @@ class TranslationPreview:
if k in source_row
}
elif source_row:
# No key columns configured — store the full row as fallback
source_data = dict(source_row)
# Use first language's translation as target_sql for backward compat
primary_translation = lang_entries[0].translated_value if lang_entries else ""
record = TranslationPreviewRecord(
id=str(uuid.uuid4()),
session_id=session.id,
source_sql=meta["source_text"],
target_sql=translation,
source_sql=source_text,
target_sql=primary_translation,
source_object_type="table_row",
source_object_id=str(idx),
source_object_name=f"Row {idx + 1}",
source_data=source_data,
status=status,
feedback=feedback,
status="PENDING",
feedback=None,
created_at=datetime.now(UTC),
)
self.db.add(record)
self.db.flush()
# Now set the preview_record_id for all lang entries and persist
serialized_langs = []
for le in lang_entries:
le.preview_record_id = record.id
self.db.add(le)
serialized_langs.append({
"language_code": le.language_code,
"source_language_detected": le.source_language_detected,
"translated_value": le.translated_value,
"final_value": le.final_value,
"status": le.status,
"needs_review": le.needs_review,
})
records.append({
"id": record.id,
"source_sql": record.source_sql,
@@ -326,6 +417,9 @@ class TranslationPreview:
"source_object_name": record.source_object_name,
"status": record.status,
"feedback": record.feedback,
"source_language_detected": detected_lang,
"needs_review": overall_needs_review,
"languages": serialized_langs,
})
self.db.commit()
@@ -338,15 +432,18 @@ class TranslationPreview:
"created_at": session.created_at.isoformat(),
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
"records": records,
"target_languages": target_languages,
"cost_estimate": {
"sample_size": actual_row_count,
"num_languages": num_languages,
"sample_prompt_tokens": sample_prompt_tokens,
"sample_output_tokens": sample_output_tokens,
"sample_total_tokens": sample_total_tokens,
"sample_cost": sample_cost,
"estimated_total_rows": actual_row_count * 10,
"estimated_total_rows": total_est_rows,
"estimated_tokens": total_est_tokens,
"estimated_cost": total_est_cost,
"warning": cost_warning,
},
"config_hash": config_hash,
"dict_snapshot_hash": dict_snapshot_hash,
@@ -355,15 +452,16 @@ class TranslationPreview:
logger.reflect("Preview completed", {
"session_id": session.id,
"row_count": actual_row_count,
"num_languages": num_languages,
"sample_cost": sample_cost,
})
return result
# endregion preview_rows
# region update_preview_row [TYPE Function]
# @PURPOSE: Approve, edit, or reject an individual preview row.
# @PURPOSE: Approve, edit, or reject an individual preview row (optionally per language).
# @PRE: session_id and row_id exist, session is ACTIVE.
# @POST: PreviewRecord status is updated.
# @POST: PreviewRecord status is updated. If language_code provided, only that TranslationPreviewLanguage is updated.
def update_preview_row(
self,
job_id: str,
@@ -371,6 +469,7 @@ class TranslationPreview:
action: str,
translation: str | None = None,
feedback: str | None = None,
language_code: str | None = None,
) -> dict[str, Any]:
with belief_scope("TranslationPreview.update_preview_row"):
# Find the active session for this job
@@ -397,16 +496,76 @@ class TranslationPreview:
if not record:
raise ValueError(f"Preview record '{row_id}' not found in active session")
if action == "approve":
record.status = "APPROVED"
elif action == "reject":
record.status = "REJECTED"
elif action == "edit":
record.status = "APPROVED"
if translation is not None:
record.target_sql = translation
# If language_code specified, operate on the specific TranslationPreviewLanguage
if language_code:
lang_entry = (
self.db.query(TranslationPreviewLanguage)
.filter(
TranslationPreviewLanguage.preview_record_id == row_id,
TranslationPreviewLanguage.language_code == language_code,
)
.first()
)
if not lang_entry:
raise ValueError(f"Language entry '{language_code}' not found for record '{row_id}'")
if action == "approve":
lang_entry.status = "approved"
elif action == "reject":
lang_entry.status = "rejected"
elif action == "edit":
lang_entry.status = "edited"
if translation is not None:
lang_entry.translated_value = translation
lang_entry.final_value = translation
lang_entry.user_edit = translation
else:
raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.")
# Update record-level status based on all language entries
all_langs = (
self.db.query(TranslationPreviewLanguage)
.filter(TranslationPreviewLanguage.preview_record_id == row_id)
.all()
)
all_approved = all(le.status in ("approved", "edited") for le in all_langs)
any_rejected = any(le.status == "rejected" for le in all_langs)
if all_approved:
record.status = "APPROVED"
elif any_rejected and not all_langs:
record.status = "REJECTED"
# Update target_sql to reflect primary (first) language edit
if action == "edit" and translation is not None and all_langs:
record.target_sql = all_langs[0].final_value or all_langs[0].translated_value
else:
raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.")
# Legacy behavior: operate on whole record (all languages)
if action == "approve":
record.status = "APPROVED"
# Approve all pending language entries
for lang_entry in record.languages:
if lang_entry.status == "pending":
lang_entry.status = "approved"
elif action == "reject":
record.status = "REJECTED"
for lang_entry in record.languages:
if lang_entry.status == "pending":
lang_entry.status = "rejected"
elif action == "edit":
record.status = "APPROVED"
if translation is not None:
record.target_sql = translation
# Update primary (first) language entry
if record.languages:
first_lang = record.languages[0]
first_lang.translated_value = translation
first_lang.final_value = translation
first_lang.user_edit = translation
first_lang.status = "edited"
else:
raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.")
if feedback is not None:
record.feedback = feedback
@@ -418,21 +577,37 @@ class TranslationPreview:
"row_id": row_id,
"session_id": session.id,
"status": record.status,
"language_code": language_code,
})
# Build response with per-language data
lang_responses = []
if record.languages:
for le in record.languages:
lang_responses.append({
"language_code": le.language_code,
"source_language_detected": le.source_language_detected,
"translated_value": le.translated_value,
"final_value": le.final_value,
"user_edit": le.user_edit,
"status": le.status,
"needs_review": le.needs_review,
})
return {
"id": record.id,
"source_sql": record.source_sql,
"target_sql": record.target_sql,
"status": record.status,
"feedback": record.feedback,
"languages": lang_responses,
}
# endregion update_preview_row
# region accept_preview_session [TYPE Function]
# @PURPOSE: Mark a preview session as accepted, which gates full execution.
# @PRE: job_id has an ACTIVE preview session.
# @POST: Session status changes to APPLIED.
# @POST: Session status changes to APPLIED. User edits are persisted for carry-forward.
# @SIDE_EFFECT: Future full execution calls will check for accepted session.
def accept_preview_session(self, job_id: str) -> dict[str, Any]:
with belief_scope("TranslationPreview.accept_preview_session"):
@@ -464,6 +639,12 @@ class TranslationPreview:
.all()
)
# Resolve target languages
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"] if job else ["en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
return {
"id": session.id,
"job_id": job_id,
@@ -471,6 +652,7 @@ class TranslationPreview:
"created_by": session.created_by,
"created_at": session.created_at.isoformat(),
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
"target_languages": target_languages,
"records": [
{
"id": r.id,
@@ -478,6 +660,28 @@ class TranslationPreview:
"target_sql": r.target_sql,
"status": r.status,
"feedback": r.feedback,
"source_language_detected": (
r.languages[0].source_language_detected
if r.languages and r.languages[0].source_language_detected
else None
),
"needs_review": (
r.languages[0].needs_review
if r.languages
else False
),
"languages": [
{
"language_code": le.language_code,
"source_language_detected": le.source_language_detected,
"translated_value": le.translated_value,
"final_value": le.final_value,
"user_edit": le.user_edit,
"status": le.status,
"needs_review": le.needs_review,
}
for le in (r.languages or [])
] if r.languages else [],
}
for r in records
],
@@ -487,7 +691,7 @@ class TranslationPreview:
# region get_preview_session [TYPE Function]
# @PURPOSE: Get the latest preview session for a job with its records.
# @PRE: job_id exists.
# @POST: Returns session data with records or raises ValueError.
# @POST: Returns session data with per-language records or raises ValueError.
def get_preview_session(self, job_id: str) -> dict[str, Any]:
with belief_scope("TranslationPreview.get_preview_session"):
session = (
@@ -505,6 +709,12 @@ class TranslationPreview:
.all()
)
# Resolve target languages
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"] if job else ["en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
return {
"id": session.id,
"job_id": job_id,
@@ -512,6 +722,7 @@ class TranslationPreview:
"created_by": session.created_by,
"created_at": session.created_at.isoformat(),
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
"target_languages": target_languages,
"records": [
{
"id": r.id,
@@ -522,6 +733,28 @@ class TranslationPreview:
"source_object_name": r.source_object_name,
"status": r.status,
"feedback": r.feedback,
"source_language_detected": (
r.languages[0].source_language_detected
if r.languages and r.languages[0].source_language_detected
else None
),
"needs_review": (
r.languages[0].needs_review
if r.languages
else False
),
"languages": [
{
"language_code": le.language_code,
"source_language_detected": le.source_language_detected,
"translated_value": le.translated_value,
"final_value": le.final_value,
"user_edit": le.user_edit,
"status": le.status,
"needs_review": le.needs_review,
}
for le in (r.languages or [])
] if r.languages else [],
}
for r in records
],
@@ -752,11 +985,11 @@ class TranslationPreview:
# endregion _call_openai_compatible
# region _parse_llm_response [TYPE Function]
# @PURPOSE: Parse the LLM JSON response into a dict of row_id -> translation.
# @PURPOSE: Parse the LLM JSON response into a dict of row_id -> per-language translations.
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
# @POST: Returns dict mapping string row_id to translation text.
# @POST: Returns dict mapping row_id to dict with 'detected_source_language' and per-language keys.
@staticmethod
def _parse_llm_response(response_text: str, expected_count: int) -> dict[str, str]:
def _parse_llm_response(response_text: str, expected_count: int, target_languages: list[str] | None = None) -> dict[str, dict[str, str]]:
with belief_scope("TranslationPreview._parse_llm_response"):
logger.reason(f"Raw LLM response length={len(response_text)} preview={response_text[:500]}")
@@ -779,12 +1012,37 @@ class TranslationPreview:
logger.explore(f"LLM response has no 'rows' array, keys={list(data.keys())} text_preview={response_text[:300]}")
raise ValueError("LLM response missing 'rows' array")
translations: dict[str, str] = {}
translations: dict[str, dict[str, str]] = {}
for item in rows:
row_id = str(item.get("row_id", ""))
translation = str(item.get("translation", ""))
if row_id:
translations[row_id] = translation
if not row_id:
continue
detected_lang = str(item.get("detected_source_language", "und")) if item.get("detected_source_language") else "und"
result: dict[str, str] = {
"detected_source_language": detected_lang,
}
# Multi-language format: extract per-language keys
# Keys that are NOT row_id, detected_source_language, or translation are language codes
has_language_data = False
if target_languages:
for lang_code in target_languages:
lang_val = item.get(lang_code)
if lang_val is not None and str(lang_val).strip():
result[lang_code] = str(lang_val)
has_language_data = True
# Fallback to old format (single "translation" key)
if not has_language_data:
translation = item.get("translation")
if translation is not None:
result["translation"] = str(translation)
else:
# Skip rows with no translation data at all
continue
translations[row_id] = result
if len(translations) < expected_count:
logger.explore(

View File

@@ -0,0 +1,152 @@
# #region ContextAwarePromptBuilder [C:2] [TYPE Module] [SEMANTICS translate, prompt, context, dictionary]
# @BRIEF Pure-function prompt builder that enhances dictionary entries with context annotations.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [DictionaryEntry:Class]
# @RATIONALE: Pure functions only — no I/O, no DB access. Separated from executor for testability.
# @REJECTED: Embedding context inline in the executor would make it untestable without mocking DB.
#
# Typical workflow:
# 1. Call build_context_entries(dictionary_entries, row_context) to get annotated, prioritized entries
# 2. Each entry is rendered via render_entry() with optional priority flag
# 3. Jaccard similarity >= 0.5 triggers priority flagging
import json
from typing import Any
# #region ContextAwarePromptBuilder [C:2] [TYPE Class]
# @BRIEF Build LLM prompts with context-aware dictionary entries and similarity-based priority.
class ContextAwarePromptBuilder:
"""Build LLM prompts with context-aware dictionary entries.
Pure function — no I/O, no DB access.
"""
@staticmethod
def render_entry(entry: Any, priority: bool = False, row_context: dict | None = None) -> str:
"""Render a dictionary entry for the LLM prompt.
Args:
entry: DictionaryEntry-like object (must have source_term, target_term, has_context,
context_data, usage_notes attrs or dict keys).
priority: Whether this entry should be flagged as high priority.
row_context: Optional row context dict (unused in rendering, kept for API symmetry).
Returns:
Rendered prompt line string.
"""
# Support both object attrs and dict access
if isinstance(entry, dict):
source_term = entry.get("source_term", "")
target_term = entry.get("target_term", "")
has_context = entry.get("has_context", False)
context_data = entry.get("context_data")
usage_notes = entry.get("usage_notes")
else:
source_term = getattr(entry, "source_term", "")
target_term = getattr(entry, "target_term", "")
has_context = getattr(entry, "has_context", False)
context_data = getattr(entry, "context_data", None)
usage_notes = getattr(entry, "usage_notes", None)
# Build base line
line = f'"{source_term}" -> "{target_term}"'
# Add context annotation if present
if has_context and context_data:
context_items = []
if isinstance(context_data, dict):
context_items = [f"{k}={v}" for k, v in context_data.items()]
elif isinstance(context_data, str):
try:
parsed = json.loads(context_data)
if isinstance(parsed, dict):
context_items = [f"{k}={v}" for k, v in parsed.items()]
else:
context_items = [str(context_data)]
except (json.JSONDecodeError, TypeError):
context_items = [str(context_data)]
context_str = ", ".join(context_items)
# Truncate if too long (500 tokens ≈ 2000 chars)
if len(context_str) > 2000:
context_str = context_str[:1997] + "...[truncated]"
line = f'"{source_term}" (context: {context_str}) -> "{target_term}"'
# Add usage notes
if usage_notes:
notes = str(usage_notes)[:200] # cap at 200 chars
line += f" # Usage: {notes}"
# Add priority prefix
if priority:
line = f"# PRIORITY (context match) — {line}"
return line
@staticmethod
def compute_context_similarity(entry_context: dict | None, row_context: dict | None) -> float:
"""Jaccard similarity between entry context and row context. Returns 0.0-1.0.
Compares the sets of lowercased string values from both contexts.
Returns 1.0 for identical contexts, 0.0 for disjoint or missing.
"""
if not entry_context or not row_context:
return 0.0
entry_vals = set(str(v).lower() for v in entry_context.values() if v is not None)
row_vals = set(str(v).lower() for v in row_context.values() if v is not None)
if not entry_vals or not row_vals:
return 0.0
intersection = entry_vals & row_vals
union = entry_vals | row_vals
return len(intersection) / len(union)
@staticmethod
def build_context_entries(
dictionary_entries: list[Any],
row_context: dict | None = None,
) -> list[str]:
"""Build prioritized dictionary entry list with context annotations.
Args:
dictionary_entries: List of DictionaryEntry-like objects or dicts.
row_context: Optional dict of current row's context columns.
Returns:
List of rendered prompt strings, sorted with priority entries first.
"""
results: list[tuple[Any, bool]] = []
for entry in dictionary_entries:
priority = False
if row_context:
# Extract entry context_data
if isinstance(entry, dict):
entry_context = entry.get("context_data")
else:
entry_context = getattr(entry, "context_data", None)
if entry_context:
similarity = ContextAwarePromptBuilder.compute_context_similarity(
entry_context, row_context
)
priority = similarity >= 0.5
results.append((entry, priority))
# Sort: priority first, then non-priority
results.sort(key=lambda x: (not x[1]))
return [
ContextAwarePromptBuilder.render_entry(entry, priority, row_context)
for entry, priority in results
]
# #endregion ContextAwarePromptBuilder
# #endregion ContextAwarePromptBuilder

View File

@@ -10,6 +10,7 @@
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity.
import re
import uuid
from datetime import UTC, datetime
from typing import Any
@@ -27,6 +28,7 @@ from ...schemas.translate import (
TranslateJobResponse,
TranslateJobUpdate,
)
from .dictionary import _validate_bcp47
# Supported database dialects for translation
SUPPORTED_DIALECTS = {
@@ -227,6 +229,16 @@ class TranslateJobService:
logger.warning(f"[TranslateJobService] Dialect detection failed: {e}")
dialect = payload.source_dialect
# Resolve target_languages: accept new format (list) or legacy single language
target_languages = payload.target_languages
if not target_languages and payload.target_language:
target_languages = [payload.target_language]
if target_languages:
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
for lang in target_languages:
_validate_bcp47(lang, "target_languages")
# Build job instance
job = TranslationJob(
id=str(uuid.uuid4()),
@@ -245,6 +257,7 @@ class TranslateJobService:
target_column=payload.target_column,
context_columns=payload.context_columns or [],
target_language=payload.target_language,
target_languages=target_languages,
provider_id=payload.provider_id,
batch_size=payload.batch_size,
upsert_strategy=payload.upsert_strategy,
@@ -284,6 +297,22 @@ class TranslateJobService:
update_data = payload.model_dump(exclude_unset=True)
dict_ids = update_data.pop("dictionary_ids", None)
# Backward compat: if only target_language is set (old API), wrap into target_languages
if "target_language" in update_data and "target_languages" not in update_data:
target_languages = [update_data["target_language"]] if update_data["target_language"] else []
update_data["target_languages"] = target_languages
elif "target_languages" in update_data and "target_language" not in update_data:
# Keep deprecated field in sync
if update_data["target_languages"]:
update_data["target_language"] = update_data["target_languages"][0]
# Validate BCP-47 for target_languages
if update_data.get("target_languages"):
if not isinstance(update_data["target_languages"], list):
update_data["target_languages"] = [str(update_data["target_languages"])]
for lang in update_data["target_languages"]:
_validate_bcp47(lang, "target_languages")
for field, value in update_data.items():
if hasattr(job, field):
setattr(job, field, value)
@@ -366,6 +395,7 @@ class TranslateJobService:
target_column=source.target_column,
context_columns=source.context_columns,
target_language=source.target_language,
target_languages=source.target_languages,
provider_id=source.provider_id,
batch_size=source.batch_size,
upsert_strategy=source.upsert_strategy,
@@ -468,6 +498,7 @@ def job_to_response(job: TranslationJob, dict_ids: list[str] | None = None) -> T
target_column=job.target_column,
context_columns=job.context_columns or [],
target_language=job.target_language,
target_languages=job.target_languages,
provider_id=job.provider_id,
batch_size=job.batch_size or 50,
upsert_strategy=job.upsert_strategy or "MERGE",
@@ -546,5 +577,462 @@ def get_datasource_columns(
)
# #endregion DatasourceColumnsService
# #region InlineCorrectionService [C:3] [TYPE Class] [SEMANTICS translate, correction, inline, dictionary]
# @BRIEF Service for inline editing translated values and submitting corrections to dictionaries.
class InlineCorrectionService:
"""Handle inline correction of translated values with optional dictionary submission."""
@staticmethod
def apply_inline_edit(
db: Session,
run_id: str,
record_id: str,
language_code: str,
final_value: str,
submit_to_dictionary: bool = False,
dictionary_id: str | None = None,
current_user: str | None = None,
context_data_override: dict | None = None,
usage_notes: str | None = None,
keep_context: bool = True,
) -> dict:
"""Apply an inline edit to a TranslationLanguage entry.
Updates final_value and user_edit fields. Optionally submits to dictionary.
Returns the updated TranslationLanguage data as a dict.
Args:
context_data_override: If provided, overrides auto-captured context data.
usage_notes: Usage notes for the dictionary entry.
keep_context: If False, clear context on the dictionary entry.
"""
from ...models.translate import TranslationLanguage, TranslationRecord
# Find the language entry
lang_entry = (
db.query(TranslationLanguage)
.filter(
TranslationLanguage.record_id == record_id,
TranslationLanguage.language_code == language_code,
)
.first()
)
if not lang_entry:
raise ValueError(f"Language entry '{language_code}' not found for record '{record_id}'")
# Verify the record belongs to the run
record = (
db.query(TranslationRecord)
.filter(
TranslationRecord.id == record_id,
TranslationRecord.run_id == run_id,
)
.first()
)
if not record:
raise ValueError(
f"Translation record '{record_id}' not found in run '{run_id}'"
)
# Apply the edit
lang_entry.final_value = final_value
lang_entry.user_edit = final_value
if lang_entry.status in ("pending", "translated"):
lang_entry.status = "edited"
db.flush()
# Optionally submit to dictionary
dict_result = None
if submit_to_dictionary and dictionary_id:
try:
dict_result = InlineCorrectionService.submit_correction_to_dict(
db=db,
record_id=record_id,
language_code=language_code,
dictionary_id=dictionary_id,
corrected_value=final_value,
current_user=current_user,
context_data_override=context_data_override,
usage_notes=usage_notes,
keep_context=keep_context,
)
except Exception as e:
# Dictionary submission failure should not block the edit
dict_result = {"error": str(e), "action": "failed"}
db.commit()
db.refresh(lang_entry)
from ...schemas.translate import TranslationLanguageResponse
response = TranslationLanguageResponse.model_validate(lang_entry)
result = response.model_dump()
if dict_result:
result["dictionary_result"] = dict_result
return result
@staticmethod
def submit_correction_to_dict(
db: Session,
record_id: str,
language_code: str,
dictionary_id: str,
corrected_value: str,
current_user: str | None = None,
context_data_override: dict | None = None,
usage_notes: str | None = None,
keep_context: bool = True,
) -> dict:
"""Submit a correction from an inline edit to the dictionary.
Fetches the TranslationLanguage entry, auto-captures source text and context,
and creates/updates a DictionaryEntry with correct language pair.
Returns conflict info if an entry already exists.
Args:
context_data_override: If provided, overrides auto-captured context_data.
usage_notes: Optional usage notes to store on the dictionary entry.
keep_context: If False, sets has_context=False and clears context_data.
"""
from ...core.logger import belief_scope, logger
from ...models.translate import DictionaryEntry, TranslationLanguage, TranslationRecord
from ._utils import _normalize_term
with belief_scope("InlineCorrectionService.submit_correction_to_dict"):
# Find the language entry
lang_entry = (
db.query(TranslationLanguage)
.filter(
TranslationLanguage.record_id == record_id,
TranslationLanguage.language_code == language_code,
)
.first()
)
if not lang_entry:
raise ValueError(f"Language entry '{language_code}' not found for record '{record_id}'")
# Find the record for source text and context
record = (
db.query(TranslationRecord)
.filter(TranslationRecord.id == record_id)
.first()
)
if not record:
raise ValueError(f"Translation record '{record_id}' not found")
# Get source text from the record
source_term = record.source_sql or record.source_object_name or ""
if not source_term:
raise ValueError("No source text available for this record")
# Determine language pair
source_language = lang_entry.source_language_detected or "und"
target_language = language_code
# Prepare context data — auto-capture from source row
context_data = {}
if record.source_data:
context_data["source_data"] = record.source_data
if record.source_object_type:
context_data["source_object_type"] = record.source_object_type
if record.source_object_name:
context_data["source_object_name"] = record.source_object_name
if record.source_object_id:
context_data["source_object_id"] = record.source_object_id
# Apply context_data_override if provided (user edited)
if context_data_override is not None:
context_data = context_data_override
context_source = "manual"
else:
context_source = "auto"
# Handle keep_context=False (user explicitly removed context)
if not keep_context:
context_data = None
context_source = "manual"
# Normalize source term
normalized = _normalize_term(source_term)
# Check for existing entry with same (dictionary_id, source_term_norm, source_language, target_language)
existing = (
db.query(DictionaryEntry)
.filter(
DictionaryEntry.dictionary_id == dictionary_id,
DictionaryEntry.source_term_normalized == normalized,
DictionaryEntry.source_language == source_language,
DictionaryEntry.target_language == target_language,
)
.first()
)
result = {
"action": "created",
"entry_id": None,
"conflict": None,
"message": None,
}
if existing:
# Conflict: return conflict info
result["action"] = "conflict_detected"
result["conflict"] = {
"source_term": source_term,
"existing_target_term": existing.target_term,
"submitted_target_term": corrected_value,
"action": "keep_existing",
}
result["message"] = (
f"Existing entry found: '{existing.target_term}' "
f"for source '{source_term}' ({source_language}{target_language})"
)
logger.reason("Correction conflict detected", result)
return result
# Create new entry
entry = DictionaryEntry(
dictionary_id=dictionary_id,
source_term=source_term.strip(),
source_term_normalized=normalized,
target_term=corrected_value.strip(),
source_language=source_language,
target_language=target_language,
context_data=context_data if context_data else None,
context_notes=None,
has_context=bool(context_data) and keep_context,
context_source=context_source,
usage_notes=usage_notes,
origin_source_language=source_language,
origin_run_id=record.run_id,
origin_row_key=record.id,
origin_user_id=current_user,
)
db.add(entry)
db.flush()
result["entry_id"] = entry.id
result["message"] = (
f"Entry created for '{source_term}' ({source_language}) → "
f"'{corrected_value}' ({target_language})"
)
logger.reflect("Dictionary entry created from correction", result)
return result
# #endregion InlineCorrectionService
# #region BulkFindReplaceService [C:3] [TYPE Class] [SEMANTICS translate, bulk, find, replace, regex]
# @BRIEF Service for bulk find-and-replace operations on translated values.
class BulkFindReplaceService:
"""Handle bulk find-and-replace on TranslationLanguage entries."""
@staticmethod
def _compile_pattern(pattern: str, is_regex: bool) -> re.Pattern:
"""Compile a search pattern (regex or plain text)."""
import re
if is_regex:
return re.compile(pattern)
return re.compile(re.escape(pattern))
@staticmethod
def _find_matching_entries(
db: Session,
run_id: str,
pattern: str,
is_regex: bool,
target_language: str,
) -> list[Any]:
"""Find all TranslationLanguage entries matching the pattern."""
from ...models.translate import TranslationLanguage, TranslationRecord
# We scan entries for the target language in this run
entries = (
db.query(TranslationLanguage)
.join(
TranslationRecord,
TranslationRecord.id == TranslationLanguage.record_id,
)
.filter(
TranslationRecord.run_id == run_id,
TranslationLanguage.language_code == target_language,
)
.all()
)
return entries
@staticmethod
def preview(
db: Session,
run_id: str,
pattern: str,
is_regex: bool,
target_language: str,
) -> list[dict]:
"""Scan TranslationLanguage entries and return matching items without applying changes."""
from ...models.translate import TranslationRecord
compiled = BulkFindReplaceService._compile_pattern(pattern, is_regex)
entries = BulkFindReplaceService._find_matching_entries(
db, run_id, pattern, is_regex, target_language
)
preview_items = []
for entry in entries:
current_value = entry.final_value or entry.translated_value or ""
if compiled.search(current_value):
# Get source term for context
record = (
db.query(TranslationRecord)
.filter(TranslationRecord.id == entry.record_id)
.first()
)
source_term = record.source_sql if record else ""
new_value = compiled.sub(
BulkFindReplaceService._get_replacement(pattern, is_regex),
current_value,
)
if new_value != current_value:
preview_items.append({
"record_id": entry.record_id,
"language_code": entry.language_code,
"source_term": source_term or "",
"current_value": current_value,
"new_value": new_value,
"source_language_detected": entry.source_language_detected,
})
return preview_items
@staticmethod
def _get_replacement(pattern: str, is_regex: bool) -> str:
"""Return replacement pattern (for regex, use backrefs; for plain, use literal)."""
# The replacement_text is passed through at apply time
return pattern # Placeholder — actual replacement is done in apply()
@staticmethod
def apply(
db: Session,
run_id: str,
pattern: str,
is_regex: bool,
replacement_text: str,
target_language: str,
submit_to_dictionary: bool = False,
dictionary_id: str | None = None,
usage_notes: str | None = None,
current_user: str | None = None,
submit_to_dictionary_with_context: bool = False,
) -> dict:
"""Apply find-and-replace to matching TranslationLanguage entries.
Returns dict with rows_affected, corrections_submitted, and preview list.
Args:
submit_to_dictionary_with_context: If True, auto-capture source row context
when submitting to dictionary.
"""
from ...core.logger import belief_scope, logger
from ...models.translate import DictionaryEntry, TranslationRecord
with belief_scope("BulkFindReplaceService.apply"):
compiled = BulkFindReplaceService._compile_pattern(pattern, is_regex)
entries = BulkFindReplaceService._find_matching_entries(
db, run_id, pattern, is_regex, target_language
)
preview_items = []
rows_affected = 0
corrections_submitted = 0
# Track unique (source_term, replacement) -> first record for context capture
seen_term_replacements: dict[str, str] = {}
for entry in entries:
current_value = entry.final_value or entry.translated_value or ""
if compiled.search(current_value):
new_value = compiled.sub(replacement_text, current_value)
if new_value != current_value:
# Apply the replacement
entry.final_value = new_value
entry.user_edit = new_value
if entry.status in ("pending", "translated"):
entry.status = "edited"
rows_affected += 1
# Get source term for preview
record = (
db.query(TranslationRecord)
.filter(TranslationRecord.id == entry.record_id)
.first()
)
source_term = record.source_sql if record else ""
# Optionally submit to dictionary
if submit_to_dictionary and dictionary_id:
try:
dict_result = InlineCorrectionService.submit_correction_to_dict(
db=db,
record_id=entry.record_id,
language_code=entry.language_code,
dictionary_id=dictionary_id,
corrected_value=new_value,
current_user=current_user,
)
if dict_result.get("action") in ("created", "updated"):
corrections_submitted += 1
# Track context for unique (source_term, replacement) pairs
if submit_to_dictionary_with_context and record:
term_key = f"{source_term}::{new_value}"
if term_key not in seen_term_replacements:
seen_term_replacements[term_key] = entry.record_id
# Auto-capture context from this row
context_data = {}
if record.source_data:
context_data["source_data"] = record.source_data
if record.source_object_type:
context_data["source_object_type"] = record.source_object_type
if record.source_object_name:
context_data["source_object_name"] = record.source_object_name
if record.source_object_id:
context_data["source_object_id"] = record.source_object_id
# Update the just-created entry's context
if dict_result.get("action") == "created" and dict_result.get("entry_id"):
d_entry = (
db.query(DictionaryEntry)
.filter(DictionaryEntry.id == dict_result["entry_id"])
.first()
)
if d_entry:
d_entry.context_data = context_data if context_data else None
d_entry.has_context = bool(context_data)
d_entry.context_source = "bulk"
except Exception as e:
logger.explore(
"Bulk replace: dictionary submission failed",
extra={"record_id": entry.record_id, "error": str(e)},
)
preview_items.append({
"record_id": entry.record_id,
"language_code": entry.language_code,
"source_term": source_term or "",
"current_value": current_value,
"new_value": new_value,
"source_language_detected": entry.source_language_detected,
})
db.commit()
result = {
"rows_affected": rows_affected,
"corrections_submitted": corrections_submitted,
"preview": preview_items,
}
logger.reason("Bulk replace completed", result)
return result
# #endregion BulkFindReplaceService
# #endregion TranslateJobService
# #endregion TranslateJobService