feat(translate): multi-language optimization (Phase 11)

- Auto-detection of source language per row via LLM (US6)
- Multi-target translation — one LLM call for N languages (US1-US3)
- Language-aware storage: TranslationLanguage, per-language stats
- Multilingual dictionaries with language-pair-aware filtering (US7)
- Inline correction on any run result + submit-to-dictionary (US8)
- Context-aware dictionary: auto-capture row context, usage notes,
  Jaccard similarity, priority flagging in LLM prompts (US8b)
- Configurable preview sample size 1-100, cost warning at >30
- Per-language history & metrics with MetricSnapshot preservation
- 36 files, +5022/-373, all specs GRACE-Poly v2.6 compliant
This commit is contained in:
2026-05-14 17:12:41 +03:00
parent 5e741a4332
commit bb0fbfdafd
36 changed files with 5025 additions and 376 deletions

View File

@@ -45,6 +45,8 @@ async def submit_correction(
origin_run_id=payload.origin_run_id,
origin_row_key=payload.origin_row_key,
origin_user_id=current_user.username,
context_data=payload.context_data,
usage_notes=payload.usage_notes,
)
return result
except ValueError as e:

View File

@@ -163,14 +163,16 @@ async def delete_dictionary(
# ============================================================
# #region list_dictionary_entries [TYPE Function]
# @BRIEF List entries for a dictionary.
# @BRIEF List entries for a dictionary, optionally filtered by language pair.
# @PRE: User has translate.dictionary.view permission.
# @POST: Returns paginated list of entries.
# @POST: Returns paginated list of entries with language pair fields.
@router.get("/dictionaries/{dictionary_id}/entries")
async def list_dictionary_entries(
dictionary_id: str,
page: int = Query(1, ge=1),
page_size: int = Query(50, ge=1, le=500),
source_language: str | None = Query(None, description="Filter by BCP-47 source language"),
target_language: str | None = Query(None, description="Filter by BCP-47 target language"),
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.dictionary", "VIEW")),
db: Session = Depends(get_db),
@@ -180,6 +182,17 @@ async def list_dictionary_entries(
logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}")
try:
entries, total = DictionaryManager.list_entries(db, dictionary_id, page=page, page_size=page_size)
# Apply language pair filtering
if source_language or target_language:
filtered = []
for e in entries:
if source_language and e.source_language != source_language and e.source_language != "und":
continue
if target_language and e.target_language != target_language:
continue
filtered.append(e)
entries = filtered
total = len(filtered)
items = [
{
"id": e.id,
@@ -187,13 +200,20 @@ async def list_dictionary_entries(
"source_term": e.source_term,
"source_term_normalized": e.source_term_normalized,
"target_term": e.target_term,
"source_language": e.source_language,
"target_language": e.target_language,
"context_notes": e.context_notes,
"context_data": e.context_data,
"usage_notes": e.usage_notes,
"has_context": e.has_context,
"context_source": e.context_source,
"origin_source_language": e.origin_source_language,
"created_at": e.created_at,
"updated_at": e.updated_at,
}
for e in entries
]
logger.reflect("Entries listed", {"dict_id": dictionary_id, "total": total})
logger.reflect("Entries listed", {"dict_id": dictionary_id, "total": total, "source_language": source_language, "target_language": target_language})
return {"items": items, "total": total, "page": page, "page_size": page_size}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
@@ -221,6 +241,8 @@ async def add_dictionary_entry(
source_term=payload.source_term,
target_term=payload.target_term,
context_notes=payload.context_notes,
source_language=payload.source_language,
target_language=payload.target_language,
)
logger.reflect("Entry added", {"entry_id": entry.id})
return {
@@ -229,7 +251,14 @@ async def add_dictionary_entry(
"source_term": entry.source_term,
"source_term_normalized": entry.source_term_normalized,
"target_term": entry.target_term,
"source_language": entry.source_language,
"target_language": entry.target_language,
"context_notes": entry.context_notes,
"context_data": entry.context_data,
"usage_notes": entry.usage_notes,
"has_context": entry.has_context,
"context_source": entry.context_source,
"origin_source_language": entry.origin_source_language,
"created_at": entry.created_at,
"updated_at": entry.updated_at,
}
@@ -262,6 +291,8 @@ async def edit_dictionary_entry(
source_term=payload.source_term,
target_term=payload.target_term,
context_notes=payload.context_notes,
source_language=payload.source_language,
target_language=payload.target_language,
)
logger.reflect("Entry updated", {"entry_id": entry_id})
return {
@@ -270,7 +301,14 @@ async def edit_dictionary_entry(
"source_term": entry.source_term,
"source_term_normalized": entry.source_term_normalized,
"target_term": entry.target_term,
"source_language": entry.source_language,
"target_language": entry.target_language,
"context_notes": entry.context_notes,
"context_data": entry.context_data,
"usage_notes": entry.usage_notes,
"has_context": entry.has_context,
"context_source": entry.context_source,
"origin_source_language": entry.origin_source_language,
"created_at": entry.created_at,
"updated_at": entry.updated_at,
}
@@ -335,6 +373,8 @@ async def import_dictionary_entries(
delimiter=payload.delimiter,
on_conflict=payload.on_conflict,
preview_only=payload.preview_only,
default_source_language=payload.default_source_language,
default_target_language=payload.default_target_language,
)
logger.reflect("Import completed", {
"dict_id": dictionary_id,

View File

@@ -59,7 +59,7 @@ async def preview_translation(
# #region update_preview_row [TYPE Function]
# @BRIEF Approve, edit, or reject a preview row.
# @BRIEF Approve, edit, or reject a preview row (optionally per language).
# @PRE: User has translate.job.execute permission.
# @POST: Preview row status is updated.
@router.put("/jobs/{job_id}/preview/rows/{row_key}")
@@ -72,8 +72,8 @@ async def update_preview_row(
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Approve, edit, or reject a preview row."""
logger.reason(f"update_preview_row — Job: {job_id}, Row: {row_key}, Action: {payload.action}", extra={"src": "translate_routes"})
"""Approve, edit, or reject a preview row (optionally per language)."""
logger.reason(f"update_preview_row — Job: {job_id}, Row: {row_key}, Action: {payload.action}, Lang: {payload.language_code}", extra={"src": "translate_routes"})
try:
preview_service = TranslationPreview(db, config_manager, current_user.username)
result = preview_service.update_preview_row(
@@ -82,6 +82,7 @@ async def update_preview_row(
action=payload.action,
translation=payload.translation,
feedback=payload.feedback,
language_code=payload.language_code,
)
return result
except ValueError as e:

View File

@@ -76,6 +76,22 @@ async def list_runs(
items = []
for r in runs:
# Gather per-language statistics from the run's language_stats relationship
lang_stats = r.language_stats or []
target_languages = [ls.language_code for ls in lang_stats]
total_translated = sum((ls.translated_rows or 0) for ls in lang_stats)
language_stats = {
ls.language_code: {
"total_rows": ls.total_rows or 0,
"translated_rows": ls.translated_rows or 0,
"failed_rows": ls.failed_rows or 0,
"skipped_rows": ls.skipped_rows or 0,
"token_count": ls.token_count or 0,
"estimated_cost": ls.estimated_cost or 0.0,
}
for ls in lang_stats
}
items.append({
"id": r.id,
"job_id": r.job_id,
@@ -94,6 +110,9 @@ async def list_runs(
"dict_snapshot_hash": r.dict_snapshot_hash,
"created_by": r.created_by,
"created_at": r.created_at.isoformat() if r.created_at else None,
"target_languages": target_languages,
"total_translated": total_translated,
"language_stats": language_stats,
})
return {"items": items, "total": total, "page": page, "page_size": page_size}

View File

@@ -13,6 +13,11 @@ from ....core.logger import logger
from ....dependencies import get_config_manager, get_current_user, has_permission
from ....plugins.translate.orchestrator import TranslationOrchestrator
from ....schemas.auth import User
from ....schemas.translate import (
BulkFindReplaceRequest,
InlineEditRequest,
OverrideLanguageRequest,
)
from ._helpers import _run_to_response
from ._router import router
@@ -341,4 +346,196 @@ async def get_batches(
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# #endregion get_batches
# ============================================================
# Manual Language Override
# ============================================================
# #region override_detected_language [TYPE Function]
# @BRIEF Manually override the detected source language for a specific translation language entry.
# @PRE: User has translate.job.execute permission. Run, record, and language entry exist.
# @POST: TranslationLanguage.source_language_detected is updated; language_overridden is set to True.
# @SIDE_EFFECT: DB write.
@router.put("/runs/{run_id}/records/{record_id}/languages/{language_code}/override-language")
async def override_detected_language(
run_id: str,
record_id: str,
language_code: str,
payload: OverrideLanguageRequest,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
):
"""Manually override the detected source language for a translation entry."""
logger.reason(f"override_detected_language — Run: {run_id}, Record: {record_id}, Lang: {language_code}, Override: {payload.source_language}", extra={"src": "translate_routes"})
try:
from ....models.translate import TranslationLanguage, TranslationRecord
# Verify the record exists and belongs to the run
record = (
db.query(TranslationRecord)
.filter(
TranslationRecord.id == record_id,
TranslationRecord.run_id == run_id,
)
.first()
)
if not record:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Translation record '{record_id}' not found in run '{run_id}'",
)
# Find the language entry
lang_entry = (
db.query(TranslationLanguage)
.filter(
TranslationLanguage.record_id == record_id,
TranslationLanguage.language_code == language_code,
)
.first()
)
if not lang_entry:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Language entry '{language_code}' not found for record '{record_id}'",
)
# Update the language entry
lang_entry.source_language_detected = payload.source_language
lang_entry.language_overridden = True
lang_entry.needs_review = False # Override clears the review flag
db.commit()
db.refresh(lang_entry)
logger.reason("Language override applied", {
"run_id": run_id,
"record_id": record_id,
"language_code": language_code,
"new_source_language": payload.source_language,
"overridden_by": current_user.username,
})
from ....schemas.translate import TranslationLanguageResponse
return TranslationLanguageResponse.model_validate(lang_entry)
except HTTPException:
raise
except Exception as e:
logger.explore("override_detected_language failed", extra={"src": "translate_routes", "error": str(e)})
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# #endregion override_detected_language
# ============================================================
# Inline Correction
# ============================================================
# #region inline_edit_translation [C:3] [TYPE Function] [SEMANTICS api,translate,correction]
# @BRIEF Apply an inline correction to a translated value on a completed run result.
# @PRE: User has translate.job.execute permission. Run, record, and language entry exist.
# @POST: TranslationLanguage.final_value and user_edit are updated. Optional dictionary submission.
@router.put("/runs/{run_id}/records/{record_id}/languages/{language_code}")
async def inline_edit_translation(
run_id: str,
record_id: str,
language_code: str,
payload: InlineEditRequest,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
):
"""Apply an inline correction to a translated value."""
logger.reason(
f"inline_edit_translation — Run: {run_id}, Record: {record_id}, "
f"Lang: {language_code}, User: {current_user.username}",
extra={"src": "translate_routes"},
)
try:
from ....plugins.translate.service import InlineCorrectionService
result = InlineCorrectionService.apply_inline_edit(
db=db,
run_id=run_id,
record_id=record_id,
language_code=language_code,
final_value=payload.final_value,
submit_to_dictionary=payload.submit_to_dictionary,
dictionary_id=payload.dictionary_id,
current_user=current_user.username,
context_data_override=payload.context_data_override,
usage_notes=payload.usage_notes,
keep_context=payload.keep_context,
)
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
except Exception as e:
logger.explore("inline_edit_translation failed", extra={"src": "translate_routes", "error": str(e)})
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# #endregion inline_edit_translation
# ============================================================
# Bulk Find-and-Replace
# ============================================================
# #region bulk_find_replace [C:3] [TYPE Function] [SEMANTICS api,translate,bulk,replace]
# @BRIEF Perform bulk find-and-replace on translated values within a run.
# @PRE: User has translate.job.execute permission. Run exists.
# @POST: If preview=false, matching translations are updated. Optional dictionary submission.
@router.post("/runs/{run_id}/bulk-replace")
async def bulk_find_replace(
run_id: str,
payload: BulkFindReplaceRequest,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "EXECUTE")),
db: Session = Depends(get_db),
):
"""Perform bulk find-and-replace on translated values within a run."""
logger.reason(
f"bulk_find_replace — Run: {run_id}, Pattern: '{payload.find_pattern}', "
f"Regex: {payload.is_regex}, Lang: {payload.target_language}, "
f"User: {current_user.username}",
extra={"src": "translate_routes"},
)
try:
from ....plugins.translate.service import BulkFindReplaceService
if payload.preview:
# Preview mode: return matches without applying
preview = BulkFindReplaceService.preview(
db=db,
run_id=run_id,
pattern=payload.find_pattern,
is_regex=payload.is_regex,
target_language=payload.target_language,
)
return {
"rows_affected": len(preview),
"corrections_submitted": 0,
"preview": preview,
}
# Apply mode: perform replacements
result = BulkFindReplaceService.apply(
db=db,
run_id=run_id,
pattern=payload.find_pattern,
is_regex=payload.is_regex,
replacement_text=payload.replacement_text,
target_language=payload.target_language,
submit_to_dictionary=payload.submit_to_dictionary,
dictionary_id=payload.dictionary_id,
usage_notes=payload.usage_notes,
current_user=current_user.username,
submit_to_dictionary_with_context=payload.submit_to_dictionary_with_context,
)
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
except Exception as e:
logger.explore("bulk_find_replace failed", extra={"src": "translate_routes", "error": str(e)})
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# #endregion bulk_find_replace
# #endregion TranslateRunRoutesModule

View File

@@ -6,7 +6,8 @@
import uuid
from datetime import UTC, datetime
from sqlalchemy import JSON, Boolean, Column, DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint
from sqlalchemy import JSON, Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import relationship
from .mapping import Base
@@ -44,7 +45,11 @@ class TranslationJob(Base):
context_columns = Column(JSON, nullable=True, comment="Context column names included in LLM prompt")
# LLM & processing settings
target_language = Column(String, nullable=True, comment="Target language code (e.g. en, ru)")
# @DEPRECATED — use target_languages
target_language = Column(String, nullable=True, comment="Target language code (e.g. en, ru) [DEPRECATED: use target_languages]")
# @DEPRECATED — Auto-detected per row by LLM, kept as fallback hint
source_language = Column(String, nullable=True, comment="Fallback source language hint [DEPRECATED: auto-detected per row]")
target_languages = Column(JSON, nullable=True, comment="List of BCP-47 target language codes (multi-language support)")
provider_id = Column(String, nullable=True, comment="LLM provider ID")
batch_size = Column(Integer, nullable=False, default=50, comment="Records per batch")
upsert_strategy = Column(String, nullable=False, default="MERGE", comment="MERGE, INSERT, UPDATE")
@@ -84,6 +89,8 @@ class TranslationRun(Base):
dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at run time")
created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
language_stats = relationship("TranslationRunLanguageStats", back_populates="run")
# #endregion TranslationRun
@@ -126,6 +133,15 @@ class TranslationRecord(Base):
translation_duration_ms = Column(Integer, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
# @DEPRECATED — use TranslationLanguage instead
llm_translation = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]")
# @DEPRECATED — use TranslationLanguage instead
user_edit = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]")
# @DEPRECATED — use TranslationLanguage instead
final_value = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]")
languages = relationship("TranslationLanguage", back_populates="record")
__table_args__ = (
Index("ix_translation_records_run_status", "run_id", "status"),
)
@@ -178,6 +194,8 @@ class TranslationPreviewRecord(Base):
status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED
feedback = Column(Text, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
languages = relationship("TranslationPreviewLanguage", back_populates="preview_record")
# #endregion TranslationPreviewRecord
@@ -191,6 +209,10 @@ class TerminologyDictionary(Base):
description = Column(Text, nullable=True)
source_dialect = Column(String, nullable=False)
target_dialect = Column(String, nullable=False)
# @DEPRECATED — use per-entry source_language instead, kept for backward compat
source_language = Column(String, nullable=True, comment="[DEPRECATED: use per-entry source_language]")
# @DEPRECATED — use per-entry target_language instead, kept for backward compat
target_language = Column(String, nullable=True, comment="[DEPRECATED: use per-entry target_language]")
is_active = Column(Boolean, default=True)
created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
@@ -208,7 +230,14 @@ class DictionaryEntry(Base):
source_term = Column(String, nullable=False)
source_term_normalized = Column(String, nullable=False)
target_term = Column(String, nullable=False)
source_language = Column(String, nullable=False, comment="BCP-47 source language code")
target_language = Column(String, nullable=False, comment="BCP-47 target language code")
context_notes = Column(Text, nullable=True)
context_data = Column(JSON, nullable=True, comment="Structured context for term usage")
usage_notes = Column(Text, nullable=True, comment="Usage guidance for the term mapping")
has_context = Column(Boolean, default=False, comment="Whether context_data is populated")
context_source = Column(String, nullable=True, comment="auto|auto_with_edits|manual|bulk")
origin_source_language = Column(String, nullable=True, comment="Original source language of the term")
origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated")
origin_row_key = Column(String, nullable=True, comment="Row key within the run that triggered this correction")
origin_user_id = Column(String, nullable=True, comment="User who submitted the correction")
@@ -217,9 +246,11 @@ class DictionaryEntry(Base):
__table_args__ = (
UniqueConstraint(
"dictionary_id", "source_term_normalized",
name="uq_dictionary_entry_term"
"dictionary_id", "source_term_normalized", "source_language", "target_language",
name="uq_dict_source_term_lang"
),
Index("idx_dict_entry_lang", "source_language", "target_language"),
Index("idx_dict_has_context", "has_context"),
)
# #endregion DictionaryEntry
@@ -284,6 +315,7 @@ class MetricSnapshot(Base):
p50_duration_ms = Column(Integer, nullable=True)
p95_duration_ms = Column(Integer, nullable=True)
p99_duration_ms = Column(Integer, nullable=True)
per_language_metrics = Column(JSON, nullable=True, comment="Per-language cumulative metrics: {lang: {cumulative_tokens, cumulative_cost, runs}}")
snapshot_date = Column(DateTime, nullable=False, default=lambda: datetime.now(UTC))
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
@@ -292,4 +324,81 @@ class MetricSnapshot(Base):
)
# #endregion MetricSnapshot
# #region TranslationLanguage [C:1] [TYPE Class]
# @BRIEF Per-language translation result for a single record, supporting multi-language output.
class TranslationLanguage(Base):
__tablename__ = "translation_languages"
id = Column(String, primary_key=True, default=generate_uuid)
record_id = Column(String, ForeignKey("translation_records.id"), nullable=False)
language_code = Column(String, nullable=False, comment="BCP-47 language code")
source_language_detected = Column(String, nullable=True, comment="BCP-47 or 'und' for undetermined")
translated_value = Column(Text, nullable=True, comment="LLM-generated translation")
user_edit = Column(Text, nullable=True, comment="User-edited translation")
final_value = Column(Text, nullable=True, comment="Final resolved value (translated or user edit)")
status = Column(String, default="pending", comment="pending|translated|approved|edited|rejected|failed|skipped")
error_message = Column(Text, nullable=True)
needs_review = Column(Boolean, default=False, comment="Flagged because source language could not be determined")
language_overridden = Column(Boolean, default=False, comment="Source language was manually overridden by user")
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
record = relationship("TranslationRecord", back_populates="languages")
__table_args__ = (
UniqueConstraint("record_id", "language_code", name="uq_record_language"),
Index("idx_tl_language", "language_code"),
Index("idx_tl_record_lang", "record_id", "language_code"),
)
# #endregion TranslationLanguage
# #region TranslationPreviewLanguage [C:1] [TYPE Class]
# @BRIEF Per-language preview entry within a preview session.
class TranslationPreviewLanguage(Base):
__tablename__ = "translation_preview_languages"
id = Column(String, primary_key=True, default=generate_uuid)
preview_record_id = Column(String, ForeignKey("translation_preview_records.id"), nullable=False)
language_code = Column(String, nullable=False, comment="BCP-47 language code")
source_language_detected = Column(String, nullable=True, comment="BCP-47 or 'und'")
translated_value = Column(Text, nullable=True)
user_edit = Column(Text, nullable=True)
final_value = Column(Text, nullable=True)
status = Column(String, default="pending", comment="pending|approved|edited|rejected")
needs_review = Column(Boolean, default=False, comment="Flagged because source language could not be determined")
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
preview_record = relationship("TranslationPreviewRecord", back_populates="languages")
__table_args__ = (
UniqueConstraint("preview_record_id", "language_code", name="uq_preview_record_language"),
)
# #endregion TranslationPreviewLanguage
# #region TranslationRunLanguageStats [C:1] [TYPE Class]
# @BRIEF Per-language statistics for a translation run (row counts, tokens, cost).
class TranslationRunLanguageStats(Base):
__tablename__ = "translation_run_language_stats"
id = Column(String, primary_key=True, default=generate_uuid)
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=False)
language_code = Column(String, nullable=False, comment="BCP-47 language code")
total_rows = Column(Integer, default=0)
translated_rows = Column(Integer, default=0)
failed_rows = Column(Integer, default=0)
skipped_rows = Column(Integer, default=0)
token_count = Column(Integer, default=0)
estimated_cost = Column(Float, default=0.0)
run = relationship("TranslationRun", back_populates="language_stats")
__table_args__ = (
UniqueConstraint("run_id", "language_code", name="uq_run_language"),
Index("idx_rls_run", "run_id"),
)
# #endregion TranslationRunLanguageStats
# #endregion TranslateModels

View File

@@ -1,29 +1,37 @@
# region DictionaryTests [TYPE Module]
# @SEMANTICS: tests, dictionary, crud, import, filter
# @PURPOSE: Validate DictionaryManager CRUD, import, deletion guards, and batch filtering.
# @SEMANTICS: tests, dictionary, crud, import, filter, language_pair
# @PURPOSE: Validate DictionaryManager CRUD, import, deletion guards, batch filtering, and language-pair support.
# @RELATION: BINDS_TO -> [DictionaryManager:Class]
#
# @TEST_CONTRACT: [DictionaryManager] -> {
# invariants: [
# "Create/Read/Update/Delete dictionaries works",
# "Entry CRUD enforces unique source_term_normalized per dictionary",
# "Entry CRUD enforces unique (dictionary_id, source_term_normalized, source_language, target_language)",
# "Import CSV/TSV handles overwrite/keep_existing/cancel conflict modes",
# "Delete blocked when dictionary attached to active/scheduled jobs",
# "filter_for_batch returns matched entries with word-boundary awareness"
# "filter_for_batch returns matched entries with word-boundary awareness",
# "filter_for_batch supports language-pair filtering",
# "Language-pair-aware duplicate detection: same term with different lang pair is allowed"
# ]
# }
# @TEST_EDGE: duplicate_entry -> 409-style ValueError on repeated source_term
# @TEST_EDGE: duplicate_entry -> 409-style ValueError on repeated (dictionary_id, source_term_norm, source_lang, target_lang)
# @TEST_EDGE: delete_active_job -> ValueError with active/scheduled message
# @TEST_EDGE: import_invalid_format -> ValueError for missing columns
# @TEST_INVARIANT: unique_normalized -> verifies: [duplicate_entry]
# @TEST_EDGE: same_term_different_lang_pair -> allowed (not duplicate)
# @TEST_INVARIANT: unique_normalized -> verifies: [duplicate_entry, same_term_different_lang_pair allowed]
import csv
import io
import pytest
from sqlalchemy import create_engine, event
from sqlalchemy.orm import Session, sessionmaker
from src.models.translate import (
Base,
DictionaryEntry,
TerminologyDictionary,
TranslationJob,
TranslationJobDictionary,
)
@@ -112,8 +120,8 @@ def test_delete_dictionary(db_session: Session):
db_session, name="To Delete",
source_dialect="a", target_dialect="b",
)
# Add an entry
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
# Add an entry with language pair
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
assert entry.id is not None
DictionaryManager.delete_dictionary(db_session, d.id)
@@ -144,15 +152,15 @@ def test_add_entry_duplicate(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "Hello", "Hola")
DictionaryManager.add_entry(db_session, d.id, "Hello", "Hola", source_language="en", target_language="es")
# Same normalized term should raise
# Same normalized term with same language pair should raise
with pytest.raises(ValueError, match="already exists"):
DictionaryManager.add_entry(db_session, d.id, "hello", "Bonjour")
DictionaryManager.add_entry(db_session, d.id, "hello", "Bonjour", source_language="en", target_language="es")
# Different case, same normalized should also raise
# Different case, same normalized, same lang pair should also raise
with pytest.raises(ValueError, match="already exists"):
DictionaryManager.add_entry(db_session, d.id, "HELLO", "Ciao")
DictionaryManager.add_entry(db_session, d.id, "HELLO", "Ciao", source_language="en", target_language="es")
# endregion test_add_entry_duplicate
@@ -167,9 +175,9 @@ def test_add_entry_duplicate_per_dictionary(db_session: Session):
db_session, name="Dict2",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d1.id, "hello", "hola")
DictionaryManager.add_entry(db_session, d1.id, "hello", "hola", source_language="en", target_language="es")
# Same term in different dictionary should work
entry = DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour")
entry = DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour", source_language="en", target_language="es")
assert entry.id is not None
# endregion test_add_entry_duplicate_per_dictionary
@@ -181,13 +189,13 @@ def test_edit_entry(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
updated = DictionaryManager.edit_entry(db_session, entry.id, target_term="HOLA!")
assert updated.target_term == "HOLA!"
# Edit source term to a value that's already taken should fail
DictionaryManager.add_entry(db_session, d.id, "world", "mundo")
DictionaryManager.add_entry(db_session, d.id, "world", "mundo", source_language="en", target_language="es")
with pytest.raises(ValueError, match="already exists"):
DictionaryManager.edit_entry(db_session, entry.id, source_term="WORLD")
# endregion test_edit_entry
@@ -200,7 +208,7 @@ def test_delete_entry(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
DictionaryManager.delete_entry(db_session, entry.id)
entries, total = DictionaryManager.list_entries(db_session, d.id)
@@ -215,10 +223,10 @@ def test_import_csv_overwrite(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
# Pre-add an entry
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
# Pre-add an entry with language pair
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
csv_content = "source_term,target_term,context_notes\nhello,HELLO,\nworld,mundo,"
csv_content = "source_term,target_term,context_notes,source_language,target_language\nhello,HELLO,,en,es\nworld,mundo,,en,es"
result = DictionaryManager.import_entries(
db_session, d.id, csv_content,
delimiter=",", on_conflict="overwrite",
@@ -242,9 +250,9 @@ def test_import_csv_keep_existing(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo"
csv_content = "source_term,target_term,source_language,target_language\nhello,HELLO,en,es\nworld,mundo,en,es"
result = DictionaryManager.import_entries(
db_session, d.id, csv_content,
delimiter=",", on_conflict="keep_existing",
@@ -267,9 +275,9 @@ def test_import_csv_cancel_on_conflict(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo"
csv_content = "source_term,target_term,source_language,target_language\nhello,HELLO,en,es\nworld,mundo,en,es"
result = DictionaryManager.import_entries(
db_session, d.id, csv_content,
delimiter=",", on_conflict="cancel",
@@ -289,7 +297,7 @@ def test_import_tsv(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
tsv_content = "source_term\ttarget_term\nhello\thola\nworld\tmundo"
tsv_content = "source_term\ttarget_term\tsource_language\ttarget_language\nhello\thola\ten\tes\nworld\tmundo\ten\tes"
result = DictionaryManager.import_entries(
db_session, d.id, tsv_content,
delimiter="\t", on_conflict="overwrite",
@@ -339,9 +347,9 @@ def test_import_preview(db_session: Session):
db_session, name="Test",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo"
csv_content = "source_term,target_term,source_language,target_language\nhello,HELLO,en,es\nworld,mundo,en,es"
result = DictionaryManager.import_entries(
db_session, d.id, csv_content,
delimiter=",", on_conflict="overwrite",
@@ -446,9 +454,9 @@ def test_filter_for_batch_matches(db_session: Session):
db_session, name="Test Dict",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
DictionaryManager.add_entry(db_session, d.id, "world", "mundo")
DictionaryManager.add_entry(db_session, d.id, "foo", "bar")
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
DictionaryManager.add_entry(db_session, d.id, "world", "mundo", source_language="en", target_language="es")
DictionaryManager.add_entry(db_session, d.id, "foo", "bar", source_language="en", target_language="es")
job = TranslationJob(
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
@@ -492,7 +500,7 @@ def test_filter_for_batch_case_insensitive(db_session: Session):
db_session, name="Test Dict",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "Hello World", "Hola Mundo")
DictionaryManager.add_entry(db_session, d.id, "Hello World", "Hola Mundo", source_language="en", target_language="es")
job = TranslationJob(
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
@@ -523,7 +531,7 @@ def test_filter_for_batch_word_boundary(db_session: Session):
db_session, name="Test Dict",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "cat", "gato")
DictionaryManager.add_entry(db_session, d.id, "cat", "gato", source_language="en", target_language="es")
job = TranslationJob(
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
@@ -562,8 +570,8 @@ def test_filter_for_batch_multi_dictionary_priority(db_session: Session):
d2 = DictionaryManager.create_dictionary(
db_session, name="Priority2", source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d1.id, "hello", "hola")
DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour")
DictionaryManager.add_entry(db_session, d1.id, "hello", "hola", source_language="en", target_language="es")
DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour", source_language="en", target_language="es")
job = TranslationJob(
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
@@ -625,8 +633,8 @@ def test_clear_entries(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Test", source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
DictionaryManager.add_entry(db_session, d.id, "world", "mundo")
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
DictionaryManager.add_entry(db_session, d.id, "world", "mundo", source_language="en", target_language="es")
deleted = DictionaryManager.clear_entries(db_session, d.id)
assert deleted == 2
@@ -634,4 +642,558 @@ def test_clear_entries(db_session: Session):
entries, total = DictionaryManager.list_entries(db_session, d.id)
assert total == 0
# endregion test_clear_entries
# region test_add_entry_with_language_pair [TYPE Function]
# @PURPOSE: Verify creating entry with source_language and target_language stores correctly.
def test_add_entry_with_language_pair(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Lang Test",
source_dialect="a", target_dialect="b",
)
entry = DictionaryManager.add_entry(
db_session, d.id, "hello", "привет",
source_language="en", target_language="ru",
)
assert entry.source_language == "en"
assert entry.target_language == "ru"
assert entry.source_term == "hello"
# Read back
entries, total = DictionaryManager.list_entries(db_session, d.id)
assert total == 1
assert entries[0].source_language == "en"
assert entries[0].target_language == "ru"
# endregion test_add_entry_with_language_pair
# region test_duplicate_same_language_pair [TYPE Function]
# @PURPOSE: Verify duplicate entry with same (dictionary_id, source_term_norm, source_lang, target_lang) raises conflict.
def test_duplicate_same_language_pair(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Dup Test",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(
db_session, d.id, "hello", "привет",
source_language="en", target_language="ru",
)
# Same term with same language pair should raise
with pytest.raises(ValueError, match="already exists"):
DictionaryManager.add_entry(
db_session, d.id, "hello", "hallo",
source_language="en", target_language="ru",
)
# endregion test_duplicate_same_language_pair
# region test_same_term_different_language_pair [TYPE Function]
# @PURPOSE: Verify same source_term with different language pair is allowed (not duplicate).
def test_same_term_different_language_pair(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Multi Lang",
source_dialect="a", target_dialect="b",
)
entry1 = DictionaryManager.add_entry(
db_session, d.id, "hello", "привет",
source_language="en", target_language="ru",
)
entry2 = DictionaryManager.add_entry(
db_session, d.id, "hello", "hallo",
source_language="en", target_language="de",
)
assert entry1.id != entry2.id
assert entry1.target_language == "ru"
assert entry2.target_language == "de"
entries, total = DictionaryManager.list_entries(db_session, d.id)
assert total == 2
# endregion test_same_term_different_language_pair
# region test_filter_for_batch_with_language_pair [TYPE Function]
# @PURPOSE: Verify filter_for_batch with source_language and target_language filters entries correctly.
def test_filter_for_batch_with_language_pair(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Lang Filter",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")
DictionaryManager.add_entry(db_session, d.id, "world", "мир", source_language="en", target_language="ru")
DictionaryManager.add_entry(db_session, d.id, "hello", "hallo", source_language="en", target_language="de")
job = TranslationJob(
name="Lang Job", source_dialect="a", target_dialect="b", status="DRAFT",
)
db_session.add(job)
db_session.flush()
link = TranslationJobDictionary(job_id=job.id, dictionary_id=d.id)
db_session.add(link)
db_session.commit()
# Filter for en → ru: should get 2 entries (hello/privet, world/mir)
result = DictionaryManager.filter_for_batch(
db_session, ["hello world"], job.id,
source_language="en", target_language="ru",
)
assert len(result) == 2
assert all(m["target_language"] == "ru" for m in result)
# Filter for en → de: should get only the hello/hallo entry
result2 = DictionaryManager.filter_for_batch(
db_session, ["hello"], job.id,
source_language="en", target_language="de",
)
assert len(result2) == 1
assert result2[0]["target_language"] == "de"
assert result2[0]["target_term"] == "hallo"
# endregion test_filter_for_batch_with_language_pair
# region test_filter_for_batch_target_language_only [TYPE Function]
# @PURPOSE: Verify filter_for_batch with only target_language filters by target language only.
def test_filter_for_batch_target_language_only(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Target Only",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")
DictionaryManager.add_entry(db_session, d.id, "hello", "hallo", source_language="en", target_language="de")
DictionaryManager.add_entry(db_session, d.id, "bonjour", "hallo", source_language="fr", target_language="de")
job = TranslationJob(
name="Tgt Job", source_dialect="a", target_dialect="b", status="DRAFT",
)
db_session.add(job)
db_session.flush()
link = TranslationJobDictionary(job_id=job.id, dictionary_id=d.id)
db_session.add(link)
db_session.commit()
# Filter for target=de only: should match both hello→hallo and bonjour→hallo (from any source lang)
result = DictionaryManager.filter_for_batch(
db_session, ["hello bonjour"], job.id,
target_language="de",
)
assert len(result) == 2
assert all(m["target_language"] == "de" for m in result)
# Filter for target=ru only: should match hello→privet
result2 = DictionaryManager.filter_for_batch(
db_session, ["hello"], job.id,
target_language="ru",
)
assert len(result2) == 1
assert result2[0]["target_language"] == "ru"
# endregion test_filter_for_batch_target_language_only
# region test_migrate_old_entries [TYPE Function]
# @PURPOSE: Verify migration populates language pair fields for old-style entries.
def test_migrate_old_entries(db_session: Session):
# Create a dictionary with deprecated target_language set
d = TerminologyDictionary(
name="Old Dict",
source_dialect="a",
target_dialect="b",
target_language="ru",
)
db_session.add(d)
db_session.flush()
# Create old-style entries without explicit language pair (source_language=und, target_language=und)
entry1 = DictionaryEntry(
dictionary_id=d.id,
source_term="hello",
source_term_normalized="hello",
target_term="привет",
source_language="und",
target_language="und",
)
entry2 = DictionaryEntry(
dictionary_id=d.id,
source_term="world",
source_term_normalized="world",
target_term="мир",
source_language="und",
target_language="und",
origin_source_language="en",
)
db_session.add(entry1)
db_session.add(entry2)
db_session.commit()
# Run migration
result = DictionaryManager.migrate_old_entries(db_session)
# Verify migration counts
assert result["total_processed"] >= 2
assert result["migrated_source"] >= 1 # entry2 had origin_source_language
assert result["migrated_target"] >= 2 # both should get target_language from dictionary
# Verify entry1 got target_language from dictionary
db_session.refresh(entry1)
assert entry1.target_language == "ru"
# source_language should remain "und" (no origin_source_language)
assert entry1.source_language == "und"
# Verify entry2 got source from origin and target from dictionary
db_session.refresh(entry2)
assert entry2.source_language == "en"
assert entry2.target_language == "ru"
# endregion test_migrate_old_entries
# region test_export_entries [TYPE Function]
# @PURPOSE: Verify export_entries includes language columns.
def test_export_entries(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Export Test",
source_dialect="a", target_dialect="b",
)
DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")
DictionaryManager.add_entry(db_session, d.id, "world", "мир", source_language="en", target_language="ru")
csv_output = DictionaryManager.export_entries(db_session, d.id)
assert "source_language" in csv_output
assert "target_language" in csv_output
assert "context_data" in csv_output
assert "usage_notes" in csv_output
assert "en" in csv_output
assert "ru" in csv_output
# Verify it's valid CSV
reader = csv.DictReader(io.StringIO(csv_output))
rows = list(reader)
assert len(rows) == 2
assert rows[0]["source_language"] == "en"
assert rows[0]["target_language"] == "ru"
# endregion test_export_entries
# region test_import_with_default_language [TYPE Function]
# @PURPOSE: Verify import uses default_source_language and default_target_language when columns missing.
def test_import_with_default_language(db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Default Lang Import",
source_dialect="a", target_dialect="b",
)
# CSV without language columns
csv_content = "source_term,target_term\nhello,привет\nworld,мир"
result = DictionaryManager.import_entries(
db_session, d.id, csv_content,
delimiter=",", on_conflict="overwrite",
default_source_language="en",
default_target_language="ru",
)
assert result["created"] == 2
entries, _ = DictionaryManager.list_entries(db_session, d.id)
for e in entries:
assert e.source_language == "en"
assert e.target_language == "ru"
# endregion test_import_with_default_language
# region test_context_capture_in_correction [TYPE Function]
# @PURPOSE: Verify context_data is auto-captured when submitting a correction with source row context.
def test_context_capture_in_correction(db_session: Session):
"""Test T132: Context auto-capture when submitting a correction."""
from src.plugins.translate.service import InlineCorrectionService
from src.models.translate import TranslationRecord, TranslationRun, TranslationBatch, TranslationLanguage
d = DictionaryManager.create_dictionary(
db_session, name="Test Dict",
source_dialect="a", target_dialect="b",
)
job = TranslationJob(
name="Ctx Job", source_dialect="a", target_dialect="b",
status="DRAFT",
)
db_session.add(job)
db_session.flush()
run = TranslationRun(
job_id=job.id, status="COMPLETED",
)
db_session.add(run)
db_session.flush()
batch = TranslationBatch(
run_id=run.id, batch_index=0, status="COMPLETED",
)
db_session.add(batch)
db_session.flush()
# Create a TranslationRecord with source_data (context columns)
record = TranslationRecord(
batch_id=batch.id,
run_id=run.id,
source_sql="hello world",
source_object_type="table_row",
source_object_name="Row 0",
source_object_id="0",
source_data={"schema": "public", "table": "users", "column": "name"},
status="SUCCESS",
)
db_session.add(record)
db_session.flush()
# Create language entries for test languages
for lang_code in ("ru", "de", "fr", "it"):
tl = TranslationLanguage(
record_id=record.id,
language_code=lang_code,
source_language_detected="en",
translated_value="translated",
final_value="translated",
status="translated",
)
db_session.add(tl)
db_session.commit()
# Submit correction to dictionary
result = InlineCorrectionService.submit_correction_to_dict(
db=db_session,
record_id=record.id,
language_code="ru",
dictionary_id=d.id,
corrected_value="привет мир (исправлено)",
current_user="test_user",
)
assert result["action"] == "created"
assert result["entry_id"] is not None
# Verify context was auto-captured
entry = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result["entry_id"]).first()
assert entry is not None
assert entry.has_context is True
assert entry.context_source == "auto"
assert entry.context_data is not None
# source_data should contain the row's context columns
assert "source_data" in entry.context_data
assert entry.context_data["source_object_type"] == "table_row"
assert entry.context_data["source_object_name"] == "Row 0"
# Test context_data_override (use different language to avoid conflict)
result2 = InlineCorrectionService.submit_correction_to_dict(
db=db_session,
record_id=record.id,
language_code="de", # different target language avoids conflict
dictionary_id=d.id,
corrected_value="another correction",
current_user="test_user",
context_data_override={"custom_key": "custom_value"},
)
assert result2["action"] == "created"
entry2 = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result2["entry_id"]).first()
assert entry2.context_data == {"custom_key": "custom_value"}
assert entry2.context_source == "manual"
# Test keep_context=False
result3 = InlineCorrectionService.submit_correction_to_dict(
db=db_session,
record_id=record.id,
language_code="fr", # different target language avoids conflict
dictionary_id=d.id,
corrected_value="no context",
current_user="test_user",
keep_context=False,
)
assert result3["action"] == "created"
entry3 = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result3["entry_id"]).first()
assert entry3.context_data is None
assert entry3.has_context is False
assert entry3.context_source == "manual"
# Test usage_notes
result4 = InlineCorrectionService.submit_correction_to_dict(
db=db_session,
record_id=record.id,
language_code="it", # different target language avoids conflict
dictionary_id=d.id,
corrected_value="with notes",
current_user="test_user",
usage_notes="Use this for user-facing columns only",
)
assert result4["action"] == "created"
entry4 = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result4["entry_id"]).first()
assert entry4.usage_notes == "Use this for user-facing columns only"
# endregion test_context_capture_in_correction
# region test_jaccard_similarity [TYPE Function]
# @PURPOSE: Verify Jaccard similarity computation in ContextAwarePromptBuilder.
def test_jaccard_similarity():
"""Test T132: Jaccard similarity = 1.0 for identical, 0.0 for disjoint."""
from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder
builder = ContextAwarePromptBuilder()
# Identical contexts
ctx1 = {"schema": "public", "table": "users"}
ctx2 = {"schema": "public", "table": "users"}
sim = builder.compute_context_similarity(ctx1, ctx2)
assert sim == 1.0, f"Expected 1.0, got {sim}"
# Disjoint contexts
ctx3 = {"schema": "private"}
sim = builder.compute_context_similarity(ctx1, ctx3)
assert sim == 0.0, f"Expected 0.0, got {sim}"
# Partial overlap
ctx4 = {"schema": "public", "table": "orders"}
sim = builder.compute_context_similarity(ctx1, ctx4)
# intersection: {"public"} union: {"public", "users", "orders"} => 1/3 ≈ 0.33
assert 0.33 <= sim <= 0.34, f"Expected ~0.33, got {sim}"
# Empty/missing contexts
assert builder.compute_context_similarity(None, ctx1) == 0.0
assert builder.compute_context_similarity(ctx1, None) == 0.0
assert builder.compute_context_similarity({}, ctx1) == 0.0
# Case-insensitive matching (same keys, different case)
ctx5 = {"schema": "PUBLIC", "table": "USERS"}
sim = builder.compute_context_similarity(ctx1, ctx5)
assert sim == 1.0, f"Expected 1.0 for case-insensitive, got {sim}"
# endregion test_jaccard_similarity
# region test_context_truncation [TYPE Function]
# @PURPOSE: Verify context string truncation at ~2000 chars.
def test_context_truncation():
"""Test T132: Context string truncation in render_entry."""
from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder
builder = ContextAwarePromptBuilder()
# Build context data that would produce a very long context string
long_val = "x" * 500
long_context = {f"key{i}": long_val for i in range(10)}
# Create an entry-like dict with this context
entry = {
"source_term": "hello",
"target_term": "world",
"has_context": True,
"context_data": long_context,
"usage_notes": None,
}
rendered = builder.render_entry(entry, priority=False)
# The context part should be truncated
assert len(rendered) < 2200, f"Rendered length {len(rendered)} exceeds expected max"
assert "...[truncated]" in rendered, "Expected truncation marker in output"
# Short context should not be truncated
short_entry = {
"source_term": "hello",
"target_term": "world",
"has_context": True,
"context_data": {"table": "users"},
"usage_notes": None,
}
short_rendered = builder.render_entry(short_entry, priority=False)
assert "...[truncated]" not in short_rendered
assert "(context: table=users)" in short_rendered
# Priority prefix
priority_rendered = builder.render_entry(short_entry, priority=True)
assert priority_rendered.startswith("# PRIORITY (context match)")
# Usage notes
notes_entry = {
"source_term": "hello",
"target_term": "world",
"has_context": False,
"context_data": None,
"usage_notes": "Use for admin panels",
}
notes_rendered = builder.render_entry(notes_entry, priority=False)
assert "# Usage: Use for admin panels" in notes_rendered
# endregion test_context_truncation
# region test_render_entry_dict_and_object [TYPE Function]
# @PURPOSE: Verify render_entry handles both dicts and objects.
def test_render_entry_dict_and_object():
"""Test T132: render_entry accepts dict or object with same results."""
from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder
builder = ContextAwarePromptBuilder()
# Dict entry
dict_entry = {
"source_term": "hello",
"target_term": "hola",
"has_context": False,
"context_data": None,
"usage_notes": None,
}
dict_result = builder.render_entry(dict_entry)
# Object-like entry using a simple class
class FakeEntry:
source_term = "hello"
target_term = "hola"
has_context = False
context_data = None
usage_notes = None
obj_result = builder.render_entry(FakeEntry())
assert dict_result == obj_result
assert dict_result == '"hello" -> "hola"'
# endregion test_render_entry_dict_and_object
# region test_build_context_entries_prioritization [TYPE Function]
# @PURPOSE: Verify build_context_entries sorts priority entries first.
def test_build_context_entries_prioritization():
"""Test T132: build_context_entries sorts priority entries before non-priority."""
from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder
builder = ContextAwarePromptBuilder()
entries = [
{
"source_term": "high_match",
"target_term": "alta_coincidencia",
"has_context": True,
"context_data": {"schema": "public", "table": "users"},
"usage_notes": None,
},
{
"source_term": "no_match",
"target_term": "sin_coincidencia",
"has_context": True,
"context_data": {"schema": "private", "table": "config"},
"usage_notes": None,
},
{
"source_term": "no_context",
"target_term": "sin_contexto",
"has_context": False,
"context_data": None,
"usage_notes": None,
},
]
# Row context that matches the first entry
row_context = {"schema": "public", "table": "users"}
rendered = builder.build_context_entries(entries, row_context)
# The first entry should be priority (similarity=1.0 >= 0.5)
assert len(rendered) == 3
assert rendered[0].startswith("# PRIORITY (context match)"), f"Expected priority first, got: {rendered[0]}"
assert '"high_match"' in rendered[0]
# The non-priority entries should not have priority prefix
assert not rendered[1].startswith("# PRIORITY")
assert not rendered[2].startswith("# PRIORITY")
# endregion test_build_context_entries_prioritization
# endregion DictionaryTests

View File

@@ -11,6 +11,7 @@
# @TEST_EDGE: invalid_run_status -> raises ValueError
# @TEST_EDGE: executor_failure -> run is marked FAILED
import json
from datetime import UTC, datetime
from unittest.mock import MagicMock, patch
@@ -18,11 +19,14 @@ import pytest
from src.models.translate import (
TranslationJob,
TranslationLanguage,
TranslationPreviewSession,
TranslationRecord,
TranslationRun,
TranslationRunLanguageStats,
)
from src.plugins.translate.events import TranslationEventLog
from src.plugins.translate.executor import TranslationExecutor
from src.plugins.translate.orchestrator import TranslationOrchestrator
@@ -598,4 +602,249 @@ class TestTranslationOrchestrator:
# endregion test_get_run_records
# endregion test_get_run_history
# endregion TestTranslationOrchestrator
# region TestTranslationExecutorMultiLang [TYPE Class]
# @PURPOSE: Tests for multi-language LLM translation execution: parsing, per-language entries, source-as-reference, backward compat.
class TestTranslationExecutorMultiLang:
# region test_parse_multi_language_response [TYPE Function]
# @PURPOSE: _parse_llm_response handles multi-language JSON format correctly.
def test_parse_multi_language_response(self) -> None:
response = json.dumps({
"rows": [
{"row_id": 0, "detected_source_language": "fr", "ru": "текст", "en": "text", "de": "Text"},
{"row_id": 1, "detected_source_language": "en", "ru": "дашборд", "en": "dashboard", "de": "Dashboard"},
]
})
result = TranslationExecutor._parse_llm_response(
response, expected_count=2, target_languages=["ru", "en", "de"]
)
assert len(result) == 2
assert "0" in result
assert result["0"]["detected_source_language"] == "fr"
assert result["0"]["ru"] == "текст"
assert result["0"]["en"] == "text"
assert result["0"]["de"] == "Text"
assert "1" in result
assert result["1"]["en"] == "dashboard"
assert result["1"]["ru"] == "дашборд"
# endregion test_parse_multi_language_response
# region test_parse_legacy_format_backward_compat [TYPE Function]
# @PURPOSE: _parse_llm_response still handles legacy single "translation" key format.
def test_parse_legacy_format_backward_compat(self) -> None:
response = json.dumps({
"rows": [
{"row_id": 0, "translation": "текст", "detected_source_language": "fr"},
{"row_id": 1, "translation": "дашборд", "detected_source_language": "en"},
]
})
result = TranslationExecutor._parse_llm_response(
response, expected_count=2, target_languages=["ru"]
)
assert len(result) == 2
assert result["0"]["translation"] == "текст"
assert result["1"]["detected_source_language"] == "en"
# endregion test_parse_legacy_format_backward_compat
# region test_multi_language_translation_language_entries [TYPE Function]
# @PURPOSE: Multi-language LLM response creates per-language TranslationLanguage entries.
def test_multi_language_translation_language_entries(self) -> None:
db = MagicMock()
config_manager = MagicMock()
# Mock job with multi-language target_languages
job = MagicMock(spec=TranslationJob)
job.id = "job-ml-1"
job.target_languages = ["ru", "en"]
job.target_language = None
job.target_dialect = "postgresql"
job.source_dialect = "postgresql"
job.translation_column = "name"
job.provider_id = "provider-1"
job.batch_size = 50
executor = TranslationExecutor(db, config_manager, "test-user")
# Patch _call_llm to return multi-language response
multi_lang_response = json.dumps({
"rows": [
{"row_id": "0", "detected_source_language": "fr", "ru": "текст", "en": "text"},
{"row_id": "1", "detected_source_language": "en", "ru": "дашборд", "en": "dashboard"},
]
})
batch_rows = [
{"row_index": "0", "source_text": "texte", "source_object_name": "Row 0"},
{"row_index": "1", "source_text": "dashboard", "source_object_name": "Row 1"},
]
with patch.object(executor, '_call_llm', return_value=multi_lang_response):
result = executor._call_llm_for_batch(
job=job,
run_id="run-ml-1",
batch_rows=batch_rows,
dict_matches=[],
batch_id="batch-ml-1",
)
assert result["successful"] == 2
assert result["failed"] == 0
assert result["skipped"] == 0
# Check that TranslationLanguage entries were created for each row per language
# Each row should have 2 language entries (ru, en) = 4 total
lang_added_calls = [
call for call in db.add.call_args_list
if isinstance(call[0][0], TranslationLanguage)
]
# Should have 4 language entries (2 rows x 2 languages)
assert len(lang_added_calls) >= 4, f"Expected at least 4 TranslationLanguage entries, got {len(lang_added_calls)}"
# Group by language code
lang_codes: list[str] = []
for call in lang_added_calls:
lang_entry = call[0][0]
lang_codes.append(lang_entry.language_code)
assert "ru" in lang_codes
assert "en" in lang_codes
# Each language should appear twice (once per row)
assert lang_codes.count("ru") == 2
assert lang_codes.count("en") == 2
# endregion test_multi_language_translation_language_entries
# region test_source_as_reference [TYPE Function]
# @PURPOSE: When detected source language matches a target language, store original text verbatim.
def test_source_as_reference(self) -> None:
db = MagicMock()
config_manager = MagicMock()
job = MagicMock(spec=TranslationJob)
job.id = "job-sar-1"
job.target_languages = ["fr", "en"]
job.target_language = None
job.target_dialect = "postgresql"
job.source_dialect = "postgresql"
job.translation_column = "name"
job.provider_id = "provider-1"
job.batch_size = 50
executor = TranslationExecutor(db, config_manager, "test-user")
# Source is French, target_languages include French
response = json.dumps({
"rows": [
{"row_id": "0", "detected_source_language": "fr", "fr": "texte français", "en": "french text"},
]
})
batch_rows = [
{"row_index": "0", "source_text": "texte français", "source_object_name": "Row 0"},
]
with patch.object(executor, '_call_llm', return_value=response):
result = executor._call_llm_for_batch(
job=job,
run_id="run-sar-1",
batch_rows=batch_rows,
dict_matches=[],
batch_id="batch-sar-1",
)
assert result["successful"] == 1
# Find the TranslationLanguage for French — should use source text (original)
lang_calls = [
call for call in db.add.call_args_list
if isinstance(call[0][0], TranslationLanguage)
]
fr_entry = None
en_entry = None
for call in lang_calls:
le = call[0][0]
if le.language_code == "fr":
fr_entry = le
elif le.language_code == "en":
en_entry = le
assert fr_entry is not None, "FR language entry should exist"
# Source-as-reference: fr translation should be the original source text
assert fr_entry.translated_value == "texte français", (
f"Expected source text 'texte français' for fr, got '{fr_entry.translated_value}'"
)
assert en_entry is not None, "EN language entry should exist"
# EN should get the LLM-translated value
assert en_entry.translated_value == "french text"
# endregion test_source_as_reference
# region test_per_language_stats_on_execute_run [TYPE Function]
# @PURPOSE: execute_run with multi-language job creates and populates TranslationRunLanguageStats.
def test_per_language_stats_on_execute_run(self, mock_job: MagicMock) -> None:
db = MagicMock()
config_manager = MagicMock()
# Configure mock_job for multi-language
mock_job.target_languages = ["ru", "en"]
mock_job.target_language = None
mock_job.target_table = "target_tbl"
# Mock run
run = MagicMock()
run.id = "run-stats-1"
run.job_id = "job-123"
run.status = "PENDING"
# Mock completed run returned by executor
completed_run = MagicMock()
completed_run.id = "run-stats-1"
completed_run.job_id = "job-123"
completed_run.status = "COMPLETED"
completed_run.total_records = 5
completed_run.successful_records = 5
completed_run.failed_records = 0
completed_run.skipped_records = 0
completed_run.completed_at = datetime.now(UTC)
completed_run.error_message = None
completed_run.insert_status = "success"
completed_run.superset_execution_id = "q-1"
completed_run.superset_execution_log = {}
# Mock queries: job -> mock_job, run -> completed_run
db.query.return_value.filter.return_value.first.side_effect = [
mock_job,
completed_run,
]
orch = TranslationOrchestrator(db, config_manager, "test-user")
with patch.object(orch, 'event_log'), \
patch.object(orch, '_generate_and_insert_sql',
return_value={"status": "success", "query_id": "q-1", "rows_affected": 5}), \
patch.object(orch, '_update_language_stats') as mock_update_stats, \
patch('src.plugins.translate.orchestrator.TranslationExecutor') as MockExecutor:
mock_executor_instance = MagicMock()
mock_executor_instance.execute_run.return_value = completed_run
MockExecutor.return_value = mock_executor_instance
result = orch.execute_run(run)
assert result.status == "COMPLETED"
# Verify that TranslationRunLanguageStats entries were created
stats_added = [
call for call in db.add.call_args_list
if isinstance(call[0][0], TranslationRunLanguageStats)
]
# 2 language stats entries (ru, en)
assert len(stats_added) >= 2, (
f"Expected at least 2 TranslationRunLanguageStats entries, got {len(stats_added)}"
)
codes = [call[0][0].language_code for call in stats_added]
assert "ru" in codes
assert "en" in codes
# Verify _update_language_stats was called
mock_update_stats.assert_called_once()
# endregion test_per_language_stats_on_execute_run
# endregion TestTranslationExecutorMultiLang
# endregion OrchestratorTests

View File

@@ -420,8 +420,13 @@ class TestTranslationPreview:
assert tokens > 0
assert tokens < len(prompt) # tokens typically fewer than chars
output_tokens = TokenEstimator.estimate_output_tokens(10)
assert output_tokens == 500
# Single language
output_tokens = TokenEstimator.estimate_output_tokens(10, num_languages=1)
assert output_tokens == 600 # 10 * 1 * 50 * 1.2
# Multi-language
output_tokens_multi = TokenEstimator.estimate_output_tokens(10, num_languages=3)
assert output_tokens_multi == 1800 # 10 * 3 * 50 * 1.2
cost = TokenEstimator.estimate_cost(1000, 0.002)
assert cost == 0.002
@@ -431,28 +436,65 @@ class TestTranslationPreview:
# endregion test_cost_estimation
# region test_preview_parse_llm_response [TYPE Function]
# @PURPOSE: Verify LLM JSON response parsing.
# @PURPOSE: Verify LLM JSON response parsing includes detected_source_language.
def test_preview_parse_llm_response(self):
"""Parse LLM response should extract translations keyed by row_id."""
"""Parse LLM response should extract translations with detected_source_language per row."""
response = json.dumps({
"rows": [
{"row_id": "0", "translation": "Привет"},
{"row_id": "0", "translation": "Привет", "detected_source_language": "en"},
{"row_id": "1", "translation": "Мир"},
]
})
result = TranslationPreview._parse_llm_response(response, 2)
assert result == {"0": "Привет", "1": "Мир"}
assert result == {
"0": {"translation": "Привет", "detected_source_language": "en"},
"1": {"translation": "Мир", "detected_source_language": "und"},
}
# endregion test_preview_parse_llm_response
# region test_preview_parse_llm_response_multilang [TYPE Function]
# @PURPOSE: Verify multi-language LLM response parsing.
def test_preview_parse_llm_response_multilang(self):
"""Parse multi-language LLM response should extract per-language translations."""
response = json.dumps({
"rows": [
{"row_id": "0", "detected_source_language": "fr", "ru": "текст", "en": "text", "de": "Text"},
{"row_id": "1", "detected_source_language": "en", "ru": "привет", "en": "hello"},
]
})
result = TranslationPreview._parse_llm_response(response, 2, target_languages=["ru", "en", "de"])
assert result == {
"0": {"detected_source_language": "fr", "ru": "текст", "en": "text", "de": "Text"},
"1": {"detected_source_language": "en", "ru": "привет", "en": "hello"},
}
# Verify per-language extraction
assert result["0"]["ru"] == "текст"
assert result["0"]["en"] == "text"
assert result["1"]["ru"] == "привет"
# endregion test_preview_parse_llm_response_multilang
# region test_preview_parse_llm_response_with_code_block [TYPE Function]
# @PURPOSE: Verify LLM response parsing handles markdown code blocks.
def test_preview_parse_llm_response_with_code_block(self):
"""Parse LLM response should handle markdown code block wrapping."""
response = "```json\n{\n \"rows\": [\n {\"row_id\": \"0\", \"translation\": \"Test\"}\n ]\n}\n```"
result = TranslationPreview._parse_llm_response(response, 1)
assert result == {"0": "Test"}
assert result == {"0": {"translation": "Test", "detected_source_language": "und"}}
# endregion test_preview_parse_llm_response_with_code_block
# region test_cost_warning [TYPE Function]
# @PURPOSE: Verify cost warning for large sample sizes.
def test_cost_warning(self):
"""Cost warning should appear for sample_size > 30."""
warning = TokenEstimator.check_cost_warning(31, 2, 5000, 0.01)
assert warning is not None
assert "Large preview" in warning
assert "5000" in warning
no_warning = TokenEstimator.check_cost_warning(10, 2, 1000, 0.002)
assert no_warning is None
# endregion test_cost_warning
# region test_preview_compute_config_hash [TYPE Function]
# @PURPOSE: Verify config hash computation is deterministic.
def test_preview_compute_config_hash(self):

View File

@@ -14,7 +14,7 @@ import io
import re
from typing import Any
from sqlalchemy import func
from sqlalchemy import func, or_
from sqlalchemy.orm import Session
from ...core.logger import belief_scope, logger
@@ -27,6 +27,23 @@ from ...models.translate import (
from ._utils import _detect_delimiter, _normalize_term
# #region _validate_bcp47 [C:2] [TYPE Function] [SEMANTICS validation, bcp47, language]
# @BRIEF Validate that a language tag is a non-empty BCP-47 string.
def _validate_bcp47(tag: str, field_name: str) -> None:
"""Validate that tag is a non-empty BCP-47 string (basic check)."""
if not tag or not tag.strip():
raise ValueError(f"{field_name} must be a non-empty BCP-47 language tag")
tag = tag.strip()
# Basic BCP-47 validation: must match lang[-script][-region][-variant]* pattern
import re as _re
if not _re.match(r'^[a-zA-Z]{2,8}(-[a-zA-Z0-9]{1,8})*$', tag):
raise ValueError(
f"{field_name} is not a valid BCP-47 tag: '{tag}'. "
"Expected format like 'en', 'ru', 'zh-CN', 'pt-BR'."
)
# #endregion _validate_bcp47
# #region DictionaryManager [C:4] [TYPE Class]
# @BRIEF Manages terminology dictionaries and their entries with referential integrity.
# @PRE: Database session is open and valid.
@@ -34,12 +51,14 @@ from ._utils import _detect_delimiter, _normalize_term
# @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
class DictionaryManager:
# region DictionaryManager.create_dictionary [TYPE Function]
# @PURPOSE: Create a new terminology dictionary.
# @PRE: payload contains name, source_dialect, target_dialect.
# @PURPOSE: Create a new terminology dictionary (language independent at dictionary level; language pairs are per-entry).
# @PRE: name is provided.
# @POST: New TerminologyDictionary row is created and returned.
@staticmethod
def create_dictionary(
db: Session, name: str, source_dialect: str, target_dialect: str,
db: Session, name: str,
source_dialect: str = "",
target_dialect: str = "",
created_by: str | None = None, description: str | None = None,
is_active: bool = True,
) -> TerminologyDictionary:
@@ -48,8 +67,8 @@ class DictionaryManager:
dictionary = TerminologyDictionary(
name=name,
description=description,
source_dialect=source_dialect,
target_dialect=target_dialect,
source_dialect=source_dialect or "",
target_dialect=target_dialect or "",
is_active=is_active,
created_by=created_by,
)
@@ -166,53 +185,66 @@ class DictionaryManager:
# endregion DictionaryManager.list_dictionaries
# region DictionaryManager.add_entry [TYPE Function]
# @PURPOSE: Add an entry to a dictionary, enforcing unique source_term_normalized.
# @PRE: dict_id exists. source_term and target_term are non-empty.
# @PURPOSE: Add an entry to a dictionary with language-pair-aware uniqueness validation.
# @PRE: dict_id exists. source_term, target_term are non-empty. source_language, target_language are valid BCP-47.
# @POST: New DictionaryEntry row is created or raises on duplicate.
@staticmethod
def add_entry(
db: Session, dict_id: str, source_term: str, target_term: str,
context_notes: str | None = None,
source_language: str = "und",
target_language: str = "und",
) -> DictionaryEntry:
with belief_scope("DictionaryManager.add_entry"):
_validate_bcp47(source_language, "source_language")
_validate_bcp47(target_language, "target_language")
normalized = _normalize_term(source_term)
# Uniqueness is now (dictionary_id, source_term_normalized, source_language, target_language)
existing = (
db.query(DictionaryEntry)
.filter(
DictionaryEntry.dictionary_id == dict_id,
DictionaryEntry.source_term_normalized == normalized,
DictionaryEntry.source_language == source_language,
DictionaryEntry.target_language == target_language,
)
.first()
)
if existing:
raise ValueError(
f"Duplicate entry: '{source_term}' already exists in this dictionary "
f"(id={existing.id}). Use overwrite or keep_existing conflict mode."
f"Duplicate entry: '{source_term}' (lang: {source_language}{target_language}) "
f"already exists in this dictionary (id={existing.id}). "
"Use overwrite or keep_existing conflict mode."
)
logger.reason("Adding dictionary entry", {"dict_id": dict_id, "term": source_term})
logger.reason("Adding dictionary entry", {"dict_id": dict_id, "term": source_term, "src_lang": source_language, "tgt_lang": target_language})
entry = DictionaryEntry(
dictionary_id=dict_id,
source_term=source_term.strip(),
source_term_normalized=normalized,
target_term=target_term.strip(),
context_notes=context_notes.strip() if context_notes else None,
source_language=source_language.strip(),
target_language=target_language.strip(),
)
db.add(entry)
db.commit()
db.refresh(entry)
logger.reflect("Entry added", {"entry_id": entry.id})
logger.reflect("Entry added", {"entry_id": entry.id, "src_lang": source_language, "tgt_lang": target_language})
return entry
# endregion DictionaryManager.add_entry
# region DictionaryManager.edit_entry [TYPE Function]
# @PURPOSE: Edit an existing dictionary entry with duplicate-aware normalization.
# @PURPOSE: Edit an existing dictionary entry with language-pair-aware duplicate check.
# @PRE: entry_id exists.
# @POST: Entry fields are updated.
@staticmethod
def edit_entry(
db: Session, entry_id: str, source_term: str | None = None,
target_term: str | None = None, context_notes: str | None = None,
source_language: str | None = None,
target_language: str | None = None,
) -> DictionaryEntry:
with belief_scope("DictionaryManager.edit_entry"):
entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()
@@ -220,14 +252,24 @@ class DictionaryManager:
raise ValueError(f"Entry not found: {entry_id}")
logger.reason("Editing dictionary entry", {"entry_id": entry_id})
if source_language is not None:
_validate_bcp47(source_language, "source_language")
entry.source_language = source_language.strip()
if target_language is not None:
_validate_bcp47(target_language, "target_language")
entry.target_language = target_language.strip()
if source_term is not None:
normalized = _normalize_term(source_term)
# Check uniqueness within the same dictionary
# Check uniqueness within the same dictionary + language pair
existing = (
db.query(DictionaryEntry)
.filter(
DictionaryEntry.dictionary_id == entry.dictionary_id,
DictionaryEntry.source_term_normalized == normalized,
DictionaryEntry.source_language == entry.source_language,
DictionaryEntry.target_language == entry.target_language,
DictionaryEntry.id != entry_id,
)
.first()
@@ -318,6 +360,8 @@ class DictionaryManager:
delimiter: str | None = None,
on_conflict: str = "overwrite",
preview_only: bool = False,
default_source_language: str | None = None,
default_target_language: str | None = None,
) -> dict[str, Any]:
with belief_scope("DictionaryManager.import_entries"):
# Validate dictionary
@@ -359,6 +403,8 @@ class DictionaryManager:
source_term = row.get("source_term", "").strip()
target_term = row.get("target_term", "").strip()
context_notes = row.get("context_notes", "").strip() or None
source_language = row.get("source_language", "").strip() or default_source_language or "und"
target_language = row.get("target_language", "").strip() or default_target_language or "und"
if not source_term or not target_term:
result["errors"].append({
@@ -376,6 +422,8 @@ class DictionaryManager:
.filter(
DictionaryEntry.dictionary_id == dict_id,
DictionaryEntry.source_term_normalized == normalized,
DictionaryEntry.source_language == source_language,
DictionaryEntry.target_language == target_language,
)
.first()
)
@@ -384,6 +432,8 @@ class DictionaryManager:
"source_term": source_term,
"target_term": target_term,
"context_notes": context_notes,
"source_language": source_language,
"target_language": target_language,
"is_conflict": existing is not None,
"existing_target_term": existing.target_term if existing else None,
}
@@ -395,6 +445,8 @@ class DictionaryManager:
.filter(
DictionaryEntry.dictionary_id == dict_id,
DictionaryEntry.source_term_normalized == normalized,
DictionaryEntry.source_language == source_language,
DictionaryEntry.target_language == target_language,
)
.first()
)
@@ -404,6 +456,8 @@ class DictionaryManager:
existing.source_term = source_term
existing.target_term = target_term
existing.context_notes = context_notes
existing.source_language = source_language
existing.target_language = target_language
result["updated"] += 1
elif on_conflict == "keep_existing":
result["skipped"] += 1
@@ -421,6 +475,8 @@ class DictionaryManager:
source_term_normalized=normalized,
target_term=target_term,
context_notes=context_notes,
source_language=source_language,
target_language=target_language,
)
db.add(entry)
result["created"] += 1
@@ -446,14 +502,122 @@ class DictionaryManager:
return result
# endregion DictionaryManager.import_entries
# region DictionaryManager.export_entries [TYPE Function]
# @PURPOSE: Export all entries as CSV string with language columns.
# @PRE: dict_id exists.
# @POST: Returns CSV string with header: source_term, target_term, source_language, target_language, context_notes, context_data, usage_notes.
@staticmethod
def export_entries(
db: Session, dict_id: str, delimiter: str = ",",
) -> str:
with belief_scope("DictionaryManager.export_entries"):
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
if not dictionary:
raise ValueError(f"Dictionary not found: {dict_id}")
entries = (
db.query(DictionaryEntry)
.filter(DictionaryEntry.dictionary_id == dict_id)
.order_by(DictionaryEntry.source_term.asc())
.all()
)
output = io.StringIO()
fieldnames = [
"source_term", "target_term",
"source_language", "target_language",
"context_notes", "context_data", "usage_notes",
]
writer = csv.DictWriter(output, fieldnames=fieldnames, delimiter=delimiter)
writer.writeheader()
for entry in entries:
writer.writerow({
"source_term": entry.source_term,
"target_term": entry.target_term,
"source_language": entry.source_language,
"target_language": entry.target_language,
"context_notes": entry.context_notes or "",
"context_data": entry.context_data,
"usage_notes": entry.usage_notes or "",
})
result = output.getvalue()
logger.reflect("Entries exported", {"dict_id": dict_id, "count": len(entries), "delimiter": repr(delimiter)})
return result
# endregion DictionaryManager.export_entries
# region DictionaryManager.migrate_old_entries [TYPE Function]
# @PURPOSE: Migrate existing single-language dictionary entries to populate source_language and target_language.
# @PRE: db session is open.
# @POST: Entries with "und" source_language or target_language are updated with inferred values.
# @SIDE_EFFECT: Updates DictionaryEntry rows in bulk.
@staticmethod
def migrate_old_entries(db: Session) -> dict[str, int]:
with belief_scope("DictionaryManager.migrate_old_entries"):
logger.reason("Starting migration of old dictionary entries")
migrated_source = 0
migrated_target = 0
skipped = 0
# Find all entries that may need migration (source_language is "und" or target_language is "und")
entries = (
db.query(DictionaryEntry)
.filter(
(DictionaryEntry.source_language == "und") |
(DictionaryEntry.target_language == "und")
)
.all()
)
for entry in entries:
# Try to infer source_language from origin_source_language
if entry.source_language == "und":
if entry.origin_source_language:
entry.source_language = entry.origin_source_language
migrated_source += 1
# Try to infer target_language from the parent dictionary's deprecated target_language
if entry.target_language == "und":
dictionary = (
db.query(TerminologyDictionary)
.filter(TerminologyDictionary.id == entry.dictionary_id)
.first()
)
if dictionary and dictionary.target_language:
entry.target_language = dictionary.target_language
migrated_target += 1
if entry.source_language == "und" and entry.target_language == "und":
skipped += 1
db.commit()
logger.reflect("Migration complete", {
"migrated_source": migrated_source,
"migrated_target": migrated_target,
"skipped": skipped,
"total_processed": len(entries),
})
return {
"migrated_source": migrated_source,
"migrated_target": migrated_target,
"skipped": skipped,
"total_processed": len(entries),
}
# endregion DictionaryManager.migrate_old_entries
# region DictionaryManager.filter_for_batch [TYPE Function]
# @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.
# @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job,
# optionally filtered by language pair.
# @PRE: job_id exists and source_texts is a list of strings.
# @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries.
# @SIDE_EFFECT: Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.
@staticmethod
def filter_for_batch(
db: Session, source_texts: list[str], job_id: str,
source_language: str | None = None,
target_language: str | None = None,
) -> list[dict[str, Any]]:
with belief_scope("DictionaryManager.filter_for_batch"):
# Get dictionaries attached to this job
@@ -508,6 +672,18 @@ class DictionaryManager:
if not norm:
continue
# Apply language pair filtering
if source_language is not None:
src = source_language.strip().lower()
entry_src = entry.source_language.strip().lower()
if entry_src != src and entry_src != "und":
continue
if target_language is not None:
tgt = target_language.strip().lower()
entry_tgt = entry.target_language.strip().lower()
if entry_tgt != tgt:
continue
# Word-boundary-aware matching
# Build pattern: \bterm\b (case-insensitive)
# Escape regex special chars in the search term
@@ -528,6 +704,12 @@ class DictionaryManager:
"dictionary_id": entry.dictionary_id,
"dictionary_name": dict_map.get(entry.dictionary_id, ""),
"context_notes": entry.context_notes,
"context_data": entry.context_data,
"has_context": entry.has_context,
"context_source": entry.context_source,
"usage_notes": entry.usage_notes,
"source_language": entry.source_language,
"target_language": entry.target_language,
})
# Sort by dictionary link priority (order of dict_ids from link order)
@@ -539,6 +721,8 @@ class DictionaryManager:
"source_texts": len(source_texts),
"matches": len(matched),
"dictionaries_used": len(dictionaries),
"source_language": source_language,
"target_language": target_language,
})
return matched
# endregion DictionaryManager.filter_for_batch
@@ -560,6 +744,8 @@ class DictionaryManager:
origin_row_key: str | None = None,
origin_user_id: str | None = None,
on_conflict: str = "overwrite",
context_data: dict[str, Any] | None = None,
usage_notes: str | None = None,
) -> dict[str, Any]:
with belief_scope("DictionaryManager.submit_correction"):
# Validate dictionary exists
@@ -568,11 +754,22 @@ class DictionaryManager:
raise ValueError(f"Dictionary '{dict_id}' not found")
normalized = _normalize_term(source_term)
entry_src_lang = dictionary.source_dialect or "und"
entry_tgt_lang = dictionary.target_dialect or "und"
# Find existing entry: match exact language pair OR old-style "und"/"und" entries
existing = (
db.query(DictionaryEntry)
.filter(
DictionaryEntry.dictionary_id == dict_id,
DictionaryEntry.source_term_normalized == normalized,
or_(
# Exact language pair match
(DictionaryEntry.source_language == entry_src_lang) &
(DictionaryEntry.target_language == entry_tgt_lang),
# Old-style entries without language pair
(DictionaryEntry.source_language == "und") &
(DictionaryEntry.target_language == "und"),
),
)
.first()
)
@@ -606,6 +803,12 @@ class DictionaryManager:
existing.origin_row_key = origin_row_key
if origin_user_id:
existing.origin_user_id = origin_user_id
if context_data is not None:
existing.context_data = context_data
existing.has_context = bool(context_data)
existing.context_source = "auto"
if usage_notes is not None:
existing.usage_notes = usage_notes
db.flush()
result["action"] = "updated"
result["entry_id"] = existing.id
@@ -621,12 +824,18 @@ class DictionaryManager:
result["message"] = "Correction cancelled by conflict mode"
return result
else:
# Create new entry
# Create new entry — derive language pair from dictionary
entry = DictionaryEntry(
dictionary_id=dict_id,
source_term=source_term.strip(),
source_term_normalized=normalized,
target_term=corrected_target_term.strip(),
source_language=entry_src_lang,
target_language=entry_tgt_lang,
context_data=context_data,
usage_notes=usage_notes,
has_context=bool(context_data),
context_source="auto" if context_data else None,
origin_run_id=origin_run_id,
origin_row_key=origin_row_key,
origin_user_id=origin_user_id,
@@ -678,11 +887,19 @@ class DictionaryManager:
continue
normalized = _normalize_term(source_term)
bulk_src_lang = dictionary.source_dialect or "und"
bulk_tgt_lang = dictionary.target_dialect or "und"
existing = (
db.query(DictionaryEntry)
.filter(
DictionaryEntry.dictionary_id == dict_id,
DictionaryEntry.source_term_normalized == normalized,
or_(
(DictionaryEntry.source_language == bulk_src_lang) &
(DictionaryEntry.target_language == bulk_tgt_lang),
(DictionaryEntry.source_language == "und") &
(DictionaryEntry.target_language == "und"),
),
)
.first()
)
@@ -729,6 +946,8 @@ class DictionaryManager:
source_term=source_term,
source_term_normalized=normalized,
target_term=corrected_target,
source_language=bulk_src_lang,
target_language=bulk_tgt_lang,
origin_run_id=corr.get("origin_run_id"),
origin_row_key=corr.get("origin_row_key"),
origin_user_id=origin_user_id,

View File

@@ -191,6 +191,18 @@ class TranslationEventLog:
logger.reflect("No expired events to prune", {})
return {"pruned": 0, "snapshot_id": None}
# Compute per-language metrics from expired events before pruning
per_language: dict[str, dict[str, int | float]] = {}
expired_events = expired_query.all()
for evt in expired_events:
event_data = evt.event_data or {}
lang = event_data.get("language_code") or event_data.get("target_language") or "_unknown_"
if lang not in per_language:
per_language[lang] = {"cumulative_tokens": 0, "cumulative_cost": 0.0, "runs": 0}
per_language[lang]["cumulative_tokens"] += event_data.get("token_count", 0) or 0
per_language[lang]["cumulative_cost"] += event_data.get("cost", 0.0) or 0.0
per_language[lang]["runs"] += 1 if event_data.get("run_id") else 0
# Create MetricSnapshot before pruning
snapshot = MetricSnapshot(
id=str(uuid.uuid4()),
@@ -198,6 +210,7 @@ class TranslationEventLog:
key_hash=f"prune_{cutoff.timestamp():.0f}",
covers_events_before=cutoff,
total_records=total_expired,
per_language_metrics=per_language or None,
snapshot_date=datetime.now(UTC),
)
self.db.add(snapshot)

View File

@@ -27,6 +27,7 @@ from ...core.logger import belief_scope, logger
from ...models.translate import (
TranslationBatch,
TranslationJob,
TranslationLanguage,
TranslationPreviewRecord,
TranslationPreviewSession,
TranslationRecord,
@@ -36,6 +37,7 @@ from ...services.llm_prompt_templates import render_prompt
from ...services.llm_provider import LLMProviderService
from .dictionary import DictionaryManager
from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE
from .prompt_builder import ContextAwarePromptBuilder
# #region MAX_RETRIES_PER_BATCH [TYPE Constant]
# @BRIEF Maximum number of retries for a single batch before marking it failed.
@@ -62,6 +64,7 @@ class TranslationExecutor:
self.current_user = current_user
self.on_batch_progress = on_batch_progress
self._current_run_id: str | None = None
self._preview_edits_cache: dict[str, dict[str, str]] | None = None # key_hash -> {lang_code: edited_value}
# region execute_run [TYPE Function]
# @PURPOSE: Run full translation execution for a TranslationRun.
@@ -84,6 +87,9 @@ class TranslationExecutor:
"batch_size": job.batch_size,
})
# Load preview edits for carry-forward
self._load_preview_edits(job.id)
# Mark run as RUNNING
run.status = "RUNNING"
run.started_at = datetime.now(UTC)
@@ -410,6 +416,70 @@ class TranslationExecutor:
return []
# endregion _extract_chart_data_rows
# region _load_preview_edits [TYPE Function]
# @PURPOSE: Load user edits from accepted preview session for carry-forward.
# @PRE: job_id exists.
# @POST: Populates _preview_edits_cache with key_hash -> {lang_code: edited_value}.
# @SIDE_EFFECT: Queries TranslationPreviewLanguage and TranslationPreviewRecord.
def _load_preview_edits(self, job_id: str) -> None:
"""Load preview edits for carry-forward during execution."""
from ...models.translate import TranslationPreviewLanguage, TranslationPreviewRecord, TranslationPreviewSession
with belief_scope("TranslationExecutor._load_preview_edits"):
session = (
self.db.query(TranslationPreviewSession)
.filter(
TranslationPreviewSession.job_id == job_id,
TranslationPreviewSession.status == "APPLIED",
)
.order_by(TranslationPreviewSession.created_at.desc())
.first()
)
if not session:
logger.reason("No applied preview session found — no edits to carry forward", {
"job_id": job_id,
})
self._preview_edits_cache = {}
return
records = (
self.db.query(TranslationPreviewRecord)
.filter(TranslationPreviewRecord.session_id == session.id)
.all()
)
edits: dict[str, dict[str, str]] = {}
for rec in records:
if not rec.source_data:
continue
# Compute key hash from source_data
key_hash = self._compute_key_hash(rec.source_data)
edited_langs: dict[str, str] = {}
for lang_entry in (rec.languages or []):
if lang_entry.status in ("edited", "approved") and lang_entry.user_edit:
edited_langs[lang_entry.language_code] = lang_entry.final_value or lang_entry.user_edit
logger.reason("Carrying forward preview edit", {
"key_hash": key_hash,
"language_code": lang_entry.language_code,
})
if edited_langs:
edits[key_hash] = edited_langs
self._preview_edits_cache = edits
logger.reason(f"Loaded {len(edits)} preview edits for carry-forward", {
"job_id": job_id,
})
# endregion _load_preview_edits
# region _compute_key_hash [TYPE Function]
# @PURPOSE: Compute a stable hash from source_data dict for matching preview edits.
@staticmethod
def _compute_key_hash(source_data: dict) -> str:
import hashlib
stable = json.dumps(source_data, sort_keys=True)
return hashlib.sha256(stable.encode()).hexdigest()[:16]
# endregion _compute_key_hash
# region _process_batch [TYPE Function]
# @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records.
# @PRE: job and batch_rows are valid.
@@ -452,17 +522,39 @@ class TranslationExecutor:
self.db, source_texts, job.id
)
# For each row, determine if we need LLM translation or can use approved translation
# For each row, determine if we need LLM translation or can use approved/preview edit
rows_for_llm = []
pre_translated = []
for row in batch_rows:
if row.get("approved_translation"):
pre_translated.append(row)
else:
rows_for_llm.append(row)
continue
# Check for preview edits carry-forward
source_data = row.get("source_data") or {}
if source_data and self._preview_edits_cache:
key_hash = self._compute_key_hash(source_data)
preview_edit = self._preview_edits_cache.get(key_hash)
if preview_edit:
# Use the first edited language's value as the approved translation
first_edit = next(iter(preview_edit.values()), None)
if first_edit:
row["approved_translation"] = first_edit
logger.reason("Using preview edit carry-forward", {
"key_hash": key_hash,
"langs": list(preview_edit.keys()),
})
pre_translated.append(row)
continue
rows_for_llm.append(row)
# Handle pre-translated (approved) rows
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
for row in pre_translated:
record = TranslationRecord(
id=str(uuid.uuid4()),
@@ -477,6 +569,20 @@ class TranslationExecutor:
status="SUCCESS",
)
self.db.add(record)
# Create per-language entry for each target language (pre-approved)
for lang_code in target_languages:
lang_entry = TranslationLanguage(
id=str(uuid.uuid4()),
record_id=record.id,
language_code=lang_code,
source_language_detected="und",
translated_value=row.get("approved_translation"),
final_value=row.get("approved_translation"),
status="translated",
needs_review=False,
)
self.db.add(lang_entry)
result["successful"] += 1
# Process rows needing LLM translation
@@ -524,21 +630,31 @@ class TranslationExecutor:
batch_id: str,
) -> dict[str, int]:
with belief_scope("TranslationExecutor._call_llm_for_batch"):
# Build dictionary section
# Build dictionary section using ContextAwarePromptBuilder
dictionary_section = ""
if dict_matches:
# Get row_context from first batch row if available
row_context = batch_rows[0].get("source_data") if batch_rows else None
# Use ContextAwarePromptBuilder for context-aware annotations
annotated_entries = ContextAwarePromptBuilder.build_context_entries(
dict_matches, row_context
)
glossary_lines = []
for m in dict_matches:
glossary_lines.append(
f"- '{m['source_term']}' -> '{m['target_term']}'"
f"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}"
)
for annotated_line in annotated_entries:
glossary_lines.append(f"- {annotated_line}")
dictionary_section = (
"Terminology dictionary (use these translations when applicable):\n"
+ "\n".join(glossary_lines)
+ "\n\n"
)
# Resolve target languages
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
target_languages_str = ", ".join(target_languages)
# Build rows JSON for LLM
rows_json = json.dumps([
{
@@ -548,12 +664,13 @@ class TranslationExecutor:
for idx, row in enumerate(batch_rows)
], indent=2)
# Build prompt
# Build prompt (use multi-language format)
prompt = render_prompt(
DEFAULT_EXECUTION_PROMPT_TEMPLATE,
{
"source_language": job.source_dialect or "SQL",
"target_language": job.target_language or job.target_dialect or "en",
"target_language": target_languages_str,
"target_languages": target_languages_str,
"source_dialect": job.source_dialect or "",
"target_dialect": job.target_dialect or "",
"translation_column": job.translation_column or "",
@@ -607,9 +724,9 @@ class TranslationExecutor:
self.db.add(record)
return {"successful": 0, "failed": len(batch_rows), "skipped": 0, "retries": retries}
# Parse LLM response
# Parse LLM response (multi-language aware)
try:
translations = self._parse_llm_response(llm_response, len(batch_rows))
translations = self._parse_llm_response(llm_response, len(batch_rows), target_languages=target_languages)
except ValueError as e:
# Parse failure — mark all rows as SKIPPED
logger.explore("LLM response parse failed", {
@@ -646,16 +763,20 @@ class TranslationExecutor:
for row in batch_rows:
row_id = str(row.get("row_index", ""))
translation = translations.get(row_id)
translation_data = translations.get(row_id)
source_text = row.get("source_text", "")
detected_lang = "und"
if translation_data:
detected_lang = translation_data.get("detected_source_language", "und")
if translation is None:
if translation_data is None:
# NULL translation — skip
skipped += 1
record = TranslationRecord(
id=str(uuid.uuid4()),
batch_id=batch_id,
run_id=run_id,
source_sql=row.get("source_text", ""),
source_sql=source_text,
target_sql="",
source_object_type="table_row",
source_object_id=row.get("row_index"),
@@ -667,14 +788,32 @@ class TranslationExecutor:
self.db.add(record)
continue
if translation.strip() == "":
# Empty translation — skip
# Collect per-language translated values
per_lang_values: dict[str, str] = {}
has_any_translation = False
# First try multi-language format (per-language keys)
for lang_code in target_languages:
lang_val = translation_data.get(lang_code)
if lang_val is not None and str(lang_val).strip():
per_lang_values[lang_code] = str(lang_val)
has_any_translation = True
# Fallback to legacy single "translation" key format
if not has_any_translation:
translation_text = translation_data.get("translation", "")
if translation_text.strip():
per_lang_values[target_languages[0]] = translation_text
has_any_translation = True
if not has_any_translation:
# Empty/all-empty translations — skip
skipped += 1
record = TranslationRecord(
id=str(uuid.uuid4()),
batch_id=batch_id,
run_id=run_id,
source_sql=row.get("source_text", ""),
source_sql=source_text,
target_sql="",
source_object_type="table_row",
source_object_id=row.get("row_index"),
@@ -686,13 +825,16 @@ class TranslationExecutor:
self.db.add(record)
continue
# Use the first language's value as the primary target_sql for backward compat
primary_translation = next(iter(per_lang_values.values()), "")
successful += 1
record = TranslationRecord(
id=str(uuid.uuid4()),
batch_id=batch_id,
run_id=run_id,
source_sql=row.get("source_text", ""),
target_sql=translation,
source_sql=source_text,
target_sql=primary_translation,
source_object_type="table_row",
source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
@@ -701,6 +843,37 @@ class TranslationExecutor:
)
self.db.add(record)
# Create per-language entries — one TranslationLanguage per language_code
for lang_code in target_languages:
lang_translation = per_lang_values.get(lang_code, "")
# Source-as-reference: if detected source language matches this target,
# store the original text verbatim (no translation needed)
if detected_lang != "und" and str(lang_code).lower() == str(detected_lang).lower():
lang_translation = source_text
# Check for undetermined source language
lang_needs_review = (detected_lang == "und")
if lang_needs_review:
logger.explore("undetected language", {
"record_id": row_id,
"language_code": lang_code,
"source_text": source_text[:100],
})
lang_entry = TranslationLanguage(
id=str(uuid.uuid4()),
record_id=record.id,
language_code=lang_code,
source_language_detected=detected_lang,
translated_value=lang_translation or "",
final_value=lang_translation or "",
status="translated",
needs_review=lang_needs_review,
)
self.db.add(lang_entry)
return {
"successful": successful,
"failed": failed,
@@ -806,11 +979,13 @@ class TranslationExecutor:
# endregion _call_openai_compatible
# region _parse_llm_response [TYPE Function]
# @PURPOSE: Parse LLM JSON response into dict of row_id -> translation.
# @PURPOSE: Parse LLM JSON response into dict of row_id -> per-language translations.
# Supports multi-language format: {"row_id": 0, "detected_source_language": "fr", "ru": "текст", "en": "text"}
# Backward-compat: old format {"row_id": 0, "translation": "text", "detected_source_language": "fr"}
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
# @POST: Returns dict mapping row_id to translation text.
# @POST: Returns dict mapping row_id to dict with 'detected_source_language' and per-language codes.
@staticmethod
def _parse_llm_response(response_text: str, expected_count: int) -> dict[str, str]:
def _parse_llm_response(response_text: str, expected_count: int, target_languages: list[str] | None = None) -> dict[str, dict[str, str]]:
with belief_scope("TranslationExecutor._parse_llm_response"):
try:
data = json.loads(response_text)
@@ -830,15 +1005,41 @@ class TranslationExecutor:
if not isinstance(rows, list):
raise ValueError("LLM response missing 'rows' array")
translations: dict[str, str] = {}
translations: dict[str, dict[str, str]] = {}
for item in rows:
row_id = str(item.get("row_id", ""))
translation = item.get("translation")
if translation is None:
# Skip NULL translations — they'll be handled by caller
if not row_id:
continue
if row_id:
translations[row_id] = str(translation)
detected_lang = str(item.get("detected_source_language", "und")) if item.get("detected_source_language") else "und"
result: dict[str, str] = {
"detected_source_language": detected_lang,
}
# Multi-language format: extract per-language keys
has_language_data = False
if target_languages:
for lang_code in target_languages:
lang_val = item.get(lang_code)
if lang_val is not None and str(lang_val).strip():
result[lang_code] = str(lang_val)
has_language_data = True
# Fallback to legacy single "translation" key format
if not has_language_data:
translation = item.get("translation")
if translation is not None:
result["translation"] = str(translation)
has_language_data = True
if has_language_data:
translations[row_id] = result
if len(translations) < expected_count:
logger.explore(
f"LLM returned fewer translations expected={expected_count} "
f"got={len(translations)}"
)
return translations
# endregion _parse_llm_response

View File

@@ -21,6 +21,7 @@ from ...models.translate import (
MetricSnapshot,
TranslationEvent,
TranslationRun,
TranslationRunLanguageStats,
TranslationSchedule,
)
@@ -121,17 +122,37 @@ class TranslationMetrics:
# indicates cutoff
pass
# Live events (<90 days) for token/cost
cutoff = datetime.now(UTC)
live_events = (
self.db.query(TranslationEvent)
.filter(
TranslationEvent.job_id == job_id,
TranslationEvent.event_type.in_(["TRANSLATION_PHASE_COMPLETED", "RUN_COMPLETED"]),
TranslationEvent.created_at > cutoff, # events newer than snapshot
)
# Per-language metrics from TranslationRunLanguageStats (live runs)
per_language: dict[str, dict[str, int | float]] = {}
lang_stats_rows = (
self.db.query(TranslationRunLanguageStats)
.join(TranslationRun, TranslationRunLanguageStats.run_id == TranslationRun.id)
.filter(TranslationRun.job_id == job_id)
.all()
)
for ls in lang_stats_rows:
lang = ls.language_code
if lang not in per_language:
per_language[lang] = {"tokens": 0, "cost": 0.0, "runs": 0, "translated_rows": 0}
per_language[lang]["tokens"] += ls.token_count or 0
per_language[lang]["cost"] += ls.estimated_cost or 0.0
per_language[lang]["runs"] += 1
per_language[lang]["translated_rows"] += ls.translated_rows or 0
# Merge in per-language data from MetricSnapshot (pruned period)
latest_snapshot = (
self.db.query(MetricSnapshot)
.filter(MetricSnapshot.job_id == job_id)
.order_by(MetricSnapshot.snapshot_date.desc())
.first()
)
if latest_snapshot and latest_snapshot.per_language_metrics:
for lang, snap_data in latest_snapshot.per_language_metrics.items():
if lang not in per_language:
per_language[lang] = {"tokens": 0, "cost": 0.0, "runs": 0, "translated_rows": 0}
per_language[lang]["tokens"] += snap_data.get("cumulative_tokens", 0)
per_language[lang]["cost"] += snap_data.get("cumulative_cost", 0.0)
per_language[lang]["runs"] += snap_data.get("runs", 0)
return {
"job_id": job_id,
@@ -148,6 +169,7 @@ class TranslationMetrics:
"avg_duration_ms": int(avg_duration) if avg_duration else None,
"last_run_at": last_run.created_at.isoformat() if last_run else None,
"next_scheduled_run": next_schedule.last_run_at.isoformat() if next_schedule and next_schedule.last_run_at else None,
"per_language_metrics": per_language,
}
# endregion get_job_metrics

View File

@@ -30,9 +30,11 @@ from ...core.logger import belief_scope, logger
from ...models.translate import (
TranslationBatch,
TranslationJob,
TranslationLanguage,
TranslationPreviewSession,
TranslationRecord,
TranslationRun,
TranslationRunLanguageStats,
)
from .events import TranslationEventLog
from .executor import TranslationExecutor
@@ -201,6 +203,27 @@ class TranslationOrchestrator:
created_by=self.current_user,
)
# Initialize per-language stats
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
language_stats_map: dict[str, TranslationRunLanguageStats] = {}
for lang_code in target_languages:
lang_stat = TranslationRunLanguageStats(
id=str(uuid.uuid4()),
run_id=run.id,
language_code=lang_code,
total_rows=0,
translated_rows=0,
failed_rows=0,
skipped_rows=0,
token_count=0,
estimated_cost=0.0,
)
self.db.add(lang_stat)
language_stats_map[lang_code] = lang_stat
self.db.flush()
# Dispatch executor
executor = TranslationExecutor(
self.db, self.config_manager, self.current_user,
@@ -227,6 +250,9 @@ class TranslationOrchestrator:
self.db.commit()
return run
# Aggregate per-language statistics after executor completes
self._update_language_stats(run.id, language_stats_map)
# Record translation phase complete
self.event_log.log_event(
job_id=job.id,
@@ -745,6 +771,25 @@ class TranslationOrchestrator:
# Get event summary
event_summary = self.event_log.get_run_event_summary(run_id)
# Get language stats
language_stats_entries = (
self.db.query(TranslationRunLanguageStats)
.filter(TranslationRunLanguageStats.run_id == run_id)
.all()
)
language_stats = [
{
"language_code": ls.language_code,
"total_rows": ls.total_rows or 0,
"translated_rows": ls.translated_rows or 0,
"failed_rows": ls.failed_rows or 0,
"skipped_rows": ls.skipped_rows or 0,
"token_count": ls.token_count or 0,
"estimated_cost": ls.estimated_cost or 0.0,
}
for ls in language_stats_entries
]
return {
"id": run.id,
"job_id": run.job_id,
@@ -759,6 +804,7 @@ class TranslationOrchestrator:
"insert_status": run.insert_status,
"superset_execution_id": run.superset_execution_id,
"batch_count": batch_count,
"language_stats": language_stats,
"event_invariants": {
"has_run_started": event_summary["has_run_started"],
"terminal_event_count": event_summary["terminal_event_count"],
@@ -863,6 +909,80 @@ class TranslationOrchestrator:
]
# endregion get_run_history
# region _update_language_stats [TYPE Function]
# @PURPOSE: Aggregate TranslationLanguage entries by language_code and update TranslationRunLanguageStats.
# @PRE: run_id and language_stats_map are valid. DB session is available.
# @POST: Language stats are updated with row counts and estimated tokens/cost.
# @SIDE_EFFECT: DB writes on language_stats objects.
def _update_language_stats(
self,
run_id: str,
language_stats_map: dict[str, TranslationRunLanguageStats],
) -> None:
with belief_scope("TranslationOrchestrator._update_language_stats"):
# Get all records for this run to join with TranslationLanguage
records = (
self.db.query(TranslationRecord)
.filter(TranslationRecord.run_id == run_id)
.all()
)
record_ids = [r.id for r in records]
if not record_ids:
logger.reason("No records for language stats aggregation", {"run_id": run_id})
return
# Get all language entries for this run's records
lang_entries = (
self.db.query(TranslationLanguage)
.filter(TranslationLanguage.record_id.in_(record_ids))
.all()
)
# Aggregate by language_code
from collections import defaultdict
agg: dict[str, dict[str, int]] = defaultdict(lambda: {"total": 0, "translated": 0, "failed": 0, "skipped": 0})
for le in lang_entries:
code = le.language_code
agg[code]["total"] += 1
if le.status in ("translated", "approved", "edited"):
agg[code]["translated"] += 1
elif le.status == "failed":
agg[code]["failed"] += 1
elif le.status == "skipped":
agg[code]["skipped"] += 1
# Estimate tokens: heuristic based on character count of translated values
total_chars = sum(
len(le.translated_value or "") for le in lang_entries if le.translated_value
)
total_tokens = max(1, total_chars // 4) # ~4 chars per token
cost_per_token = 0.002 / 1000 # $0.002 per 1K tokens
# Update each language stat entry
for lang_code, lang_stat in language_stats_map.items():
data = agg.get(lang_code, {"total": 0, "translated": 0, "failed": 0, "skipped": 0})
lang_stat.total_rows = data["total"]
lang_stat.translated_rows = data["translated"]
lang_stat.failed_rows = data["failed"]
lang_stat.skipped_rows = data["skipped"]
# Proportional token split: share tokens across languages
num_langs = len(language_stats_map)
if num_langs > 0:
lang_stat.token_count = total_tokens // num_langs
lang_stat.estimated_cost = round((lang_stat.token_count / 1000) * cost_per_token, 6)
self.db.flush()
logger.reason("Language stats updated", {
"run_id": run_id,
"languages": list(language_stats_map.keys()),
"total_tokens_est": total_tokens,
})
# endregion _update_language_stats
# region _compute_config_hash [TYPE Function]
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
@staticmethod

View File

@@ -29,6 +29,7 @@ from ...core.superset_client import SupersetClient
from ...models.translate import (
TranslationJob,
TranslationJobDictionary,
TranslationPreviewLanguage,
TranslationPreviewRecord,
TranslationPreviewSession,
)
@@ -38,16 +39,19 @@ from .dictionary import DictionaryManager
# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [TYPE Constant]
# @BRIEF Default prompt template for batch LLM translation execution (no context columns — faster).
# Supports both single-language and multi-language modes via {target_languages} placeholder.
DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
"Translate the following database content from {source_language} to {target_language}.\n\n"
"Translate the following database content from {source_language} to the following language(s): {target_languages}.\n\n"
"Source dialect: {source_dialect}\n"
"Target dialect: {target_dialect}\n"
"Target dialect(s): {target_dialect}\n"
"Column to translate: {translation_column}\n\n"
"{dictionary_section}"
"For each row, provide an accurate translation of the text.\n\n"
"For each row, provide an accurate translation of the text into each target language.\n\n"
"Rows to translate:\n{rows_json}\n\n"
"Respond with a JSON object in this exact format:\n"
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
'{{"rows": [{{"row_id": "<row_index>", "detected_source_language": "<bcp47_or_und>", "<language_code_1>": "<translation_in_lang_1>", "<language_code_2>": "<translation_in_lang_2>"}}]}}\n'
"For each row, return the detected source language as a BCP-47 tag (e.g. 'en', 'ru', 'fr'), or 'und' if uncertain.\n"
"Include a separate key for EACH target language code with the translated text in that language.\n"
"Each row_id must match the index provided. Return exactly {row_count} entries."
)
# #endregion DEFAULT_EXECUTION_PROMPT_TEMPLATE
@@ -56,16 +60,18 @@ DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [TYPE Constant]
# @BRIEF Default prompt template for LLM translation preview.
DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
"Translate the following database content from {source_language} to {target_language}.\n\n"
"Translate the following database content.\n\n"
"Source dialect: {source_dialect}\n"
"Target dialect: {target_dialect}\n"
"Column to translate: {translation_column}\n\n"
"{dictionary_section}"
"For each row, provide an accurate translation of the '{translation_column}' value.\n"
"Translate to the following languages: {target_languages}\n\n"
"For each row, provide an accurate translation of the '{translation_column}' value into each language.\n"
"Consider the context columns when determining the meaning of the text.\n\n"
"Rows to translate:\n{rows_json}\n\n"
"Respond with a JSON object in this exact format:\n"
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
'{{"rows": [{{"row_id": "<row_index>", "detected_source_language": "<bcp47_or_und>", "language_code_1": "<translation_in_lang_1>", "language_code_2": "<translation_in_lang_2>"}}]}}\n'
"For each row, return the detected source language as a BCP-47 tag (e.g. 'en', 'ru', 'fr'), or 'und' if uncertain.\n"
"Include a separate key for EACH target language code with the translated text in that language.\n"
"Each row_id must match the index provided. Return exactly {row_count} entries."
)
# #endregion DEFAULT_PREVIEW_PROMPT_TEMPLATE
@@ -80,7 +86,9 @@ class TokenEstimator:
CHARS_PER_TOKEN_ESTIMATE: float = 4.0
OUTPUT_TOKENS_PER_ROW_ESTIMATE: int = 50
MULTI_LANG_FACTOR: float = 1.2 # Overhead for multi-language in one call
TOKEN_COST_PER_1K: float = 0.002 # Default cost per 1K tokens
COST_WARNING_THRESHOLD: int = 30 # Show warning above this sample size
# region estimate_prompt_tokens [TYPE Function]
# @PURPOSE: Estimate token count for a prompt string.
@@ -94,12 +102,12 @@ class TokenEstimator:
# endregion estimate_prompt_tokens
# region estimate_output_tokens [TYPE Function]
# @PURPOSE: Estimate output token count for translating N rows.
# @PRE: row_count >= 0.
# @PURPOSE: Estimate output token count for translating N rows across N languages.
# @PRE: row_count >= 0, num_languages >= 1.
# @POST: Returns estimated output token count.
@staticmethod
def estimate_output_tokens(row_count: int) -> int:
return row_count * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE
def estimate_output_tokens(row_count: int, num_languages: int = 1) -> int:
return int(row_count * num_languages * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE * TokenEstimator.MULTI_LANG_FACTOR)
# endregion estimate_output_tokens
# region estimate_cost [TYPE Function]
@@ -112,6 +120,20 @@ class TokenEstimator:
return round((total_tokens / 1000) * rate, 6)
# endregion estimate_cost
# region check_cost_warning [TYPE Function]
# @PURPOSE: Generate cost warning for large previews.
# @PRE: sample_size > 0, num_languages >= 1.
# @POST: Returns warning string or None.
@staticmethod
def check_cost_warning(sample_size: int, num_languages: int, estimated_tokens: int, estimated_cost: float) -> str | None:
if sample_size > TokenEstimator.COST_WARNING_THRESHOLD:
return (
f"Large preview — estimated {estimated_tokens} tokens, ~${estimated_cost:.4f} cost "
f"(across {num_languages} languages)"
)
return None
# endregion check_cost_warning
# #endregion TokenEstimator
@@ -134,10 +156,10 @@ class TranslationPreview:
self.current_user = current_user
# region preview_rows [TYPE Function]
# @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.
# @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for multi-language translation, create preview session with per-language records.
# @PRE: job_id exists and job has source_datasource_id, translation_column configured.
# @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session.
# @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.
# @POST: Returns TranslationPreviewResponse with per-language records, cost estimation, and persistent session.
# @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord + TranslationPreviewLanguage rows.
def preview_rows(
self,
job_id: str,
@@ -157,6 +179,12 @@ class TranslationPreview:
if not job.translation_column:
raise ValueError("Job must have a translation column configured for preview")
# Resolve target languages: prefer target_languages list, fallback to single target_language
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
num_languages = len(target_languages)
# 2. Compute config hash and dict snapshot hash
config_hash = self._compute_config_hash(job)
dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id)
@@ -224,40 +252,48 @@ class TranslationPreview:
+ "\n\n"
)
# 6. Build LLM prompt
# 6. Build LLM prompt with multi-language instructions
rows_json = json.dumps([
{"row_id": str(m["row_index"]), "text": m["source_text"], "context": m["context_data"]}
for m in row_meta
], indent=2)
target_languages_str = ", ".join(target_languages)
template = prompt_template or DEFAULT_PREVIEW_PROMPT_TEMPLATE
prompt = render_prompt(template, {
"source_language": job.source_dialect or "SQL",
"target_language": job.target_language or job.target_dialect or "en",
"target_language": target_languages_str,
"source_dialect": job.source_dialect or "",
"target_dialect": job.target_dialect or "",
"target_languages": target_languages_str,
"translation_column": job.translation_column or "",
"dictionary_section": dictionary_section,
"rows_json": rows_json,
"row_count": str(actual_row_count),
})
# 7. Estimate tokens/cost for sample and full dataset
# 7. Estimate tokens/cost for sample with multi-language factor
sample_prompt_tokens = TokenEstimator.estimate_prompt_tokens(prompt)
sample_output_tokens = TokenEstimator.estimate_output_tokens(actual_row_count)
sample_output_tokens = TokenEstimator.estimate_output_tokens(actual_row_count, num_languages)
sample_total_tokens = sample_prompt_tokens + sample_output_tokens
sample_cost = TokenEstimator.estimate_cost(sample_total_tokens)
# Estimate full dataset cost (if we knew total rows)
# Estimate full dataset cost (extrapolated)
total_est_rows = sample_size * 10 # rough extrapolation
total_est_tokens = TokenEstimator.estimate_prompt_tokens(
prompt.replace(str(actual_row_count), "{total}")
) + TokenEstimator.estimate_output_tokens(sample_size * 10) # rough extrapolation
prompt.replace(str(actual_row_count), str(total_est_rows))
) + TokenEstimator.estimate_output_tokens(total_est_rows, num_languages)
total_est_cost = TokenEstimator.estimate_cost(total_est_tokens)
# Cost warning for large previews
cost_warning = TokenEstimator.check_cost_warning(
sample_size, num_languages, sample_total_tokens, sample_cost
)
# 8. Call LLM
logger.reason("Calling LLM for preview translation", {
"provider_id": job.provider_id,
"row_count": actual_row_count,
"num_languages": num_languages,
"estimated_tokens": sample_total_tokens,
})
llm_response = self._call_llm(
@@ -265,8 +301,10 @@ class TranslationPreview:
prompt=prompt,
)
# 9. Parse LLM response
translations = self._parse_llm_response(llm_response, actual_row_count)
# 9. Parse LLM response (multi-language)
translations = self._parse_llm_response(
llm_response, actual_row_count, target_languages=target_languages
)
# 10. Create preview session
session = TranslationPreviewSession(
@@ -280,14 +318,50 @@ class TranslationPreview:
self.db.add(session)
self.db.flush()
# 11. Create preview records
# 11. Create preview records with per-language entries
records = []
for meta in row_meta:
idx = meta["row_index"]
translation = translations.get(str(idx), "")
is_rejected = False
status = "PENDING"
feedback = None
source_text = meta["source_text"]
translation_data = translations.get(str(idx), {})
detected_lang = translation_data.get("detected_source_language", "und") if isinstance(translation_data, dict) else "und"
# Extract per-language translations
lang_entries: list[TranslationPreviewLanguage] = []
for lang_code in target_languages:
# Get translation for this language from LLM response
lang_translation = translation_data.get(lang_code) if isinstance(translation_data, dict) else None
# Fallback: if no per-language data, use the single "translation" key
if not lang_translation:
if isinstance(translation_data, dict):
lang_translation = translation_data.get("translation", "")
elif isinstance(translation_data, str):
lang_translation = translation_data
else:
lang_translation = ""
# Source-as-reference: if this language code matches source, use original text
if str(lang_code).lower() == str(detected_lang).lower() and detected_lang != "und":
# Use source text as reference (no translation needed for source language)
lang_translation = source_text
lang_needs_review = detected_lang == "und"
lang_entry = TranslationPreviewLanguage(
id=str(uuid.uuid4()),
preview_record_id="", # Will set after record is created
language_code=lang_code,
source_language_detected=detected_lang,
translated_value=str(lang_translation),
final_value=str(lang_translation),
status="pending",
needs_review=lang_needs_review,
)
lang_entries.append(lang_entry)
# Determine overall status and needs_review for the record
overall_needs_review = any(le.needs_review for le in lang_entries)
# Extract source_data: store original row key columns for upsert matching
source_row = meta.get("source_row", {})
@@ -299,24 +373,41 @@ class TranslationPreview:
if k in source_row
}
elif source_row:
# No key columns configured — store the full row as fallback
source_data = dict(source_row)
# Use first language's translation as target_sql for backward compat
primary_translation = lang_entries[0].translated_value if lang_entries else ""
record = TranslationPreviewRecord(
id=str(uuid.uuid4()),
session_id=session.id,
source_sql=meta["source_text"],
target_sql=translation,
source_sql=source_text,
target_sql=primary_translation,
source_object_type="table_row",
source_object_id=str(idx),
source_object_name=f"Row {idx + 1}",
source_data=source_data,
status=status,
feedback=feedback,
status="PENDING",
feedback=None,
created_at=datetime.now(UTC),
)
self.db.add(record)
self.db.flush()
# Now set the preview_record_id for all lang entries and persist
serialized_langs = []
for le in lang_entries:
le.preview_record_id = record.id
self.db.add(le)
serialized_langs.append({
"language_code": le.language_code,
"source_language_detected": le.source_language_detected,
"translated_value": le.translated_value,
"final_value": le.final_value,
"status": le.status,
"needs_review": le.needs_review,
})
records.append({
"id": record.id,
"source_sql": record.source_sql,
@@ -326,6 +417,9 @@ class TranslationPreview:
"source_object_name": record.source_object_name,
"status": record.status,
"feedback": record.feedback,
"source_language_detected": detected_lang,
"needs_review": overall_needs_review,
"languages": serialized_langs,
})
self.db.commit()
@@ -338,15 +432,18 @@ class TranslationPreview:
"created_at": session.created_at.isoformat(),
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
"records": records,
"target_languages": target_languages,
"cost_estimate": {
"sample_size": actual_row_count,
"num_languages": num_languages,
"sample_prompt_tokens": sample_prompt_tokens,
"sample_output_tokens": sample_output_tokens,
"sample_total_tokens": sample_total_tokens,
"sample_cost": sample_cost,
"estimated_total_rows": actual_row_count * 10,
"estimated_total_rows": total_est_rows,
"estimated_tokens": total_est_tokens,
"estimated_cost": total_est_cost,
"warning": cost_warning,
},
"config_hash": config_hash,
"dict_snapshot_hash": dict_snapshot_hash,
@@ -355,15 +452,16 @@ class TranslationPreview:
logger.reflect("Preview completed", {
"session_id": session.id,
"row_count": actual_row_count,
"num_languages": num_languages,
"sample_cost": sample_cost,
})
return result
# endregion preview_rows
# region update_preview_row [TYPE Function]
# @PURPOSE: Approve, edit, or reject an individual preview row.
# @PURPOSE: Approve, edit, or reject an individual preview row (optionally per language).
# @PRE: session_id and row_id exist, session is ACTIVE.
# @POST: PreviewRecord status is updated.
# @POST: PreviewRecord status is updated. If language_code provided, only that TranslationPreviewLanguage is updated.
def update_preview_row(
self,
job_id: str,
@@ -371,6 +469,7 @@ class TranslationPreview:
action: str,
translation: str | None = None,
feedback: str | None = None,
language_code: str | None = None,
) -> dict[str, Any]:
with belief_scope("TranslationPreview.update_preview_row"):
# Find the active session for this job
@@ -397,16 +496,76 @@ class TranslationPreview:
if not record:
raise ValueError(f"Preview record '{row_id}' not found in active session")
if action == "approve":
record.status = "APPROVED"
elif action == "reject":
record.status = "REJECTED"
elif action == "edit":
record.status = "APPROVED"
if translation is not None:
record.target_sql = translation
# If language_code specified, operate on the specific TranslationPreviewLanguage
if language_code:
lang_entry = (
self.db.query(TranslationPreviewLanguage)
.filter(
TranslationPreviewLanguage.preview_record_id == row_id,
TranslationPreviewLanguage.language_code == language_code,
)
.first()
)
if not lang_entry:
raise ValueError(f"Language entry '{language_code}' not found for record '{row_id}'")
if action == "approve":
lang_entry.status = "approved"
elif action == "reject":
lang_entry.status = "rejected"
elif action == "edit":
lang_entry.status = "edited"
if translation is not None:
lang_entry.translated_value = translation
lang_entry.final_value = translation
lang_entry.user_edit = translation
else:
raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.")
# Update record-level status based on all language entries
all_langs = (
self.db.query(TranslationPreviewLanguage)
.filter(TranslationPreviewLanguage.preview_record_id == row_id)
.all()
)
all_approved = all(le.status in ("approved", "edited") for le in all_langs)
any_rejected = any(le.status == "rejected" for le in all_langs)
if all_approved:
record.status = "APPROVED"
elif any_rejected and not all_langs:
record.status = "REJECTED"
# Update target_sql to reflect primary (first) language edit
if action == "edit" and translation is not None and all_langs:
record.target_sql = all_langs[0].final_value or all_langs[0].translated_value
else:
raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.")
# Legacy behavior: operate on whole record (all languages)
if action == "approve":
record.status = "APPROVED"
# Approve all pending language entries
for lang_entry in record.languages:
if lang_entry.status == "pending":
lang_entry.status = "approved"
elif action == "reject":
record.status = "REJECTED"
for lang_entry in record.languages:
if lang_entry.status == "pending":
lang_entry.status = "rejected"
elif action == "edit":
record.status = "APPROVED"
if translation is not None:
record.target_sql = translation
# Update primary (first) language entry
if record.languages:
first_lang = record.languages[0]
first_lang.translated_value = translation
first_lang.final_value = translation
first_lang.user_edit = translation
first_lang.status = "edited"
else:
raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.")
if feedback is not None:
record.feedback = feedback
@@ -418,21 +577,37 @@ class TranslationPreview:
"row_id": row_id,
"session_id": session.id,
"status": record.status,
"language_code": language_code,
})
# Build response with per-language data
lang_responses = []
if record.languages:
for le in record.languages:
lang_responses.append({
"language_code": le.language_code,
"source_language_detected": le.source_language_detected,
"translated_value": le.translated_value,
"final_value": le.final_value,
"user_edit": le.user_edit,
"status": le.status,
"needs_review": le.needs_review,
})
return {
"id": record.id,
"source_sql": record.source_sql,
"target_sql": record.target_sql,
"status": record.status,
"feedback": record.feedback,
"languages": lang_responses,
}
# endregion update_preview_row
# region accept_preview_session [TYPE Function]
# @PURPOSE: Mark a preview session as accepted, which gates full execution.
# @PRE: job_id has an ACTIVE preview session.
# @POST: Session status changes to APPLIED.
# @POST: Session status changes to APPLIED. User edits are persisted for carry-forward.
# @SIDE_EFFECT: Future full execution calls will check for accepted session.
def accept_preview_session(self, job_id: str) -> dict[str, Any]:
with belief_scope("TranslationPreview.accept_preview_session"):
@@ -464,6 +639,12 @@ class TranslationPreview:
.all()
)
# Resolve target languages
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"] if job else ["en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
return {
"id": session.id,
"job_id": job_id,
@@ -471,6 +652,7 @@ class TranslationPreview:
"created_by": session.created_by,
"created_at": session.created_at.isoformat(),
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
"target_languages": target_languages,
"records": [
{
"id": r.id,
@@ -478,6 +660,28 @@ class TranslationPreview:
"target_sql": r.target_sql,
"status": r.status,
"feedback": r.feedback,
"source_language_detected": (
r.languages[0].source_language_detected
if r.languages and r.languages[0].source_language_detected
else None
),
"needs_review": (
r.languages[0].needs_review
if r.languages
else False
),
"languages": [
{
"language_code": le.language_code,
"source_language_detected": le.source_language_detected,
"translated_value": le.translated_value,
"final_value": le.final_value,
"user_edit": le.user_edit,
"status": le.status,
"needs_review": le.needs_review,
}
for le in (r.languages or [])
] if r.languages else [],
}
for r in records
],
@@ -487,7 +691,7 @@ class TranslationPreview:
# region get_preview_session [TYPE Function]
# @PURPOSE: Get the latest preview session for a job with its records.
# @PRE: job_id exists.
# @POST: Returns session data with records or raises ValueError.
# @POST: Returns session data with per-language records or raises ValueError.
def get_preview_session(self, job_id: str) -> dict[str, Any]:
with belief_scope("TranslationPreview.get_preview_session"):
session = (
@@ -505,6 +709,12 @@ class TranslationPreview:
.all()
)
# Resolve target languages
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"] if job else ["en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
return {
"id": session.id,
"job_id": job_id,
@@ -512,6 +722,7 @@ class TranslationPreview:
"created_by": session.created_by,
"created_at": session.created_at.isoformat(),
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
"target_languages": target_languages,
"records": [
{
"id": r.id,
@@ -522,6 +733,28 @@ class TranslationPreview:
"source_object_name": r.source_object_name,
"status": r.status,
"feedback": r.feedback,
"source_language_detected": (
r.languages[0].source_language_detected
if r.languages and r.languages[0].source_language_detected
else None
),
"needs_review": (
r.languages[0].needs_review
if r.languages
else False
),
"languages": [
{
"language_code": le.language_code,
"source_language_detected": le.source_language_detected,
"translated_value": le.translated_value,
"final_value": le.final_value,
"user_edit": le.user_edit,
"status": le.status,
"needs_review": le.needs_review,
}
for le in (r.languages or [])
] if r.languages else [],
}
for r in records
],
@@ -752,11 +985,11 @@ class TranslationPreview:
# endregion _call_openai_compatible
# region _parse_llm_response [TYPE Function]
# @PURPOSE: Parse the LLM JSON response into a dict of row_id -> translation.
# @PURPOSE: Parse the LLM JSON response into a dict of row_id -> per-language translations.
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
# @POST: Returns dict mapping string row_id to translation text.
# @POST: Returns dict mapping row_id to dict with 'detected_source_language' and per-language keys.
@staticmethod
def _parse_llm_response(response_text: str, expected_count: int) -> dict[str, str]:
def _parse_llm_response(response_text: str, expected_count: int, target_languages: list[str] | None = None) -> dict[str, dict[str, str]]:
with belief_scope("TranslationPreview._parse_llm_response"):
logger.reason(f"Raw LLM response length={len(response_text)} preview={response_text[:500]}")
@@ -779,12 +1012,37 @@ class TranslationPreview:
logger.explore(f"LLM response has no 'rows' array, keys={list(data.keys())} text_preview={response_text[:300]}")
raise ValueError("LLM response missing 'rows' array")
translations: dict[str, str] = {}
translations: dict[str, dict[str, str]] = {}
for item in rows:
row_id = str(item.get("row_id", ""))
translation = str(item.get("translation", ""))
if row_id:
translations[row_id] = translation
if not row_id:
continue
detected_lang = str(item.get("detected_source_language", "und")) if item.get("detected_source_language") else "und"
result: dict[str, str] = {
"detected_source_language": detected_lang,
}
# Multi-language format: extract per-language keys
# Keys that are NOT row_id, detected_source_language, or translation are language codes
has_language_data = False
if target_languages:
for lang_code in target_languages:
lang_val = item.get(lang_code)
if lang_val is not None and str(lang_val).strip():
result[lang_code] = str(lang_val)
has_language_data = True
# Fallback to old format (single "translation" key)
if not has_language_data:
translation = item.get("translation")
if translation is not None:
result["translation"] = str(translation)
else:
# Skip rows with no translation data at all
continue
translations[row_id] = result
if len(translations) < expected_count:
logger.explore(

View File

@@ -0,0 +1,152 @@
# #region ContextAwarePromptBuilder [C:2] [TYPE Module] [SEMANTICS translate, prompt, context, dictionary]
# @BRIEF Pure-function prompt builder that enhances dictionary entries with context annotations.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [DictionaryEntry:Class]
# @RATIONALE: Pure functions only — no I/O, no DB access. Separated from executor for testability.
# @REJECTED: Embedding context inline in the executor would make it untestable without mocking DB.
#
# Typical workflow:
# 1. Call build_context_entries(dictionary_entries, row_context) to get annotated, prioritized entries
# 2. Each entry is rendered via render_entry() with optional priority flag
# 3. Jaccard similarity >= 0.5 triggers priority flagging
import json
from typing import Any
# #region ContextAwarePromptBuilder [C:2] [TYPE Class]
# @BRIEF Build LLM prompts with context-aware dictionary entries and similarity-based priority.
class ContextAwarePromptBuilder:
"""Build LLM prompts with context-aware dictionary entries.
Pure function — no I/O, no DB access.
"""
@staticmethod
def render_entry(entry: Any, priority: bool = False, row_context: dict | None = None) -> str:
"""Render a dictionary entry for the LLM prompt.
Args:
entry: DictionaryEntry-like object (must have source_term, target_term, has_context,
context_data, usage_notes attrs or dict keys).
priority: Whether this entry should be flagged as high priority.
row_context: Optional row context dict (unused in rendering, kept for API symmetry).
Returns:
Rendered prompt line string.
"""
# Support both object attrs and dict access
if isinstance(entry, dict):
source_term = entry.get("source_term", "")
target_term = entry.get("target_term", "")
has_context = entry.get("has_context", False)
context_data = entry.get("context_data")
usage_notes = entry.get("usage_notes")
else:
source_term = getattr(entry, "source_term", "")
target_term = getattr(entry, "target_term", "")
has_context = getattr(entry, "has_context", False)
context_data = getattr(entry, "context_data", None)
usage_notes = getattr(entry, "usage_notes", None)
# Build base line
line = f'"{source_term}" -> "{target_term}"'
# Add context annotation if present
if has_context and context_data:
context_items = []
if isinstance(context_data, dict):
context_items = [f"{k}={v}" for k, v in context_data.items()]
elif isinstance(context_data, str):
try:
parsed = json.loads(context_data)
if isinstance(parsed, dict):
context_items = [f"{k}={v}" for k, v in parsed.items()]
else:
context_items = [str(context_data)]
except (json.JSONDecodeError, TypeError):
context_items = [str(context_data)]
context_str = ", ".join(context_items)
# Truncate if too long (500 tokens ≈ 2000 chars)
if len(context_str) > 2000:
context_str = context_str[:1997] + "...[truncated]"
line = f'"{source_term}" (context: {context_str}) -> "{target_term}"'
# Add usage notes
if usage_notes:
notes = str(usage_notes)[:200] # cap at 200 chars
line += f" # Usage: {notes}"
# Add priority prefix
if priority:
line = f"# PRIORITY (context match) — {line}"
return line
@staticmethod
def compute_context_similarity(entry_context: dict | None, row_context: dict | None) -> float:
"""Jaccard similarity between entry context and row context. Returns 0.0-1.0.
Compares the sets of lowercased string values from both contexts.
Returns 1.0 for identical contexts, 0.0 for disjoint or missing.
"""
if not entry_context or not row_context:
return 0.0
entry_vals = set(str(v).lower() for v in entry_context.values() if v is not None)
row_vals = set(str(v).lower() for v in row_context.values() if v is not None)
if not entry_vals or not row_vals:
return 0.0
intersection = entry_vals & row_vals
union = entry_vals | row_vals
return len(intersection) / len(union)
@staticmethod
def build_context_entries(
dictionary_entries: list[Any],
row_context: dict | None = None,
) -> list[str]:
"""Build prioritized dictionary entry list with context annotations.
Args:
dictionary_entries: List of DictionaryEntry-like objects or dicts.
row_context: Optional dict of current row's context columns.
Returns:
List of rendered prompt strings, sorted with priority entries first.
"""
results: list[tuple[Any, bool]] = []
for entry in dictionary_entries:
priority = False
if row_context:
# Extract entry context_data
if isinstance(entry, dict):
entry_context = entry.get("context_data")
else:
entry_context = getattr(entry, "context_data", None)
if entry_context:
similarity = ContextAwarePromptBuilder.compute_context_similarity(
entry_context, row_context
)
priority = similarity >= 0.5
results.append((entry, priority))
# Sort: priority first, then non-priority
results.sort(key=lambda x: (not x[1]))
return [
ContextAwarePromptBuilder.render_entry(entry, priority, row_context)
for entry, priority in results
]
# #endregion ContextAwarePromptBuilder
# #endregion ContextAwarePromptBuilder

View File

@@ -10,6 +10,7 @@
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity.
import re
import uuid
from datetime import UTC, datetime
from typing import Any
@@ -27,6 +28,7 @@ from ...schemas.translate import (
TranslateJobResponse,
TranslateJobUpdate,
)
from .dictionary import _validate_bcp47
# Supported database dialects for translation
SUPPORTED_DIALECTS = {
@@ -227,6 +229,16 @@ class TranslateJobService:
logger.warning(f"[TranslateJobService] Dialect detection failed: {e}")
dialect = payload.source_dialect
# Resolve target_languages: accept new format (list) or legacy single language
target_languages = payload.target_languages
if not target_languages and payload.target_language:
target_languages = [payload.target_language]
if target_languages:
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
for lang in target_languages:
_validate_bcp47(lang, "target_languages")
# Build job instance
job = TranslationJob(
id=str(uuid.uuid4()),
@@ -245,6 +257,7 @@ class TranslateJobService:
target_column=payload.target_column,
context_columns=payload.context_columns or [],
target_language=payload.target_language,
target_languages=target_languages,
provider_id=payload.provider_id,
batch_size=payload.batch_size,
upsert_strategy=payload.upsert_strategy,
@@ -284,6 +297,22 @@ class TranslateJobService:
update_data = payload.model_dump(exclude_unset=True)
dict_ids = update_data.pop("dictionary_ids", None)
# Backward compat: if only target_language is set (old API), wrap into target_languages
if "target_language" in update_data and "target_languages" not in update_data:
target_languages = [update_data["target_language"]] if update_data["target_language"] else []
update_data["target_languages"] = target_languages
elif "target_languages" in update_data and "target_language" not in update_data:
# Keep deprecated field in sync
if update_data["target_languages"]:
update_data["target_language"] = update_data["target_languages"][0]
# Validate BCP-47 for target_languages
if update_data.get("target_languages"):
if not isinstance(update_data["target_languages"], list):
update_data["target_languages"] = [str(update_data["target_languages"])]
for lang in update_data["target_languages"]:
_validate_bcp47(lang, "target_languages")
for field, value in update_data.items():
if hasattr(job, field):
setattr(job, field, value)
@@ -366,6 +395,7 @@ class TranslateJobService:
target_column=source.target_column,
context_columns=source.context_columns,
target_language=source.target_language,
target_languages=source.target_languages,
provider_id=source.provider_id,
batch_size=source.batch_size,
upsert_strategy=source.upsert_strategy,
@@ -468,6 +498,7 @@ def job_to_response(job: TranslationJob, dict_ids: list[str] | None = None) -> T
target_column=job.target_column,
context_columns=job.context_columns or [],
target_language=job.target_language,
target_languages=job.target_languages,
provider_id=job.provider_id,
batch_size=job.batch_size or 50,
upsert_strategy=job.upsert_strategy or "MERGE",
@@ -546,5 +577,462 @@ def get_datasource_columns(
)
# #endregion DatasourceColumnsService
# #region InlineCorrectionService [C:3] [TYPE Class] [SEMANTICS translate, correction, inline, dictionary]
# @BRIEF Service for inline editing translated values and submitting corrections to dictionaries.
class InlineCorrectionService:
"""Handle inline correction of translated values with optional dictionary submission."""
@staticmethod
def apply_inline_edit(
db: Session,
run_id: str,
record_id: str,
language_code: str,
final_value: str,
submit_to_dictionary: bool = False,
dictionary_id: str | None = None,
current_user: str | None = None,
context_data_override: dict | None = None,
usage_notes: str | None = None,
keep_context: bool = True,
) -> dict:
"""Apply an inline edit to a TranslationLanguage entry.
Updates final_value and user_edit fields. Optionally submits to dictionary.
Returns the updated TranslationLanguage data as a dict.
Args:
context_data_override: If provided, overrides auto-captured context data.
usage_notes: Usage notes for the dictionary entry.
keep_context: If False, clear context on the dictionary entry.
"""
from ...models.translate import TranslationLanguage, TranslationRecord
# Find the language entry
lang_entry = (
db.query(TranslationLanguage)
.filter(
TranslationLanguage.record_id == record_id,
TranslationLanguage.language_code == language_code,
)
.first()
)
if not lang_entry:
raise ValueError(f"Language entry '{language_code}' not found for record '{record_id}'")
# Verify the record belongs to the run
record = (
db.query(TranslationRecord)
.filter(
TranslationRecord.id == record_id,
TranslationRecord.run_id == run_id,
)
.first()
)
if not record:
raise ValueError(
f"Translation record '{record_id}' not found in run '{run_id}'"
)
# Apply the edit
lang_entry.final_value = final_value
lang_entry.user_edit = final_value
if lang_entry.status in ("pending", "translated"):
lang_entry.status = "edited"
db.flush()
# Optionally submit to dictionary
dict_result = None
if submit_to_dictionary and dictionary_id:
try:
dict_result = InlineCorrectionService.submit_correction_to_dict(
db=db,
record_id=record_id,
language_code=language_code,
dictionary_id=dictionary_id,
corrected_value=final_value,
current_user=current_user,
context_data_override=context_data_override,
usage_notes=usage_notes,
keep_context=keep_context,
)
except Exception as e:
# Dictionary submission failure should not block the edit
dict_result = {"error": str(e), "action": "failed"}
db.commit()
db.refresh(lang_entry)
from ...schemas.translate import TranslationLanguageResponse
response = TranslationLanguageResponse.model_validate(lang_entry)
result = response.model_dump()
if dict_result:
result["dictionary_result"] = dict_result
return result
@staticmethod
def submit_correction_to_dict(
db: Session,
record_id: str,
language_code: str,
dictionary_id: str,
corrected_value: str,
current_user: str | None = None,
context_data_override: dict | None = None,
usage_notes: str | None = None,
keep_context: bool = True,
) -> dict:
"""Submit a correction from an inline edit to the dictionary.
Fetches the TranslationLanguage entry, auto-captures source text and context,
and creates/updates a DictionaryEntry with correct language pair.
Returns conflict info if an entry already exists.
Args:
context_data_override: If provided, overrides auto-captured context_data.
usage_notes: Optional usage notes to store on the dictionary entry.
keep_context: If False, sets has_context=False and clears context_data.
"""
from ...core.logger import belief_scope, logger
from ...models.translate import DictionaryEntry, TranslationLanguage, TranslationRecord
from ._utils import _normalize_term
with belief_scope("InlineCorrectionService.submit_correction_to_dict"):
# Find the language entry
lang_entry = (
db.query(TranslationLanguage)
.filter(
TranslationLanguage.record_id == record_id,
TranslationLanguage.language_code == language_code,
)
.first()
)
if not lang_entry:
raise ValueError(f"Language entry '{language_code}' not found for record '{record_id}'")
# Find the record for source text and context
record = (
db.query(TranslationRecord)
.filter(TranslationRecord.id == record_id)
.first()
)
if not record:
raise ValueError(f"Translation record '{record_id}' not found")
# Get source text from the record
source_term = record.source_sql or record.source_object_name or ""
if not source_term:
raise ValueError("No source text available for this record")
# Determine language pair
source_language = lang_entry.source_language_detected or "und"
target_language = language_code
# Prepare context data — auto-capture from source row
context_data = {}
if record.source_data:
context_data["source_data"] = record.source_data
if record.source_object_type:
context_data["source_object_type"] = record.source_object_type
if record.source_object_name:
context_data["source_object_name"] = record.source_object_name
if record.source_object_id:
context_data["source_object_id"] = record.source_object_id
# Apply context_data_override if provided (user edited)
if context_data_override is not None:
context_data = context_data_override
context_source = "manual"
else:
context_source = "auto"
# Handle keep_context=False (user explicitly removed context)
if not keep_context:
context_data = None
context_source = "manual"
# Normalize source term
normalized = _normalize_term(source_term)
# Check for existing entry with same (dictionary_id, source_term_norm, source_language, target_language)
existing = (
db.query(DictionaryEntry)
.filter(
DictionaryEntry.dictionary_id == dictionary_id,
DictionaryEntry.source_term_normalized == normalized,
DictionaryEntry.source_language == source_language,
DictionaryEntry.target_language == target_language,
)
.first()
)
result = {
"action": "created",
"entry_id": None,
"conflict": None,
"message": None,
}
if existing:
# Conflict: return conflict info
result["action"] = "conflict_detected"
result["conflict"] = {
"source_term": source_term,
"existing_target_term": existing.target_term,
"submitted_target_term": corrected_value,
"action": "keep_existing",
}
result["message"] = (
f"Existing entry found: '{existing.target_term}' "
f"for source '{source_term}' ({source_language}{target_language})"
)
logger.reason("Correction conflict detected", result)
return result
# Create new entry
entry = DictionaryEntry(
dictionary_id=dictionary_id,
source_term=source_term.strip(),
source_term_normalized=normalized,
target_term=corrected_value.strip(),
source_language=source_language,
target_language=target_language,
context_data=context_data if context_data else None,
context_notes=None,
has_context=bool(context_data) and keep_context,
context_source=context_source,
usage_notes=usage_notes,
origin_source_language=source_language,
origin_run_id=record.run_id,
origin_row_key=record.id,
origin_user_id=current_user,
)
db.add(entry)
db.flush()
result["entry_id"] = entry.id
result["message"] = (
f"Entry created for '{source_term}' ({source_language}) → "
f"'{corrected_value}' ({target_language})"
)
logger.reflect("Dictionary entry created from correction", result)
return result
# #endregion InlineCorrectionService
# #region BulkFindReplaceService [C:3] [TYPE Class] [SEMANTICS translate, bulk, find, replace, regex]
# @BRIEF Service for bulk find-and-replace operations on translated values.
class BulkFindReplaceService:
"""Handle bulk find-and-replace on TranslationLanguage entries."""
@staticmethod
def _compile_pattern(pattern: str, is_regex: bool) -> re.Pattern:
"""Compile a search pattern (regex or plain text)."""
import re
if is_regex:
return re.compile(pattern)
return re.compile(re.escape(pattern))
@staticmethod
def _find_matching_entries(
db: Session,
run_id: str,
pattern: str,
is_regex: bool,
target_language: str,
) -> list[Any]:
"""Find all TranslationLanguage entries matching the pattern."""
from ...models.translate import TranslationLanguage, TranslationRecord
# We scan entries for the target language in this run
entries = (
db.query(TranslationLanguage)
.join(
TranslationRecord,
TranslationRecord.id == TranslationLanguage.record_id,
)
.filter(
TranslationRecord.run_id == run_id,
TranslationLanguage.language_code == target_language,
)
.all()
)
return entries
@staticmethod
def preview(
db: Session,
run_id: str,
pattern: str,
is_regex: bool,
target_language: str,
) -> list[dict]:
"""Scan TranslationLanguage entries and return matching items without applying changes."""
from ...models.translate import TranslationRecord
compiled = BulkFindReplaceService._compile_pattern(pattern, is_regex)
entries = BulkFindReplaceService._find_matching_entries(
db, run_id, pattern, is_regex, target_language
)
preview_items = []
for entry in entries:
current_value = entry.final_value or entry.translated_value or ""
if compiled.search(current_value):
# Get source term for context
record = (
db.query(TranslationRecord)
.filter(TranslationRecord.id == entry.record_id)
.first()
)
source_term = record.source_sql if record else ""
new_value = compiled.sub(
BulkFindReplaceService._get_replacement(pattern, is_regex),
current_value,
)
if new_value != current_value:
preview_items.append({
"record_id": entry.record_id,
"language_code": entry.language_code,
"source_term": source_term or "",
"current_value": current_value,
"new_value": new_value,
"source_language_detected": entry.source_language_detected,
})
return preview_items
@staticmethod
def _get_replacement(pattern: str, is_regex: bool) -> str:
"""Return replacement pattern (for regex, use backrefs; for plain, use literal)."""
# The replacement_text is passed through at apply time
return pattern # Placeholder — actual replacement is done in apply()
@staticmethod
def apply(
db: Session,
run_id: str,
pattern: str,
is_regex: bool,
replacement_text: str,
target_language: str,
submit_to_dictionary: bool = False,
dictionary_id: str | None = None,
usage_notes: str | None = None,
current_user: str | None = None,
submit_to_dictionary_with_context: bool = False,
) -> dict:
"""Apply find-and-replace to matching TranslationLanguage entries.
Returns dict with rows_affected, corrections_submitted, and preview list.
Args:
submit_to_dictionary_with_context: If True, auto-capture source row context
when submitting to dictionary.
"""
from ...core.logger import belief_scope, logger
from ...models.translate import DictionaryEntry, TranslationRecord
with belief_scope("BulkFindReplaceService.apply"):
compiled = BulkFindReplaceService._compile_pattern(pattern, is_regex)
entries = BulkFindReplaceService._find_matching_entries(
db, run_id, pattern, is_regex, target_language
)
preview_items = []
rows_affected = 0
corrections_submitted = 0
# Track unique (source_term, replacement) -> first record for context capture
seen_term_replacements: dict[str, str] = {}
for entry in entries:
current_value = entry.final_value or entry.translated_value or ""
if compiled.search(current_value):
new_value = compiled.sub(replacement_text, current_value)
if new_value != current_value:
# Apply the replacement
entry.final_value = new_value
entry.user_edit = new_value
if entry.status in ("pending", "translated"):
entry.status = "edited"
rows_affected += 1
# Get source term for preview
record = (
db.query(TranslationRecord)
.filter(TranslationRecord.id == entry.record_id)
.first()
)
source_term = record.source_sql if record else ""
# Optionally submit to dictionary
if submit_to_dictionary and dictionary_id:
try:
dict_result = InlineCorrectionService.submit_correction_to_dict(
db=db,
record_id=entry.record_id,
language_code=entry.language_code,
dictionary_id=dictionary_id,
corrected_value=new_value,
current_user=current_user,
)
if dict_result.get("action") in ("created", "updated"):
corrections_submitted += 1
# Track context for unique (source_term, replacement) pairs
if submit_to_dictionary_with_context and record:
term_key = f"{source_term}::{new_value}"
if term_key not in seen_term_replacements:
seen_term_replacements[term_key] = entry.record_id
# Auto-capture context from this row
context_data = {}
if record.source_data:
context_data["source_data"] = record.source_data
if record.source_object_type:
context_data["source_object_type"] = record.source_object_type
if record.source_object_name:
context_data["source_object_name"] = record.source_object_name
if record.source_object_id:
context_data["source_object_id"] = record.source_object_id
# Update the just-created entry's context
if dict_result.get("action") == "created" and dict_result.get("entry_id"):
d_entry = (
db.query(DictionaryEntry)
.filter(DictionaryEntry.id == dict_result["entry_id"])
.first()
)
if d_entry:
d_entry.context_data = context_data if context_data else None
d_entry.has_context = bool(context_data)
d_entry.context_source = "bulk"
except Exception as e:
logger.explore(
"Bulk replace: dictionary submission failed",
extra={"record_id": entry.record_id, "error": str(e)},
)
preview_items.append({
"record_id": entry.record_id,
"language_code": entry.language_code,
"source_term": source_term or "",
"current_value": current_value,
"new_value": new_value,
"source_language_detected": entry.source_language_detected,
})
db.commit()
result = {
"rows_affected": rows_affected,
"corrections_submitted": corrections_submitted,
"preview": preview_items,
}
logger.reason("Bulk replace completed", result)
return result
# #endregion BulkFindReplaceService
# #endregion TranslateJobService
# #endregion TranslateJobService

View File

@@ -6,7 +6,24 @@
from datetime import datetime
from typing import Any
from pydantic import BaseModel, Field
import re
from pydantic import BaseModel, Field, field_validator
def _validate_bcp47_list(v: list[str] | None) -> list[str] | None:
"""Validate that each item in target_languages is a valid BCP-47 tag."""
if not v:
return v
for tag in v:
if not tag or not tag.strip():
raise ValueError(f"Invalid BCP-47 tag: '{tag}' — must be a non-empty string")
if not re.match(r'^[a-zA-Z]{2,8}(-[a-zA-Z0-9]{1,8})*$', tag.strip()):
raise ValueError(
f"Invalid BCP-47 tag: '{tag}'. "
"Expected format like 'en', 'ru', 'zh-CN', 'pt-BR'."
)
return v
# #region TranslateJobCreate [TYPE Class]
@@ -26,8 +43,11 @@ class TranslateJobCreate(BaseModel):
translation_column: str | None = Field(None, description="Source column to translate")
target_column: str | None = Field(None, description="Target column for translated output (defaults to translation_column)")
context_columns: list[str] | None = Field(default_factory=list, description="Context column names")
target_language: str | None = Field(None, description="Target language code")
target_language: str | None = Field(None, description="Target language code [DEPRECATED: use target_languages]")
target_languages: list[str] | None = Field(default_factory=list, description="List of BCP-47 target language codes")
provider_id: str | None = Field(None, description="LLM provider ID")
_validate_target_languages = field_validator("target_languages")(_validate_bcp47_list)
batch_size: int = Field(50, description="Records per batch")
upsert_strategy: str = Field("MERGE", description="UPSERT strategy: MERGE, INSERT, UPDATE")
dictionary_ids: list[str] | None = Field(default_factory=list, description="Associated terminology dictionary IDs")
@@ -54,7 +74,10 @@ class TranslateJobUpdate(BaseModel):
target_column: str | None = None
context_columns: list[str] | None = None
target_language: str | None = None
target_languages: list[str] | None = None
provider_id: str | None = None
_validate_target_languages = field_validator("target_languages")(_validate_bcp47_list)
batch_size: int | None = None
upsert_strategy: str | None = None
status: str | None = None
@@ -83,6 +106,8 @@ class TranslateJobResponse(BaseModel):
target_column: str | None = None
context_columns: list[str] | None = None
target_language: str | None = None
source_language: str | None = None
target_languages: list[str] | None = None
provider_id: str | None = None
batch_size: int = 50
upsert_strategy: str = "MERGE"
@@ -154,6 +179,8 @@ class DictionaryImport(BaseModel):
delimiter: str | None = Field(None, description="Detected or forced delimiter: ',' or '\\t'. Auto-detect if omitted.")
on_conflict: str = Field("overwrite", description="'overwrite' or 'keep_existing' or 'cancel'")
preview_only: bool = Field(False, description="If true, return preview without applying")
default_source_language: str | None = Field(None, description="Default BCP-47 source language when column missing in data")
default_target_language: str | None = Field(None, description="Default BCP-47 target language when column missing in data")
# #endregion DictionaryImport
@@ -177,11 +204,13 @@ class DictionaryResponse(BaseModel):
# #region DictionaryEntryCreate [TYPE Class]
# @BRIEF Schema for adding/editing a dictionary entry.
# @BRIEF Schema for adding/editing a dictionary entry with language pair support.
class DictionaryEntryCreate(BaseModel):
source_term: str = Field(..., description="Source term to translate")
target_term: str = Field(..., description="Target/translated term")
context_notes: str | None = Field(None, description="Optional context notes")
source_language: str = Field("und", description="BCP-47 source language code (default: und)")
target_language: str = Field("und", description="BCP-47 target language code (default: und)")
class Config:
from_attributes = True
@@ -196,7 +225,14 @@ class DictionaryEntryResponse(BaseModel):
source_term: str
source_term_normalized: str
target_term: str
source_language: str | None = None
target_language: str | None = None
context_notes: str | None = None
context_data: dict[str, Any] | None = None
usage_notes: str | None = None
has_context: bool = False
context_source: str | None = None
origin_source_language: str | None = None
created_at: datetime
updated_at: datetime | None = None
@@ -232,6 +268,7 @@ class PreviewRowUpdate(BaseModel):
action: str = Field(..., description="'approve', 'reject', or 'edit'")
translation: str | None = Field(None, description="Edited translation (required for 'edit' action)")
feedback: str | None = Field(None, description="Optional feedback/comment")
language_code: str | None = Field(None, description="BCP-47 language code for per-language actions (optional, applies to all if omitted)")
# #endregion PreviewRowUpdate
@@ -245,6 +282,7 @@ class PreviewAcceptResponse(BaseModel):
created_at: datetime
expires_at: datetime | None = None
records: list['PreviewRow'] = []
target_languages: list[str] = Field(default_factory=list, description="Target languages for this preview")
class Config:
from_attributes = True
@@ -255,6 +293,7 @@ class PreviewAcceptResponse(BaseModel):
# @BRIEF Schema for cost estimation in preview response.
class CostEstimate(BaseModel):
sample_size: int = 0
num_languages: int = 1
sample_prompt_tokens: int = 0
sample_output_tokens: int = 0
sample_total_tokens: int = 0
@@ -262,6 +301,7 @@ class CostEstimate(BaseModel):
estimated_total_rows: int = 0
estimated_tokens: int = 0
estimated_cost: float = 0.0
warning: str | None = None
# #endregion CostEstimate
@@ -274,6 +314,8 @@ class TermCorrectionSubmit(BaseModel):
dictionary_id: str | None = Field(None, description="Target dictionary ID (language-filtered)")
origin_run_id: str | None = Field(None, description="Run ID from which this correction originated")
origin_row_key: str | None = Field(None, description="Row key within the run")
context_data: dict[str, Any] | None = Field(None, description="Context data for the dictionary entry")
usage_notes: str | None = Field(None, description="Usage notes for the dictionary entry")
# #endregion TermCorrectionSubmit
@@ -369,6 +411,7 @@ class TranslationRunResponse(BaseModel):
dict_snapshot_hash: str | None = None
created_by: str | None = None
created_at: datetime
language_stats: list['TranslationRunLanguageStatsResponse'] | None = None
class Config:
from_attributes = True
@@ -430,6 +473,22 @@ class AggregatedMetricsResponse(BaseModel):
# #endregion AggregatedMetricsResponse
# #region TranslationPreviewLanguageResponse [C:1] [TYPE Class]
# @BRIEF Schema for per-language preview data in API responses.
class TranslationPreviewLanguageResponse(BaseModel):
language_code: str
source_language_detected: str | None = None
translated_value: str | None = None
user_edit: str | None = None
final_value: str | None = None
status: str = "pending"
needs_review: bool = False
class Config:
from_attributes = True
# #endregion TranslationPreviewLanguageResponse
# #region PreviewRow [TYPE Class]
# @BRIEF A single row in a translation preview showing original and translated content side by side.
class PreviewRow(BaseModel):
@@ -441,6 +500,9 @@ class PreviewRow(BaseModel):
source_object_name: str | None = None
status: str = "PENDING"
feedback: str | None = None
source_language_detected: str | None = None
needs_review: bool = False
languages: list[TranslationPreviewLanguageResponse] = Field(default_factory=list, description="Per-language translation data")
# #endregion PreviewRow
@@ -455,6 +517,8 @@ class TranslationPreviewResponse(BaseModel):
created_at: datetime
expires_at: datetime | None = None
records: list[PreviewRow] = []
target_languages: list[str] = Field(default_factory=list, description="Target languages for this preview")
cost_estimate: CostEstimate | None = None
class Config:
from_attributes = True
@@ -500,4 +564,105 @@ class MetricsResponse(BaseModel):
from_attributes = True
# #endregion MetricsResponse
# #region TranslationLanguageResponse [C:1] [TYPE Class]
# @BRIEF Schema for per-language translation data in API responses.
class TranslationLanguageResponse(BaseModel):
language_code: str
source_language_detected: str | None = None
translated_value: str | None = None
user_edit: str | None = None
final_value: str | None = None
status: str = "pending"
needs_review: bool = False
language_overridden: bool = False
class Config:
from_attributes = True
# #endregion TranslationLanguageResponse
# #region TranslationRunLanguageStatsResponse [C:1] [TYPE Class]
# @BRIEF Schema for per-language statistics in run API responses.
class TranslationRunLanguageStatsResponse(BaseModel):
language_code: str
total_rows: int = 0
translated_rows: int = 0
failed_rows: int = 0
skipped_rows: int = 0
token_count: int = 0
estimated_cost: float = 0.0
class Config:
from_attributes = True
# #endregion TranslationRunLanguageStatsResponse
# #region OverrideLanguageRequest [C:1] [TYPE Class]
# @BRIEF Schema for manually overriding the detected source language of a translation.
class OverrideLanguageRequest(BaseModel):
source_language: str = Field(..., description="BCP-47 language code to override with")
# #endregion OverrideLanguageRequest
# #region BulkFindReplaceRequest [C:1] [TYPE Class]
# @BRIEF Schema for bulk find-and-replace within translations for a target language.
class BulkFindReplaceRequest(BaseModel):
find_pattern: str = Field(..., description="Text or regex pattern to find")
is_regex: bool = Field(False, description="Whether find_pattern is a regular expression")
replacement_text: str = Field(..., description="Replacement text")
target_language: str = Field(..., description="BCP-47 language code to target")
preview: bool = Field(True, description="If true, return matches without applying replacements")
submit_to_dictionary: bool = Field(False, description="If true, also submit corrections to dictionary")
dictionary_id: str | None = Field(None, description="Target dictionary ID for submitting corrections")
usage_notes: str | None = Field(None, description="Usage notes for dictionary entries")
submit_to_dictionary_with_context: bool = Field(False, description="If true, auto-capture source row context when submitting to dictionary")
# #endregion BulkFindReplaceRequest
# #region InlineEditRequest [C:1] [TYPE Class]
# @BRIEF Schema for inline editing a translated value on a completed run result.
class InlineEditRequest(BaseModel):
final_value: str = Field(..., description="Corrected/edited translation text")
submit_to_dictionary: bool = Field(False, description="If true, also save correction to dictionary")
dictionary_id: str | None = Field(None, description="Target dictionary ID for saving correction")
context_data_override: dict[str, Any] | None = Field(None, description="Override auto-captured context data for dictionary entry")
usage_notes: str | None = Field(None, description="Usage notes for the dictionary entry")
keep_context: bool = Field(True, description="If false, clear context_data and set has_context=False on dictionary entry")
# #endregion InlineEditRequest
# #region BulkReplacePreviewItem [C:1] [TYPE Class]
# @BRIEF A single item in a bulk replace preview list.
class BulkReplacePreviewItem(BaseModel):
record_id: str
language_code: str
source_term: str
current_value: str
new_value: str
source_language_detected: str | None = None
# #endregion BulkReplacePreviewItem
# #region BulkReplaceResponse [C:1] [TYPE Class]
# @BRIEF Response for bulk find-and-replace operation.
class BulkReplaceResponse(BaseModel):
rows_affected: int = 0
corrections_submitted: int = 0
preview: list[BulkReplacePreviewItem] = Field(default_factory=list)
# #endregion BulkReplaceResponse
# #region InlineCorrectionSubmit [C:1] [TYPE Class]
# @BRIEF Schema for submitting an inline correction for a specific translated record.
class InlineCorrectionSubmit(BaseModel):
record_id: str = Field(..., description="Translation record ID")
language_code: str = Field(..., description="BCP-47 language code of the translated value")
source_term: str = Field(..., description="Original source term that was incorrectly translated")
corrected_term: str = Field(..., description="Corrected translation")
dictionary_id: str = Field(..., description="Target dictionary ID to also save the correction")
source_language: str = Field(..., description="BCP-47 source language code")
# #endregion InlineCorrectionSubmit
# #endregion TranslateSchemas