feat(translate): multi-language optimization (Phase 11)

- Auto-detection of source language per row via LLM (US6)
- Multi-target translation — one LLM call for N languages (US1-US3)
- Language-aware storage: TranslationLanguage, per-language stats
- Multilingual dictionaries with language-pair-aware filtering (US7)
- Inline correction on any run result + submit-to-dictionary (US8)
- Context-aware dictionary: auto-capture row context, usage notes,
  Jaccard similarity, priority flagging in LLM prompts (US8b)
- Configurable preview sample size 1-100, cost warning at >30
- Per-language history & metrics with MetricSnapshot preservation
- 36 files, +5022/-373, all specs GRACE-Poly v2.6 compliant
This commit is contained in:
2026-05-14 17:12:41 +03:00
parent 5e741a4332
commit bb0fbfdafd
36 changed files with 5025 additions and 376 deletions

View File

@@ -30,9 +30,11 @@ from ...core.logger import belief_scope, logger
from ...models.translate import (
TranslationBatch,
TranslationJob,
TranslationLanguage,
TranslationPreviewSession,
TranslationRecord,
TranslationRun,
TranslationRunLanguageStats,
)
from .events import TranslationEventLog
from .executor import TranslationExecutor
@@ -201,6 +203,27 @@ class TranslationOrchestrator:
created_by=self.current_user,
)
# Initialize per-language stats
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
language_stats_map: dict[str, TranslationRunLanguageStats] = {}
for lang_code in target_languages:
lang_stat = TranslationRunLanguageStats(
id=str(uuid.uuid4()),
run_id=run.id,
language_code=lang_code,
total_rows=0,
translated_rows=0,
failed_rows=0,
skipped_rows=0,
token_count=0,
estimated_cost=0.0,
)
self.db.add(lang_stat)
language_stats_map[lang_code] = lang_stat
self.db.flush()
# Dispatch executor
executor = TranslationExecutor(
self.db, self.config_manager, self.current_user,
@@ -227,6 +250,9 @@ class TranslationOrchestrator:
self.db.commit()
return run
# Aggregate per-language statistics after executor completes
self._update_language_stats(run.id, language_stats_map)
# Record translation phase complete
self.event_log.log_event(
job_id=job.id,
@@ -745,6 +771,25 @@ class TranslationOrchestrator:
# Get event summary
event_summary = self.event_log.get_run_event_summary(run_id)
# Get language stats
language_stats_entries = (
self.db.query(TranslationRunLanguageStats)
.filter(TranslationRunLanguageStats.run_id == run_id)
.all()
)
language_stats = [
{
"language_code": ls.language_code,
"total_rows": ls.total_rows or 0,
"translated_rows": ls.translated_rows or 0,
"failed_rows": ls.failed_rows or 0,
"skipped_rows": ls.skipped_rows or 0,
"token_count": ls.token_count or 0,
"estimated_cost": ls.estimated_cost or 0.0,
}
for ls in language_stats_entries
]
return {
"id": run.id,
"job_id": run.job_id,
@@ -759,6 +804,7 @@ class TranslationOrchestrator:
"insert_status": run.insert_status,
"superset_execution_id": run.superset_execution_id,
"batch_count": batch_count,
"language_stats": language_stats,
"event_invariants": {
"has_run_started": event_summary["has_run_started"],
"terminal_event_count": event_summary["terminal_event_count"],
@@ -863,6 +909,80 @@ class TranslationOrchestrator:
]
# endregion get_run_history
# region _update_language_stats [TYPE Function]
# @PURPOSE: Aggregate TranslationLanguage entries by language_code and update TranslationRunLanguageStats.
# @PRE: run_id and language_stats_map are valid. DB session is available.
# @POST: Language stats are updated with row counts and estimated tokens/cost.
# @SIDE_EFFECT: DB writes on language_stats objects.
def _update_language_stats(
self,
run_id: str,
language_stats_map: dict[str, TranslationRunLanguageStats],
) -> None:
with belief_scope("TranslationOrchestrator._update_language_stats"):
# Get all records for this run to join with TranslationLanguage
records = (
self.db.query(TranslationRecord)
.filter(TranslationRecord.run_id == run_id)
.all()
)
record_ids = [r.id for r in records]
if not record_ids:
logger.reason("No records for language stats aggregation", {"run_id": run_id})
return
# Get all language entries for this run's records
lang_entries = (
self.db.query(TranslationLanguage)
.filter(TranslationLanguage.record_id.in_(record_ids))
.all()
)
# Aggregate by language_code
from collections import defaultdict
agg: dict[str, dict[str, int]] = defaultdict(lambda: {"total": 0, "translated": 0, "failed": 0, "skipped": 0})
for le in lang_entries:
code = le.language_code
agg[code]["total"] += 1
if le.status in ("translated", "approved", "edited"):
agg[code]["translated"] += 1
elif le.status == "failed":
agg[code]["failed"] += 1
elif le.status == "skipped":
agg[code]["skipped"] += 1
# Estimate tokens: heuristic based on character count of translated values
total_chars = sum(
len(le.translated_value or "") for le in lang_entries if le.translated_value
)
total_tokens = max(1, total_chars // 4) # ~4 chars per token
cost_per_token = 0.002 / 1000 # $0.002 per 1K tokens
# Update each language stat entry
for lang_code, lang_stat in language_stats_map.items():
data = agg.get(lang_code, {"total": 0, "translated": 0, "failed": 0, "skipped": 0})
lang_stat.total_rows = data["total"]
lang_stat.translated_rows = data["translated"]
lang_stat.failed_rows = data["failed"]
lang_stat.skipped_rows = data["skipped"]
# Proportional token split: share tokens across languages
num_langs = len(language_stats_map)
if num_langs > 0:
lang_stat.token_count = total_tokens // num_langs
lang_stat.estimated_cost = round((lang_stat.token_count / 1000) * cost_per_token, 6)
self.db.flush()
logger.reason("Language stats updated", {
"run_id": run_id,
"languages": list(language_stats_map.keys()),
"total_tokens_est": total_tokens,
})
# endregion _update_language_stats
# region _compute_config_hash [TYPE Function]
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
@staticmethod