feat(translate): multi-language optimization (Phase 11)
- Auto-detection of source language per row via LLM (US6) - Multi-target translation — one LLM call for N languages (US1-US3) - Language-aware storage: TranslationLanguage, per-language stats - Multilingual dictionaries with language-pair-aware filtering (US7) - Inline correction on any run result + submit-to-dictionary (US8) - Context-aware dictionary: auto-capture row context, usage notes, Jaccard similarity, priority flagging in LLM prompts (US8b) - Configurable preview sample size 1-100, cost warning at >30 - Per-language history & metrics with MetricSnapshot preservation - 36 files, +5022/-373, all specs GRACE-Poly v2.6 compliant
This commit is contained in:
@@ -30,9 +30,11 @@ from ...core.logger import belief_scope, logger
|
||||
from ...models.translate import (
|
||||
TranslationBatch,
|
||||
TranslationJob,
|
||||
TranslationLanguage,
|
||||
TranslationPreviewSession,
|
||||
TranslationRecord,
|
||||
TranslationRun,
|
||||
TranslationRunLanguageStats,
|
||||
)
|
||||
from .events import TranslationEventLog
|
||||
from .executor import TranslationExecutor
|
||||
@@ -201,6 +203,27 @@ class TranslationOrchestrator:
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
# Initialize per-language stats
|
||||
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
|
||||
if not isinstance(target_languages, list):
|
||||
target_languages = [str(target_languages)]
|
||||
language_stats_map: dict[str, TranslationRunLanguageStats] = {}
|
||||
for lang_code in target_languages:
|
||||
lang_stat = TranslationRunLanguageStats(
|
||||
id=str(uuid.uuid4()),
|
||||
run_id=run.id,
|
||||
language_code=lang_code,
|
||||
total_rows=0,
|
||||
translated_rows=0,
|
||||
failed_rows=0,
|
||||
skipped_rows=0,
|
||||
token_count=0,
|
||||
estimated_cost=0.0,
|
||||
)
|
||||
self.db.add(lang_stat)
|
||||
language_stats_map[lang_code] = lang_stat
|
||||
self.db.flush()
|
||||
|
||||
# Dispatch executor
|
||||
executor = TranslationExecutor(
|
||||
self.db, self.config_manager, self.current_user,
|
||||
@@ -227,6 +250,9 @@ class TranslationOrchestrator:
|
||||
self.db.commit()
|
||||
return run
|
||||
|
||||
# Aggregate per-language statistics after executor completes
|
||||
self._update_language_stats(run.id, language_stats_map)
|
||||
|
||||
# Record translation phase complete
|
||||
self.event_log.log_event(
|
||||
job_id=job.id,
|
||||
@@ -745,6 +771,25 @@ class TranslationOrchestrator:
|
||||
# Get event summary
|
||||
event_summary = self.event_log.get_run_event_summary(run_id)
|
||||
|
||||
# Get language stats
|
||||
language_stats_entries = (
|
||||
self.db.query(TranslationRunLanguageStats)
|
||||
.filter(TranslationRunLanguageStats.run_id == run_id)
|
||||
.all()
|
||||
)
|
||||
language_stats = [
|
||||
{
|
||||
"language_code": ls.language_code,
|
||||
"total_rows": ls.total_rows or 0,
|
||||
"translated_rows": ls.translated_rows or 0,
|
||||
"failed_rows": ls.failed_rows or 0,
|
||||
"skipped_rows": ls.skipped_rows or 0,
|
||||
"token_count": ls.token_count or 0,
|
||||
"estimated_cost": ls.estimated_cost or 0.0,
|
||||
}
|
||||
for ls in language_stats_entries
|
||||
]
|
||||
|
||||
return {
|
||||
"id": run.id,
|
||||
"job_id": run.job_id,
|
||||
@@ -759,6 +804,7 @@ class TranslationOrchestrator:
|
||||
"insert_status": run.insert_status,
|
||||
"superset_execution_id": run.superset_execution_id,
|
||||
"batch_count": batch_count,
|
||||
"language_stats": language_stats,
|
||||
"event_invariants": {
|
||||
"has_run_started": event_summary["has_run_started"],
|
||||
"terminal_event_count": event_summary["terminal_event_count"],
|
||||
@@ -863,6 +909,80 @@ class TranslationOrchestrator:
|
||||
]
|
||||
# endregion get_run_history
|
||||
|
||||
# region _update_language_stats [TYPE Function]
|
||||
# @PURPOSE: Aggregate TranslationLanguage entries by language_code and update TranslationRunLanguageStats.
|
||||
# @PRE: run_id and language_stats_map are valid. DB session is available.
|
||||
# @POST: Language stats are updated with row counts and estimated tokens/cost.
|
||||
# @SIDE_EFFECT: DB writes on language_stats objects.
|
||||
def _update_language_stats(
|
||||
self,
|
||||
run_id: str,
|
||||
language_stats_map: dict[str, TranslationRunLanguageStats],
|
||||
) -> None:
|
||||
with belief_scope("TranslationOrchestrator._update_language_stats"):
|
||||
# Get all records for this run to join with TranslationLanguage
|
||||
records = (
|
||||
self.db.query(TranslationRecord)
|
||||
.filter(TranslationRecord.run_id == run_id)
|
||||
.all()
|
||||
)
|
||||
record_ids = [r.id for r in records]
|
||||
|
||||
if not record_ids:
|
||||
logger.reason("No records for language stats aggregation", {"run_id": run_id})
|
||||
return
|
||||
|
||||
# Get all language entries for this run's records
|
||||
lang_entries = (
|
||||
self.db.query(TranslationLanguage)
|
||||
.filter(TranslationLanguage.record_id.in_(record_ids))
|
||||
.all()
|
||||
)
|
||||
|
||||
# Aggregate by language_code
|
||||
from collections import defaultdict
|
||||
agg: dict[str, dict[str, int]] = defaultdict(lambda: {"total": 0, "translated": 0, "failed": 0, "skipped": 0})
|
||||
|
||||
for le in lang_entries:
|
||||
code = le.language_code
|
||||
agg[code]["total"] += 1
|
||||
if le.status in ("translated", "approved", "edited"):
|
||||
agg[code]["translated"] += 1
|
||||
elif le.status == "failed":
|
||||
agg[code]["failed"] += 1
|
||||
elif le.status == "skipped":
|
||||
agg[code]["skipped"] += 1
|
||||
|
||||
# Estimate tokens: heuristic based on character count of translated values
|
||||
total_chars = sum(
|
||||
len(le.translated_value or "") for le in lang_entries if le.translated_value
|
||||
)
|
||||
total_tokens = max(1, total_chars // 4) # ~4 chars per token
|
||||
cost_per_token = 0.002 / 1000 # $0.002 per 1K tokens
|
||||
|
||||
# Update each language stat entry
|
||||
for lang_code, lang_stat in language_stats_map.items():
|
||||
data = agg.get(lang_code, {"total": 0, "translated": 0, "failed": 0, "skipped": 0})
|
||||
lang_stat.total_rows = data["total"]
|
||||
lang_stat.translated_rows = data["translated"]
|
||||
lang_stat.failed_rows = data["failed"]
|
||||
lang_stat.skipped_rows = data["skipped"]
|
||||
|
||||
# Proportional token split: share tokens across languages
|
||||
num_langs = len(language_stats_map)
|
||||
if num_langs > 0:
|
||||
lang_stat.token_count = total_tokens // num_langs
|
||||
lang_stat.estimated_cost = round((lang_stat.token_count / 1000) * cost_per_token, 6)
|
||||
|
||||
self.db.flush()
|
||||
|
||||
logger.reason("Language stats updated", {
|
||||
"run_id": run_id,
|
||||
"languages": list(language_stats_map.keys()),
|
||||
"total_tokens_est": total_tokens,
|
||||
})
|
||||
# endregion _update_language_stats
|
||||
|
||||
# region _compute_config_hash [TYPE Function]
|
||||
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
|
||||
@staticmethod
|
||||
|
||||
Reference in New Issue
Block a user