feat(translate): multi-language optimization (Phase 11)
- Auto-detection of source language per row via LLM (US6) - Multi-target translation — one LLM call for N languages (US1-US3) - Language-aware storage: TranslationLanguage, per-language stats - Multilingual dictionaries with language-pair-aware filtering (US7) - Inline correction on any run result + submit-to-dictionary (US8) - Context-aware dictionary: auto-capture row context, usage notes, Jaccard similarity, priority flagging in LLM prompts (US8b) - Configurable preview sample size 1-100, cost warning at >30 - Per-language history & metrics with MetricSnapshot preservation - 36 files, +5022/-373, all specs GRACE-Poly v2.6 compliant
This commit is contained in:
@@ -6,7 +6,8 @@
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, Boolean, Column, DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy import JSON, Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from .mapping import Base
|
||||
|
||||
@@ -44,7 +45,11 @@ class TranslationJob(Base):
|
||||
context_columns = Column(JSON, nullable=True, comment="Context column names included in LLM prompt")
|
||||
|
||||
# LLM & processing settings
|
||||
target_language = Column(String, nullable=True, comment="Target language code (e.g. en, ru)")
|
||||
# @DEPRECATED — use target_languages
|
||||
target_language = Column(String, nullable=True, comment="Target language code (e.g. en, ru) [DEPRECATED: use target_languages]")
|
||||
# @DEPRECATED — Auto-detected per row by LLM, kept as fallback hint
|
||||
source_language = Column(String, nullable=True, comment="Fallback source language hint [DEPRECATED: auto-detected per row]")
|
||||
target_languages = Column(JSON, nullable=True, comment="List of BCP-47 target language codes (multi-language support)")
|
||||
provider_id = Column(String, nullable=True, comment="LLM provider ID")
|
||||
batch_size = Column(Integer, nullable=False, default=50, comment="Records per batch")
|
||||
upsert_strategy = Column(String, nullable=False, default="MERGE", comment="MERGE, INSERT, UPDATE")
|
||||
@@ -84,6 +89,8 @@ class TranslationRun(Base):
|
||||
dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at run time")
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
|
||||
|
||||
language_stats = relationship("TranslationRunLanguageStats", back_populates="run")
|
||||
# #endregion TranslationRun
|
||||
|
||||
|
||||
@@ -126,6 +133,15 @@ class TranslationRecord(Base):
|
||||
translation_duration_ms = Column(Integer, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
|
||||
|
||||
# @DEPRECATED — use TranslationLanguage instead
|
||||
llm_translation = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]")
|
||||
# @DEPRECATED — use TranslationLanguage instead
|
||||
user_edit = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]")
|
||||
# @DEPRECATED — use TranslationLanguage instead
|
||||
final_value = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]")
|
||||
|
||||
languages = relationship("TranslationLanguage", back_populates="record")
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_translation_records_run_status", "run_id", "status"),
|
||||
)
|
||||
@@ -178,6 +194,8 @@ class TranslationPreviewRecord(Base):
|
||||
status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED
|
||||
feedback = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
|
||||
|
||||
languages = relationship("TranslationPreviewLanguage", back_populates="preview_record")
|
||||
# #endregion TranslationPreviewRecord
|
||||
|
||||
|
||||
@@ -191,6 +209,10 @@ class TerminologyDictionary(Base):
|
||||
description = Column(Text, nullable=True)
|
||||
source_dialect = Column(String, nullable=False)
|
||||
target_dialect = Column(String, nullable=False)
|
||||
# @DEPRECATED — use per-entry source_language instead, kept for backward compat
|
||||
source_language = Column(String, nullable=True, comment="[DEPRECATED: use per-entry source_language]")
|
||||
# @DEPRECATED — use per-entry target_language instead, kept for backward compat
|
||||
target_language = Column(String, nullable=True, comment="[DEPRECATED: use per-entry target_language]")
|
||||
is_active = Column(Boolean, default=True)
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
|
||||
@@ -208,7 +230,14 @@ class DictionaryEntry(Base):
|
||||
source_term = Column(String, nullable=False)
|
||||
source_term_normalized = Column(String, nullable=False)
|
||||
target_term = Column(String, nullable=False)
|
||||
source_language = Column(String, nullable=False, comment="BCP-47 source language code")
|
||||
target_language = Column(String, nullable=False, comment="BCP-47 target language code")
|
||||
context_notes = Column(Text, nullable=True)
|
||||
context_data = Column(JSON, nullable=True, comment="Structured context for term usage")
|
||||
usage_notes = Column(Text, nullable=True, comment="Usage guidance for the term mapping")
|
||||
has_context = Column(Boolean, default=False, comment="Whether context_data is populated")
|
||||
context_source = Column(String, nullable=True, comment="auto|auto_with_edits|manual|bulk")
|
||||
origin_source_language = Column(String, nullable=True, comment="Original source language of the term")
|
||||
origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated")
|
||||
origin_row_key = Column(String, nullable=True, comment="Row key within the run that triggered this correction")
|
||||
origin_user_id = Column(String, nullable=True, comment="User who submitted the correction")
|
||||
@@ -217,9 +246,11 @@ class DictionaryEntry(Base):
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"dictionary_id", "source_term_normalized",
|
||||
name="uq_dictionary_entry_term"
|
||||
"dictionary_id", "source_term_normalized", "source_language", "target_language",
|
||||
name="uq_dict_source_term_lang"
|
||||
),
|
||||
Index("idx_dict_entry_lang", "source_language", "target_language"),
|
||||
Index("idx_dict_has_context", "has_context"),
|
||||
)
|
||||
# #endregion DictionaryEntry
|
||||
|
||||
@@ -284,6 +315,7 @@ class MetricSnapshot(Base):
|
||||
p50_duration_ms = Column(Integer, nullable=True)
|
||||
p95_duration_ms = Column(Integer, nullable=True)
|
||||
p99_duration_ms = Column(Integer, nullable=True)
|
||||
per_language_metrics = Column(JSON, nullable=True, comment="Per-language cumulative metrics: {lang: {cumulative_tokens, cumulative_cost, runs}}")
|
||||
snapshot_date = Column(DateTime, nullable=False, default=lambda: datetime.now(UTC))
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
|
||||
|
||||
@@ -292,4 +324,81 @@ class MetricSnapshot(Base):
|
||||
)
|
||||
# #endregion MetricSnapshot
|
||||
|
||||
|
||||
# #region TranslationLanguage [C:1] [TYPE Class]
|
||||
# @BRIEF Per-language translation result for a single record, supporting multi-language output.
|
||||
class TranslationLanguage(Base):
|
||||
__tablename__ = "translation_languages"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
record_id = Column(String, ForeignKey("translation_records.id"), nullable=False)
|
||||
language_code = Column(String, nullable=False, comment="BCP-47 language code")
|
||||
source_language_detected = Column(String, nullable=True, comment="BCP-47 or 'und' for undetermined")
|
||||
translated_value = Column(Text, nullable=True, comment="LLM-generated translation")
|
||||
user_edit = Column(Text, nullable=True, comment="User-edited translation")
|
||||
final_value = Column(Text, nullable=True, comment="Final resolved value (translated or user edit)")
|
||||
status = Column(String, default="pending", comment="pending|translated|approved|edited|rejected|failed|skipped")
|
||||
error_message = Column(Text, nullable=True)
|
||||
needs_review = Column(Boolean, default=False, comment="Flagged because source language could not be determined")
|
||||
language_overridden = Column(Boolean, default=False, comment="Source language was manually overridden by user")
|
||||
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
|
||||
record = relationship("TranslationRecord", back_populates="languages")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("record_id", "language_code", name="uq_record_language"),
|
||||
Index("idx_tl_language", "language_code"),
|
||||
Index("idx_tl_record_lang", "record_id", "language_code"),
|
||||
)
|
||||
# #endregion TranslationLanguage
|
||||
|
||||
|
||||
# #region TranslationPreviewLanguage [C:1] [TYPE Class]
|
||||
# @BRIEF Per-language preview entry within a preview session.
|
||||
class TranslationPreviewLanguage(Base):
|
||||
__tablename__ = "translation_preview_languages"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
preview_record_id = Column(String, ForeignKey("translation_preview_records.id"), nullable=False)
|
||||
language_code = Column(String, nullable=False, comment="BCP-47 language code")
|
||||
source_language_detected = Column(String, nullable=True, comment="BCP-47 or 'und'")
|
||||
translated_value = Column(Text, nullable=True)
|
||||
user_edit = Column(Text, nullable=True)
|
||||
final_value = Column(Text, nullable=True)
|
||||
status = Column(String, default="pending", comment="pending|approved|edited|rejected")
|
||||
needs_review = Column(Boolean, default=False, comment="Flagged because source language could not be determined")
|
||||
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
|
||||
preview_record = relationship("TranslationPreviewRecord", back_populates="languages")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("preview_record_id", "language_code", name="uq_preview_record_language"),
|
||||
)
|
||||
# #endregion TranslationPreviewLanguage
|
||||
|
||||
|
||||
# #region TranslationRunLanguageStats [C:1] [TYPE Class]
|
||||
# @BRIEF Per-language statistics for a translation run (row counts, tokens, cost).
|
||||
class TranslationRunLanguageStats(Base):
|
||||
__tablename__ = "translation_run_language_stats"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=False)
|
||||
language_code = Column(String, nullable=False, comment="BCP-47 language code")
|
||||
total_rows = Column(Integer, default=0)
|
||||
translated_rows = Column(Integer, default=0)
|
||||
failed_rows = Column(Integer, default=0)
|
||||
skipped_rows = Column(Integer, default=0)
|
||||
token_count = Column(Integer, default=0)
|
||||
estimated_cost = Column(Float, default=0.0)
|
||||
|
||||
run = relationship("TranslationRun", back_populates="language_stats")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("run_id", "language_code", name="uq_run_language"),
|
||||
Index("idx_rls_run", "run_id"),
|
||||
)
|
||||
# #endregion TranslationRunLanguageStats
|
||||
|
||||
|
||||
# #endregion TranslateModels
|
||||
|
||||
Reference in New Issue
Block a user