busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(translate): remove all deprecated columns from model + DB

Browse Source 
Removed 6 legacy columns replaced by Phase 11 multi-language models:
- TranslationJob.target_language → target_languages (JSON)
- TranslationRecord.{llm_translation, user_edit, final_value} → TranslationLanguage
- TerminologyDictionary.{source_language, target_language} → DictionaryEntry per-entry

Affected files: model, schemas, 5 plugins, 4 test files
Alembic migration: aa1b2c3d4e5f (DROP COLUMN IF EXISTS)
Production DB: columns dropped via ALTER TABLE

Tests: 234 passed, 0 failed
This commit is contained in:
busya
2026-05-15 00:06:41 +03:00
parent 6717e2551d
commit 77d55c8e69
12 changed files with 95 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
backend/alembic/versions/aa1b2c3d4e5f_drop_deprecated_translate_columns.py Normal file
View File

@@ -0,0 +1,67 @@
"""Drop all deprecated columns from translate models
Deprecated columns removed:
- translation_jobs.target_language (use target_languages JSON instead)
- translation_records.llm_translation, user_edit, final_value (use TranslationLanguage instead)
- terminology_dictionaries.source_language, target_language (use per-entry DictionaryEntry instead)
Revision ID: aa1b2c3d4e5f
Revises: 2a7b8c9d0e1f
Create Date: 2026-05-14 23:59:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "aa1b2c3d4e5f"
down_revision: str | Sequence[str] | None = "2a7b8c9d0e1f"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Drop all deprecated columns using DROP COLUMN IF EXISTS for safety."""
bind = op.get_bind()
if bind.engine.name == "sqlite":
inspector = sa.inspect(bind)
_drop_if_exists_sqlite(inspector, "translation_jobs", "target_language")
_drop_if_exists_sqlite(inspector, "translation_records", "llm_translation")
_drop_if_exists_sqlite(inspector, "translation_records", "user_edit")
_drop_if_exists_sqlite(inspector, "translation_records", "final_value")
_drop_if_exists_sqlite(inspector, "terminology_dictionaries", "source_language")
_drop_if_exists_sqlite(inspector, "terminology_dictionaries", "target_language")
else:
# PostgreSQL and others: use IF EXISTS
op.execute("ALTER TABLE translation_jobs DROP COLUMN IF EXISTS target_language")
op.execute("ALTER TABLE translation_records DROP COLUMN IF EXISTS llm_translation")
op.execute("ALTER TABLE translation_records DROP COLUMN IF EXISTS user_edit")
op.execute("ALTER TABLE translation_records DROP COLUMN IF EXISTS final_value")
op.execute("ALTER TABLE terminology_dictionaries DROP COLUMN IF EXISTS source_language")
op.execute("ALTER TABLE terminology_dictionaries DROP COLUMN IF EXISTS target_language")
def downgrade() -> None:
"""Restore all deprecated columns (for rollback)."""
op.add_column("translation_jobs", sa.Column("target_language", sa.String(), nullable=True,
comment="Target language code (e.g. en, ru) [DEPRECATED: use target_languages]"))
op.add_column("translation_records", sa.Column("llm_translation", sa.Text(), nullable=True,
comment="[DEPRECATED: use TranslationLanguage]"))
op.add_column("translation_records", sa.Column("user_edit", sa.Text(), nullable=True,
comment="[DEPRECATED: use TranslationLanguage]"))
op.add_column("translation_records", sa.Column("final_value", sa.Text(), nullable=True,
comment="[DEPRECATED: use TranslationLanguage]"))
op.add_column("terminology_dictionaries", sa.Column("source_language", sa.String(), nullable=True,
comment="[DEPRECATED: use per-entry source_language]"))
op.add_column("terminology_dictionaries", sa.Column("target_language", sa.String(), nullable=True,
comment="[DEPRECATED: use per-entry target_language]"))
def _drop_if_exists_sqlite(inspector, table: str, column: str) -> None:
"""Drop a column from a SQLite table only if it exists."""
columns = [c["name"] for c in inspector.get_columns(table)]
if column in columns:
op.drop_column(table, column)

13
backend/src/models/translate.py
View File

@@ -45,8 +45,6 @@ class TranslationJob(Base):
context_columns = Column(JSON, nullable=True, comment="Context column names included in LLM prompt")
# LLM & processing settings
# @DEPRECATED — use target_languages
target_language = Column(String, nullable=True, comment="Target language code (e.g. en, ru) [DEPRECATED: use target_languages]")
# source_language removed — deprecated, auto-detected per row by LLM; use TranslationLanguage.source_language_detected instead
target_languages = Column(JSON, nullable=True, comment="List of BCP-47 target language codes (multi-language support)")
provider_id = Column(String, nullable=True, comment="LLM provider ID")
@@ -132,13 +130,6 @@ class TranslationRecord(Base):
translation_duration_ms = Column(Integer, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
# @DEPRECATED — use TranslationLanguage instead
llm_translation = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]")
# @DEPRECATED — use TranslationLanguage instead
user_edit = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]")
# @DEPRECATED — use TranslationLanguage instead
final_value = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]")
languages = relationship("TranslationLanguage", back_populates="record")
__table_args__ = (
@@ -208,10 +199,6 @@ class TerminologyDictionary(Base):
description = Column(Text, nullable=True)
source_dialect = Column(String, nullable=False)
target_dialect = Column(String, nullable=False)
# @DEPRECATED — use per-entry source_language instead, kept for backward compat
source_language = Column(String, nullable=True, comment="[DEPRECATED: use per-entry source_language]")
# @DEPRECATED — use per-entry target_language instead, kept for backward compat
target_language = Column(String, nullable=True, comment="[DEPRECATED: use per-entry target_language]")
is_active = Column(Boolean, default=True)
created_by = Column(String, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(UTC))

18
backend/src/plugins/translate/__tests__/test_dictionary.py
View File

@@ -790,14 +790,14 @@ def test_filter_for_batch_target_language_only(db_session: Session):
# region test_migrate_old_entries [TYPE Function]
# @PURPOSE: Verify migration populates language pair fields for old-style entries.
# @PURPOSE: Verify migration populates source_language from origin_source_language for old-style entries.
# NOTE: target_language inference from TerminologyDictionary.target_language was removed (deprecated column dropped).
def test_migrate_old_entries(db_session: Session):
# Create a dictionary with deprecated target_language set
# Create a dictionary (no deprecated target_language column)
d = TerminologyDictionary(
name="Old Dict",
source_dialect="a",
target_dialect="b",
target_language="ru",
)
db_session.add(d)
db_session.flush()
@@ -830,18 +830,18 @@ def test_migrate_old_entries(db_session: Session):
# Verify migration counts
assert result["total_processed"] >= 2
assert result["migrated_source"] >= 1 # entry2 had origin_source_language
assert result["migrated_target"] >= 2 # both should get target_language from dictionary
# migrated_target is 0 because dictionary-level target_language fallback was removed
assert result["migrated_target"] == 0
# Verify entry1 got target_language from dictionary
# Verify entry1: source remains "und" (no origin_source_language), target remains "und"
db_session.refresh(entry1)
assert entry1.target_language == "ru"
# source_language should remain "und" (no origin_source_language)
assert entry1.target_language == "und"
assert entry1.source_language == "und"
# Verify entry2 got source from origin and target from dictionary
# Verify entry2 got source from origin, target remains "und" (no dictionary-level fallback)
db_session.refresh(entry2)
assert entry2.source_language == "en"
assert entry2.target_language == "ru"
assert entry2.target_language == "und"
# endregion test_migrate_old_entries

5
backend/src/plugins/translate/__tests__/test_orchestrator.py
View File

@@ -49,7 +49,7 @@ def mock_job() -> MagicMock:
job.target_key_cols = ["id"]
job.source_key_cols = ["id"]
job.source_table = "source_data"
job.target_language = "en"
job.target_languages = ["en"]
job.provider_id = "provider-1"
job.batch_size = 50
job.upsert_strategy = "MERGE"
@@ -659,7 +659,6 @@ class TestTranslationExecutorMultiLang:
job = MagicMock(spec=TranslationJob)
job.id = "job-ml-1"
job.target_languages = ["ru", "en"]
job.target_language = None
job.target_dialect = "postgresql"
job.source_dialect = "postgresql"
job.translation_column = "name"
@@ -723,7 +722,6 @@ class TestTranslationExecutorMultiLang:
job = MagicMock(spec=TranslationJob)
job.id = "job-sar-1"
job.target_languages = ["fr", "en"]
job.target_language = None
job.target_dialect = "postgresql"
job.source_dialect = "postgresql"
job.translation_column = "name"
@@ -785,7 +783,6 @@ class TestTranslationExecutorMultiLang:
# Configure mock_job for multi-language
mock_job.target_languages = ["ru", "en"]
mock_job.target_language = None
mock_job.target_table = "target_tbl"
# Mock run

2
backend/src/plugins/translate/__tests__/test_preview.py
View File

@@ -30,7 +30,7 @@ def _make_mock_job(**overrides):
job.source_table = overrides.get("source_table", "source_table")
job.translation_column = overrides.get("translation_column", "title")
job.context_columns = overrides.get("context_columns", ["category", "description"])
job.target_language = overrides.get("target_language", "ru")
job.target_languages = overrides.get("target_languages", ["ru"])
job.provider_id = overrides.get("provider_id", "provider-1")
job.batch_size = overrides.get("batch_size", 50)
job.upsert_strategy = overrides.get("upsert_strategy", "MERGE")

12
backend/src/plugins/translate/dictionary.py
View File

@@ -578,16 +578,8 @@ class DictionaryManager:
entry.source_language = entry.origin_source_language
migrated_source += 1
# Try to infer target_language from the parent dictionary's deprecated target_language
if entry.target_language == "und":
dictionary = (
db.query(TerminologyDictionary)
.filter(TerminologyDictionary.id == entry.dictionary_id)
.first()
)
if dictionary and dictionary.target_language:
entry.target_language = dictionary.target_language
migrated_target += 1
# Note: target_language inference from dictionary-level column was removed
# because TerminologyDictionary.target_language was deprecated and dropped.
if entry.source_language == "und" and entry.target_language == "und":
skipped += 1

4
backend/src/plugins/translate/executor.py
View File

@@ -553,7 +553,7 @@ class TranslationExecutor:
rows_for_llm.append(row)
# Handle pre-translated (approved) rows
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
target_languages = job.target_languages or [job.target_dialect or "en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
@@ -652,7 +652,7 @@ class TranslationExecutor:
)
# Resolve target languages
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
target_languages = job.target_languages or [job.target_dialect or "en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
target_languages_str = ", ".join(target_languages)

4
backend/src/plugins/translate/orchestrator.py
View File

@@ -106,7 +106,6 @@ class TranslationOrchestrator:
"translation_column": job.translation_column,
"target_column": job.target_column,
"context_columns": job.context_columns,
"target_language": job.target_language,
"provider_id": job.provider_id,
"batch_size": job.batch_size,
"upsert_strategy": job.upsert_strategy,
@@ -204,7 +203,7 @@ class TranslationOrchestrator:
)
# Initialize per-language stats
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
target_languages = job.target_languages or [job.target_dialect or "en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
language_stats_map: dict[str, TranslationRunLanguageStats] = {}
@@ -1007,7 +1006,6 @@ class TranslationOrchestrator:
"source_datasource_id": job.source_datasource_id,
"translation_column": job.translation_column,
"context_columns": job.context_columns,
"target_language": job.target_language,
"target_languages": sorted(job.target_languages) if job.target_languages else [],
"provider_id": job.provider_id,
"batch_size": job.batch_size,

9
backend/src/plugins/translate/preview.py
View File

@@ -179,8 +179,8 @@ class TranslationPreview:
if not job.translation_column:
raise ValueError("Job must have a translation column configured for preview")
# Resolve target languages: prefer target_languages list, fallback to single target_language
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
# Resolve target languages from job configuration
target_languages = job.target_languages or [job.target_dialect or "en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
num_languages = len(target_languages)
@@ -643,7 +643,7 @@ class TranslationPreview:
# Resolve target languages
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"] if job else ["en"]
target_languages = job.target_languages or [job.target_dialect or "en"] if job else ["en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
@@ -713,7 +713,7 @@ class TranslationPreview:
# Resolve target languages
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"] if job else ["en"]
target_languages = job.target_languages or [job.target_dialect or "en"] if job else ["en"]
if not isinstance(target_languages, list):
target_languages = [str(target_languages)]
@@ -1065,7 +1065,6 @@ class TranslationPreview:
"source_datasource_id": job.source_datasource_id,
"translation_column": job.translation_column,
"context_columns": job.context_columns,
"target_language": job.target_language,
"provider_id": job.provider_id,
"batch_size": job.batch_size,
"upsert_strategy": job.upsert_strategy,

17
backend/src/plugins/translate/service.py
View File

@@ -256,7 +256,6 @@ class TranslateJobService:
translation_column=payload.translation_column,
target_column=payload.target_column,
context_columns=payload.context_columns or [],
target_language=payload.target_language,
target_languages=target_languages,
provider_id=payload.provider_id,
batch_size=payload.batch_size,
@@ -298,13 +297,13 @@ class TranslateJobService:
dict_ids = update_data.pop("dictionary_ids", None)
# Backward compat: if only target_language is set (old API), wrap into target_languages
if "target_language" in update_data and "target_languages" not in update_data:
target_languages = [update_data["target_language"]] if update_data["target_language"] else []
update_data["target_languages"] = target_languages
elif "target_languages" in update_data and "target_language" not in update_data:
# Keep deprecated field in sync
if update_data["target_languages"]:
update_data["target_language"] = update_data["target_languages"][0]
if "target_language" in update_data:
tl = update_data.pop("target_language")
if tl and "target_languages" not in update_data:
update_data["target_languages"] = [tl]
elif "target_languages" in update_data:
# target_languages is the canonical field now
pass
# Validate BCP-47 for target_languages
if update_data.get("target_languages"):
@@ -394,7 +393,6 @@ class TranslateJobService:
translation_column=source.translation_column,
target_column=source.target_column,
context_columns=source.context_columns,
target_language=source.target_language,
target_languages=source.target_languages,
provider_id=source.provider_id,
batch_size=source.batch_size,
@@ -497,7 +495,6 @@ def job_to_response(job: TranslationJob, dict_ids: list[str] | None = None) -> T
translation_column=job.translation_column,
target_column=job.target_column,
context_columns=job.context_columns or [],
target_language=job.target_language,
target_languages=job.target_languages,
provider_id=job.provider_id,
batch_size=job.batch_size or 50,

1
backend/src/schemas/translate.py
View File

@@ -105,7 +105,6 @@ class TranslateJobResponse(BaseModel):
translation_column: str | None = None
target_column: str | None = None
context_columns: list[str] | None = None
target_language: str | None = None
# source_language removed — deprecated, per-row auto-detected
target_languages: list[str] | None = None
provider_id: str | None = None

2
backend/tests/test_translate_jobs.py
View File

@@ -194,7 +194,7 @@ def test_create_job_valid(db_session):
assert job.context_columns == ["description"]
assert job.batch_size == 100
assert job.upsert_strategy == "MERGE"
assert job.target_language == "ru"
assert job.target_languages == ["ru"]
assert job.status == "DRAFT"
assert job.created_by == "test_user"