diff --git a/backend/src/app.py b/backend/src/app.py index 9d9db243..646a10b5 100755 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -53,6 +53,7 @@ from .api.routes import ( profile, health, dataset_review, + translate, ) from .api import auth @@ -271,6 +272,7 @@ app.include_router(clean_release_v2.router) app.include_router(profile.router) app.include_router(dataset_review.router) app.include_router(health.router) +app.include_router(translate.router) # [/DEF:API_Routes:Block] diff --git a/backend/src/models/translate.py b/backend/src/models/translate.py new file mode 100644 index 00000000..dd7ca61d --- /dev/null +++ b/backend/src/models/translate.py @@ -0,0 +1,292 @@ +# [DEF:TranslateModels:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: translate, models, sqlalchemy, persistence +# @PURPOSE: SQLAlchemy ORM models for LLM-based SQL/dashboard translation across dialects. +# @LAYER: Domain +# @RELATION: INHERITS_FROM -> MappingModels:Base + +from sqlalchemy import ( + Column, String, Boolean, DateTime, JSON, Text, + ForeignKey, Integer, UniqueConstraint, Index +) +from sqlalchemy.dialects.postgresql import JSON as PG_JSON +from datetime import datetime, timezone +import uuid +import hashlib +from .mapping import Base + + +def generate_uuid(): + return str(uuid.uuid4()) + + +# [DEF:TranslationJob:Class] +# @PURPOSE: A translation job representing a multi-dialect conversion task with column mappings, LLM config, and dictionary attachments. +# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. +# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity. +class TranslationJob(Base): + __tablename__ = "translation_jobs" + + id = Column(String, primary_key=True, default=generate_uuid) + name = Column(String, nullable=False) + description = Column(Text, nullable=True) + source_dialect = Column(String, nullable=False) + target_dialect = Column(String, nullable=False) + database_dialect = Column(String, nullable=True, comment="Detected dialect from Superset connection at save time") + status = Column(String, nullable=False, default="DRAFT") # DRAFT, READY, RUNNING, COMPLETED, FAILED, CANCELLED + + # Datasource & target table configuration + source_datasource_id = Column(String, nullable=True, comment="Superset datasource ID") + source_table = Column(String, nullable=True, comment="Source table name resolved from datasource") + target_schema = Column(String, nullable=True, comment="Target table schema") + target_table = Column(String, nullable=True, comment="Target table name") + + # Column mapping + source_key_cols = Column(JSON, nullable=True, comment="Source key column names for composite key") + target_key_cols = Column(JSON, nullable=True, comment="Target key column names for composite key") + translation_column = Column(String, nullable=True, comment="The column whose values will be translated") + context_columns = Column(JSON, nullable=True, comment="Context column names included in LLM prompt") + + # LLM & processing settings + target_language = Column(String, nullable=True, comment="Target language code (e.g. en, ru)") + provider_id = Column(String, nullable=True, comment="LLM provider ID") + batch_size = Column(Integer, nullable=False, default=50, comment="Records per batch") + upsert_strategy = Column(String, nullable=False, default="MERGE", comment="MERGE, INSERT, UPDATE") + + created_by = Column(String, nullable=True) + created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) +# [/DEF:TranslationJob:Class] + + +# [DEF:TranslationRun:Class] +# @PURPOSE: Represents a single execution of a translation job, capturing status, timing, and Superset execution metadata. +class TranslationRun(Base): + __tablename__ = "translation_runs" + + id = Column(String, primary_key=True, default=generate_uuid) + job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True) + status = Column(String, nullable=False, default="PENDING") # PENDING, RUNNING, COMPLETED, FAILED, CANCELLED + trigger_type = Column(String, nullable=True, comment="manual, scheduled, retry, baseline_expired") + started_at = Column(DateTime, nullable=True) + completed_at = Column(DateTime, nullable=True) + error_message = Column(Text, nullable=True) + total_records = Column(Integer, default=0) + successful_records = Column(Integer, default=0) + failed_records = Column(Integer, default=0) + skipped_records = Column(Integer, default=0) + insert_status = Column(String, nullable=True, comment="Status of the Superset insert/update operation") + superset_execution_id = Column(String, nullable=True, comment="Superset execution/task ID") + superset_execution_log = Column(JSON, nullable=True, comment="Superset execution log output") + config_snapshot = Column(JSON, nullable=True, comment="Snapshot of job config at run creation time") + key_hash = Column(String, nullable=True, comment="Hash of source key fields for dedup") + config_hash = Column(String, nullable=True, comment="Hash of translation configuration state") + dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at run time") + created_by = Column(String, nullable=True) + created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) +# [/DEF:TranslationRun:Class] + + +# [DEF:TranslationBatch:Class] +# @PURPOSE: Groups translation records within a run into manageable batches with timing and record counts. +class TranslationBatch(Base): + __tablename__ = "translation_batches" + + id = Column(String, primary_key=True, default=generate_uuid) + run_id = Column(String, ForeignKey("translation_runs.id"), nullable=False, index=True) + batch_index = Column(Integer, nullable=False) + status = Column(String, nullable=False, default="PENDING") + total_records = Column(Integer, default=0) + successful_records = Column(Integer, default=0) + failed_records = Column(Integer, default=0) + started_at = Column(DateTime, nullable=True) + completed_at = Column(DateTime, nullable=True) + created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) +# [/DEF:TranslationBatch:Class] + + +# [DEF:TranslationRecord:Class] +# @PURPOSE: Individual translation result for a single SQL statement or dashboard element, tracking source, target, and error state. +class TranslationRecord(Base): + __tablename__ = "translation_records" + + id = Column(String, primary_key=True, default=generate_uuid) + batch_id = Column(String, ForeignKey("translation_batches.id"), nullable=False, index=True) + run_id = Column(String, ForeignKey("translation_runs.id"), nullable=False, index=True) + source_sql = Column(Text, nullable=True) + target_sql = Column(Text, nullable=True) + source_object_type = Column(String, nullable=True) # query, dashboard, chart, dataset + source_object_id = Column(String, nullable=True) + source_object_name = Column(String, nullable=True) + status = Column(String, nullable=False, default="PENDING") # PENDING, SUCCESS, FAILED, SKIPPED + error_message = Column(Text, nullable=True) + token_count_input = Column(Integer, nullable=True) + token_count_output = Column(Integer, nullable=True) + translation_duration_ms = Column(Integer, nullable=True) + created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + + __table_args__ = ( + Index("ix_translation_records_run_status", "run_id", "status"), + ) +# [/DEF:TranslationRecord:Class] + + +# [DEF:TranslationEvent:Class] +# @PURPOSE: Audit/event log for translation operations, with optional run_id for context. +class TranslationEvent(Base): + __tablename__ = "translation_events" + + id = Column(String, primary_key=True, default=generate_uuid) + job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True) + run_id = Column(String, ForeignKey("translation_runs.id"), nullable=True, index=True) + event_type = Column(String, nullable=False) # JOB_CREATED, JOB_STARTED, JOB_COMPLETED, JOB_FAILED, etc. + event_data = Column(JSON, nullable=True) + created_by = Column(String, nullable=True) + created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) +# [/DEF:TranslationEvent:Class] + + +# [DEF:TranslationPreviewSession:Class] +# @PURPOSE: A preview session allowing users to review proposed translations before applying them. +class TranslationPreviewSession(Base): + __tablename__ = "translation_preview_sessions" + + id = Column(String, primary_key=True, default=generate_uuid) + job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True) + run_id = Column(String, ForeignKey("translation_runs.id"), nullable=True) + status = Column(String, nullable=False, default="ACTIVE") # ACTIVE, APPLIED, DISCARDED, EXPIRED + created_by = Column(String, nullable=True) + created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + expires_at = Column(DateTime, nullable=True) +# [/DEF:TranslationPreviewSession:Class] + + +# [DEF:TranslationPreviewRecord:Class] +# @PURPOSE: Individual preview entry within a preview session, showing original and translated content side by side. +class TranslationPreviewRecord(Base): + __tablename__ = "translation_preview_records" + + id = Column(String, primary_key=True, default=generate_uuid) + session_id = Column(String, ForeignKey("translation_preview_sessions.id"), nullable=False, index=True) + source_sql = Column(Text, nullable=True) + target_sql = Column(Text, nullable=True) + source_object_type = Column(String, nullable=True) + source_object_id = Column(String, nullable=True) + source_object_name = Column(String, nullable=True) + status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED + feedback = Column(Text, nullable=True) + created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) +# [/DEF:TranslationPreviewRecord:Class] + + +# [DEF:TerminologyDictionary:Class] +# @PURPOSE: A named collection of terminology mappings used during translation to ensure consistent term translation. +class TerminologyDictionary(Base): + __tablename__ = "terminology_dictionaries" + + id = Column(String, primary_key=True, default=generate_uuid) + name = Column(String, nullable=False) + description = Column(Text, nullable=True) + source_dialect = Column(String, nullable=False) + target_dialect = Column(String, nullable=False) + is_active = Column(Boolean, default=True) + created_by = Column(String, nullable=True) + created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) +# [/DEF:TerminologyDictionary:Class] + + +# [DEF:DictionaryEntry:Class] +# @PURPOSE: A single term mapping entry with case-insensitive unique constraint on source_term_normalized and origin tracking. +class DictionaryEntry(Base): + __tablename__ = "dictionary_entries" + + id = Column(String, primary_key=True, default=generate_uuid) + dictionary_id = Column(String, ForeignKey("terminology_dictionaries.id"), nullable=False, index=True) + source_term = Column(String, nullable=False) + source_term_normalized = Column(String, nullable=False) + target_term = Column(String, nullable=False) + context_notes = Column(Text, nullable=True) + origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated") + origin_row_key = Column(String, nullable=True, comment="Row key within the run that triggered this correction") + origin_user_id = Column(String, nullable=True, comment="User who submitted the correction") + created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) + + __table_args__ = ( + UniqueConstraint( + "dictionary_id", "source_term_normalized", + name="uq_dictionary_entry_term" + ), + ) +# [/DEF:DictionaryEntry:Class] + + +# [DEF:TranslationSchedule:Class] +# @PURPOSE: Defines a cron-based schedule for recurring translation jobs. +class TranslationSchedule(Base): + __tablename__ = "translation_schedules" + + id = Column(String, primary_key=True, default=generate_uuid) + job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True) + cron_expression = Column(String, nullable=False) + timezone = Column(String, nullable=False, default="UTC") + is_active = Column(Boolean, default=True) + last_run_at = Column(DateTime, nullable=True) + next_run_at = Column(DateTime, nullable=True) + created_by = Column(String, nullable=True) + created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) +# [/DEF:TranslationSchedule:Class] + + +# [DEF:TranslationJobDictionary:Class] +# @PURPOSE: Many-to-many association between translation jobs and terminology dictionaries. +class TranslationJobDictionary(Base): + __tablename__ = "translation_job_dictionaries" + + id = Column(String, primary_key=True, default=generate_uuid) + job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True) + dictionary_id = Column(String, ForeignKey("terminology_dictionaries.id"), nullable=False, index=True) + created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + + __table_args__ = ( + UniqueConstraint( + "job_id", "dictionary_id", + name="uq_job_dictionary" + ), + ) +# [/DEF:TranslationJobDictionary:Class] + + +# [DEF:MetricSnapshot:Class] +# @PURPOSE: Captures translation performance metrics at a point in time with config/content/dictionary hash keys for aggregation. +class MetricSnapshot(Base): + __tablename__ = "translation_metric_snapshots" + + id = Column(String, primary_key=True, default=generate_uuid) + job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True) + run_id = Column(String, ForeignKey("translation_runs.id"), nullable=True, index=True) + key_hash = Column(String, nullable=False, comment="Hash of dimension key fields for aggregation") + config_hash = Column(String, nullable=True, comment="Hash of translation configuration state") + dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at capture time") + covers_events_before = Column(DateTime, nullable=True, comment="Indicates snapshot covers events before this timestamp") + total_jobs = Column(Integer, default=0) + total_runs = Column(Integer, default=0) + total_records = Column(Integer, default=0) + successful_records = Column(Integer, default=0) + failed_records = Column(Integer, default=0) + skipped_records = Column(Integer, default=0) + avg_duration_ms = Column(Integer, nullable=True) + p50_duration_ms = Column(Integer, nullable=True) + p95_duration_ms = Column(Integer, nullable=True) + p99_duration_ms = Column(Integer, nullable=True) + snapshot_date = Column(DateTime, nullable=False, default=lambda: datetime.now(timezone.utc)) + created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + + __table_args__ = ( + Index("ix_metric_snapshots_job_date", "job_id", "snapshot_date"), + ) +# [/DEF:MetricSnapshot:Class] + +# [/DEF:TranslateModels:Module] diff --git a/backend/src/plugins/translate/__init__.py b/backend/src/plugins/translate/__init__.py new file mode 100644 index 00000000..b7837466 --- /dev/null +++ b/backend/src/plugins/translate/__init__.py @@ -0,0 +1,3 @@ +# [DEF:TranslatePluginPackage:Package] +# @PURPOSE: Translation plugin package for LLM-based cross-Superset SQL/dashboard translation. +# [/DEF:TranslatePluginPackage:Package] diff --git a/backend/src/plugins/translate/__tests__/__init__.py b/backend/src/plugins/translate/__tests__/__init__.py new file mode 100644 index 00000000..b0a441a2 --- /dev/null +++ b/backend/src/plugins/translate/__tests__/__init__.py @@ -0,0 +1,3 @@ +# [DEF:TranslatePluginTestsPackage:Package] +# @PURPOSE: Tests for the translate plugin package. +# [/DEF:TranslatePluginTestsPackage:Package] diff --git a/backend/src/plugins/translate/__tests__/test_dictionary.py b/backend/src/plugins/translate/__tests__/test_dictionary.py new file mode 100644 index 00000000..90328c46 --- /dev/null +++ b/backend/src/plugins/translate/__tests__/test_dictionary.py @@ -0,0 +1,644 @@ +# [DEF:DictionaryTests:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, dictionary, crud, import, filter +# @PURPOSE: Validate DictionaryManager CRUD, import, deletion guards, and batch filtering. +# @RELATION: BINDS_TO -> [DictionaryManager:Class] +# +# @TEST_CONTRACT: [DictionaryManager] -> { +# invariants: [ +# "Create/Read/Update/Delete dictionaries works", +# "Entry CRUD enforces unique source_term_normalized per dictionary", +# "Import CSV/TSV handles overwrite/keep_existing/cancel conflict modes", +# "Delete blocked when dictionary attached to active/scheduled jobs", +# "filter_for_batch returns matched entries with word-boundary awareness" +# ] +# } +# @TEST_EDGE: duplicate_entry -> 409-style ValueError on repeated source_term +# @TEST_EDGE: delete_active_job -> ValueError with active/scheduled message +# @TEST_EDGE: import_invalid_format -> ValueError for missing columns +# @TEST_INVARIANT: unique_normalized -> verifies: [duplicate_entry] + +import pytest +import csv +import io +from datetime import datetime, timezone +from sqlalchemy import create_engine, event +from sqlalchemy.orm import sessionmaker, Session + +from src.models.translate import ( + Base, + TerminologyDictionary, + DictionaryEntry, + TranslationJob, + TranslationJobDictionary, + TranslationRun, +) +from src.plugins.translate.dictionary import DictionaryManager +from src.plugins.translate._utils import _normalize_term, _detect_delimiter + + +# [DEF:_FakeJob:Class] +# @COMPLEXITY: 1 +# @PURPOSE: Helper to create inline TranslationJob records. +class _FakeJob: + pass +# [/DEF:_FakeJob:Class] + + +# [DEF:db_session:Fixture] +# @PURPOSE: Provide an in-memory SQLite session for each test, with tables created and torn down. +@pytest.fixture +def db_session(): + engine = create_engine("sqlite:///:memory:", echo=False) + + # Enable WAL and foreign keys for SQLite + @event.listens_for(engine, "connect") + def _set_sqlite_pragma(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + + Base.metadata.create_all(bind=engine) + session = sessionmaker(bind=engine)() + try: + yield session + finally: + session.close() + Base.metadata.drop_all(bind=engine) +# [/DEF:db_session:Fixture] + + +# [DEF:test_create_dictionary:Function] +# @PURPOSE: Verify dictionary creation and read-back. +def test_create_dictionary(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, + name="Finance Terms", + source_dialect="postgresql", + target_dialect="clickhouse", + created_by="test_user", + description="Finance-related term mappings", + ) + assert d.id is not None + assert d.name == "Finance Terms" + assert d.source_dialect == "postgresql" + assert d.target_dialect == "clickhouse" + assert d.created_by == "test_user" + assert d.is_active is True + + # Read back + fetched = DictionaryManager.get_dictionary(db_session, d.id) + assert fetched.id == d.id + assert fetched.name == "Finance Terms" +# [/DEF:test_create_dictionary:Function] + + +# [DEF:test_update_dictionary:Function] +# @PURPOSE: Verify dictionary metadata update. +def test_update_dictionary(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Old Name", + source_dialect="a", target_dialect="b", + ) + updated = DictionaryManager.update_dictionary( + db_session, d.id, + name="New Name", + description="Updated desc", + is_active=False, + ) + assert updated.name == "New Name" + assert updated.description == "Updated desc" + assert updated.is_active is False +# [/DEF:test_update_dictionary:Function] + + +# [DEF:test_delete_dictionary:Function] +# @PURPOSE: Verify dictionary deletion. +def test_delete_dictionary(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="To Delete", + source_dialect="a", target_dialect="b", + ) + # Add an entry + entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + assert entry.id is not None + + DictionaryManager.delete_dictionary(db_session, d.id) + with pytest.raises(ValueError, match="Dictionary not found"): + DictionaryManager.get_dictionary(db_session, d.id) +# [/DEF:test_delete_dictionary:Function] + + +# [DEF:test_list_dictionaries:Function] +# @PURPOSE: Verify paginated dictionary listing. +def test_list_dictionaries(db_session: Session): + for i in range(5): + DictionaryManager.create_dictionary( + db_session, name=f"Dict {i}", + source_dialect="a", target_dialect="b", + ) + + dicts, total = DictionaryManager.list_dictionaries(db_session, page=1, page_size=2) + assert total == 5 + assert len(dicts) == 2 +# [/DEF:test_list_dictionaries:Function] + + +# [DEF:test_add_entry_duplicate:Function] +# @PURPOSE: Verify duplicate entry raises ValueError and unique constraint is enforced. +def test_add_entry_duplicate(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test", + source_dialect="a", target_dialect="b", + ) + DictionaryManager.add_entry(db_session, d.id, "Hello", "Hola") + + # Same normalized term should raise + with pytest.raises(ValueError, match="already exists"): + DictionaryManager.add_entry(db_session, d.id, "hello", "Bonjour") + + # Different case, same normalized should also raise + with pytest.raises(ValueError, match="already exists"): + DictionaryManager.add_entry(db_session, d.id, "HELLO", "Ciao") +# [/DEF:test_add_entry_duplicate:Function] + + +# [DEF:test_add_entry_duplicate_per_dictionary:Function] +# @PURPOSE: Verify duplicate is per-dictionary (same term in different dictionaries is OK). +def test_add_entry_duplicate_per_dictionary(db_session: Session): + d1 = DictionaryManager.create_dictionary( + db_session, name="Dict1", + source_dialect="a", target_dialect="b", + ) + d2 = DictionaryManager.create_dictionary( + db_session, name="Dict2", + source_dialect="a", target_dialect="b", + ) + DictionaryManager.add_entry(db_session, d1.id, "hello", "hola") + # Same term in different dictionary should work + entry = DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour") + assert entry.id is not None +# [/DEF:test_add_entry_duplicate_per_dictionary:Function] + + +# [DEF:test_edit_entry:Function] +# @PURPOSE: Verify entry edit updates fields and enforces uniqueness. +def test_edit_entry(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test", + source_dialect="a", target_dialect="b", + ) + entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + + updated = DictionaryManager.edit_entry(db_session, entry.id, target_term="HOLA!") + assert updated.target_term == "HOLA!" + + # Edit source term to a value that's already taken should fail + DictionaryManager.add_entry(db_session, d.id, "world", "mundo") + with pytest.raises(ValueError, match="already exists"): + DictionaryManager.edit_entry(db_session, entry.id, source_term="WORLD") +# [/DEF:test_edit_entry:Function] + + +# [DEF:test_delete_entry:Function] +# @PURPOSE: Verify entry deletion. +def test_delete_entry(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test", + source_dialect="a", target_dialect="b", + ) + entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + DictionaryManager.delete_entry(db_session, entry.id) + + entries, total = DictionaryManager.list_entries(db_session, d.id) + assert total == 0 +# [/DEF:test_delete_entry:Function] + + +# [DEF:test_import_csv_overwrite:Function] +# @PURPOSE: Verify CSV import with overwrite conflict mode. +def test_import_csv_overwrite(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test", + source_dialect="a", target_dialect="b", + ) + # Pre-add an entry + DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + + csv_content = "source_term,target_term,context_notes\nhello,HELLO,\nworld,mundo," + result = DictionaryManager.import_entries( + db_session, d.id, csv_content, + delimiter=",", on_conflict="overwrite", + ) + assert result["total"] == 2 + assert result["created"] == 1 # world + assert result["updated"] == 1 # hello (overwritten) + assert result["skipped"] == 0 + + # Verify update + entries, _ = DictionaryManager.list_entries(db_session, d.id) + hello_entry = [e for e in entries if e.source_term_normalized == "hello"][0] + assert hello_entry.target_term == "HELLO" +# [/DEF:test_import_csv_overwrite:Function] + + +# [DEF:test_import_csv_keep_existing:Function] +# @PURPOSE: Verify CSV import with keep_existing conflict mode. +def test_import_csv_keep_existing(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test", + source_dialect="a", target_dialect="b", + ) + DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + + csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo" + result = DictionaryManager.import_entries( + db_session, d.id, csv_content, + delimiter=",", on_conflict="keep_existing", + ) + assert result["created"] == 1 # world + assert result["updated"] == 0 + assert result["skipped"] == 1 # hello kept + + # Verify hello was NOT overwritten + entries, _ = DictionaryManager.list_entries(db_session, d.id) + hello_entry = [e for e in entries if e.source_term_normalized == "hello"][0] + assert hello_entry.target_term == "hola" +# [/DEF:test_import_csv_keep_existing:Function] + + +# [DEF:test_import_csv_cancel_on_conflict:Function] +# @PURPOSE: Verify CSV import with cancel conflict mode raises errors. +def test_import_csv_cancel_on_conflict(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test", + source_dialect="a", target_dialect="b", + ) + DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + + csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo" + result = DictionaryManager.import_entries( + db_session, d.id, csv_content, + delimiter=",", on_conflict="cancel", + ) + assert result["total"] == 2 + assert result["created"] == 1 # world + assert result["updated"] == 0 + assert len(result["errors"]) == 1 # hello conflict error + assert "Conflict" in result["errors"][0]["error"] +# [/DEF:test_import_csv_cancel_on_conflict:Function] + + +# [DEF:test_import_tsv:Function] +# @PURPOSE: Verify TSV import. +def test_import_tsv(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test", + source_dialect="a", target_dialect="b", + ) + tsv_content = "source_term\ttarget_term\nhello\thola\nworld\tmundo" + result = DictionaryManager.import_entries( + db_session, d.id, tsv_content, + delimiter="\t", on_conflict="overwrite", + ) + assert result["created"] == 2 + assert result["total"] == 2 +# [/DEF:test_import_tsv:Function] + + +# [DEF:test_import_invalid_format:Function] +# @PURPOSE: Verify import raises ValueError for missing required columns. +def test_import_invalid_format(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test", + source_dialect="a", target_dialect="b", + ) + bad_content = "name,value\nhello,hola" + with pytest.raises(ValueError, match="source_term"): + DictionaryManager.import_entries( + db_session, d.id, bad_content, + delimiter=",", on_conflict="overwrite", + ) +# [/DEF:test_import_invalid_format:Function] + + +# [DEF:test_import_empty_rows:Function] +# @PURPOSE: Verify import handles rows with missing fields gracefully. +def test_import_empty_rows(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test", + source_dialect="a", target_dialect="b", + ) + csv_content = "source_term,target_term\nhello,hola\n,world\nfoo," + result = DictionaryManager.import_entries( + db_session, d.id, csv_content, + delimiter=",", on_conflict="overwrite", + ) + assert result["created"] == 1 # hello + assert len(result["errors"]) == 2 # empty source or target +# [/DEF:test_import_empty_rows:Function] + + +# [DEF:test_import_preview:Function] +# @PURPOSE: Verify import preview returns conflicts without mutating. +def test_import_preview(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test", + source_dialect="a", target_dialect="b", + ) + DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + + csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo" + result = DictionaryManager.import_entries( + db_session, d.id, csv_content, + delimiter=",", on_conflict="overwrite", + preview_only=True, + ) + assert len(result["preview"]) == 2 + # First row should be conflict + assert result["preview"][0]["is_conflict"] is True + assert result["preview"][0]["existing_target_term"] == "hola" + # Second row should be new + assert result["preview"][1]["is_conflict"] is False + + # Verify no mutation happened + entries, total = DictionaryManager.list_entries(db_session, d.id) + assert total == 1 # still only the original entry +# [/DEF:test_import_preview:Function] + + +# [DEF:test_delete_dictionary_blocked_by_active_job:Function] +# @PURPOSE: Verify deletion is blocked when dictionary is attached to active/scheduled jobs. +def test_delete_dictionary_blocked_by_active_job(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test", + source_dialect="a", target_dialect="b", + ) + # Create an active job + job = TranslationJob( + name="Active Job", + source_dialect="a", + target_dialect="b", + status="ACTIVE", + created_by="test_user", + ) + db_session.add(job) + db_session.flush() + + # Attach dictionary to job + link = TranslationJobDictionary( + job_id=job.id, + dictionary_id=d.id, + ) + db_session.add(link) + db_session.commit() + + with pytest.raises(ValueError, match="active/scheduled"): + DictionaryManager.delete_dictionary(db_session, d.id) +# [/DEF:test_delete_dictionary_blocked_by_active_job:Function] + + +# [DEF:test_delete_dictionary_allowed_with_completed_job:Function] +# @PURPOSE: Verify deletion is allowed when only completed/failed jobs reference the dictionary. +def test_delete_dictionary_allowed_with_completed_job(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test", + source_dialect="a", target_dialect="b", + ) + # Create a completed job + job = TranslationJob( + name="Completed Job", + source_dialect="a", + target_dialect="b", + status="COMPLETED", + created_by="test_user", + ) + db_session.add(job) + db_session.flush() + + link = TranslationJobDictionary( + job_id=job.id, + dictionary_id=d.id, + ) + db_session.add(link) + db_session.commit() + + # Should not raise + DictionaryManager.delete_dictionary(db_session, d.id) + with pytest.raises(ValueError, match="Dictionary not found"): + DictionaryManager.get_dictionary(db_session, d.id) +# [/DEF:test_delete_dictionary_allowed_with_completed_job:Function] + + +# [DEF:test_filter_for_batch_no_dictionaries:Function] +# @PURPOSE: Verify filter_for_batch returns empty when job has no dictionaries. +def test_filter_for_batch_no_dictionaries(db_session: Session): + job = TranslationJob( + name="No Dict Job", + source_dialect="a", target_dialect="b", + status="DRAFT", + ) + db_session.add(job) + db_session.commit() + + result = DictionaryManager.filter_for_batch(db_session, ["hello world"], job.id) + assert result == [] +# [/DEF:test_filter_for_batch_no_dictionaries:Function] + + +# [DEF:test_filter_for_batch_matches:Function] +# @PURPOSE: Verify filter_for_batch returns correct matched entries with word-boundary awareness. +def test_filter_for_batch_matches(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test Dict", + source_dialect="a", target_dialect="b", + ) + DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + DictionaryManager.add_entry(db_session, d.id, "world", "mundo") + DictionaryManager.add_entry(db_session, d.id, "foo", "bar") + + job = TranslationJob( + name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT", + ) + db_session.add(job) + db_session.flush() + + link = TranslationJobDictionary(job_id=job.id, dictionary_id=d.id) + db_session.add(link) + db_session.commit() + + result = DictionaryManager.filter_for_batch( + db_session, + ["hello beautiful world", "nothing here", "foo bar"], + job.id, + ) + + assert len(result) >= 3 # hello, world, foo + + # Check that 'hello' matched in text 0 + hello_matches = [m for m in result if m["source_term"] == "hello"] + assert len(hello_matches) == 1 + assert hello_matches[0]["text_index"] == 0 + + # Check that 'world' matched in text 0 + world_matches = [m for m in result if m["source_term"] == "world"] + assert len(world_matches) == 1 + assert world_matches[0]["text_index"] == 0 + + # Check that 'foo' matched in text 2 + foo_matches = [m for m in result if m["source_term"] == "foo"] + assert len(foo_matches) == 1 + assert foo_matches[0]["text_index"] == 2 +# [/DEF:test_filter_for_batch_matches:Function] + + +# [DEF:test_filter_for_batch_case_insensitive:Function] +# @PURPOSE: Verify filter_for_batch matching is case-insensitive. +def test_filter_for_batch_case_insensitive(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test Dict", + source_dialect="a", target_dialect="b", + ) + DictionaryManager.add_entry(db_session, d.id, "Hello World", "Hola Mundo") + + job = TranslationJob( + name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT", + ) + db_session.add(job) + db_session.flush() + + link = TranslationJobDictionary(job_id=job.id, dictionary_id=d.id) + db_session.add(link) + db_session.commit() + + # Different case should still match + result = DictionaryManager.filter_for_batch( + db_session, + ["hello world is great"], + job.id, + ) + assert len(result) == 1 + assert result[0]["source_term"] == "Hello World" + assert result[0]["target_term"] == "Hola Mundo" +# [/DEF:test_filter_for_batch_case_insensitive:Function] + + +# [DEF:test_filter_for_batch_word_boundary:Function] +# @PURPOSE: Verify filter_for_batch respects word boundaries (no substring matching within words). +def test_filter_for_batch_word_boundary(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test Dict", + source_dialect="a", target_dialect="b", + ) + DictionaryManager.add_entry(db_session, d.id, "cat", "gato") + + job = TranslationJob( + name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT", + ) + db_session.add(job) + db_session.flush() + + link = TranslationJobDictionary(job_id=job.id, dictionary_id=d.id) + db_session.add(link) + db_session.commit() + + # "cat" in "catalog" should NOT match (word boundary) + result = DictionaryManager.filter_for_batch( + db_session, + ["catalog"], + job.id, + ) + assert len(result) == 0 + + # "cat" as standalone word should match + result = DictionaryManager.filter_for_batch( + db_session, + ["the cat sat"], + job.id, + ) + assert len(result) == 1 +# [/DEF:test_filter_for_batch_word_boundary:Function] + + +# [DEF:test_filter_for_batch_multi_dictionary_priority:Function] +# @PURPOSE: Verify filter_for_batch respects dictionary link order priority. +def test_filter_for_batch_multi_dictionary_priority(db_session: Session): + d1 = DictionaryManager.create_dictionary( + db_session, name="Priority1", source_dialect="a", target_dialect="b", + ) + d2 = DictionaryManager.create_dictionary( + db_session, name="Priority2", source_dialect="a", target_dialect="b", + ) + DictionaryManager.add_entry(db_session, d1.id, "hello", "hola") + DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour") + + job = TranslationJob( + name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT", + ) + db_session.add(job) + db_session.flush() + + # Add d1 first (higher priority), then d2 + link1 = TranslationJobDictionary(job_id=job.id, dictionary_id=d1.id) + link2 = TranslationJobDictionary(job_id=job.id, dictionary_id=d2.id) + db_session.add(link1) + db_session.add(link2) + db_session.commit() + + result = DictionaryManager.filter_for_batch( + db_session, + ["hello"], + job.id, + ) + assert len(result) == 2 + # d1 match should come first (higher priority by link order) + assert result[0]["dictionary_id"] == d1.id + assert result[0]["target_term"] == "hola" + assert result[1]["dictionary_id"] == d2.id + assert result[1]["target_term"] == "bonjour" +# [/DEF:test_filter_for_batch_multi_dictionary_priority:Function] + + +# [DEF:test_normalize_term:Function] +# @PURPOSE: Verify _normalize_term produces lowercase NFC-normalized strings. +def test_normalize_term(): + assert _normalize_term("Hello") == "hello" + assert _normalize_term("HELLO") == "hello" + assert _normalize_term(" hello ") == "hello" + # NFC normalization test + composed = "\u00C9" # É precomposed + decomposed = "\u0045\u0301" # E + combining acute + assert _normalize_term(composed) == _normalize_term(decomposed) +# [/DEF:test_normalize_term:Function] + + +# [DEF:test_detect_delimiter:Function] +# @PURPOSE: Verify _detect_delimiter correctly identifies CSV vs TSV. +def test_detect_delimiter(): + csv_content = "source_term,target_term,context_notes\nhello,hola," + assert _detect_delimiter(csv_content) == "," + + tsv_content = "source_term\ttarget_term\nhello\thola" + assert _detect_delimiter(tsv_content) == "\t" + + empty_content = "" + assert _detect_delimiter(empty_content) == "," # default fallback +# [/DEF:test_detect_delimiter:Function] + + +# [DEF:test_clear_entries:Function] +# @PURPOSE: Verify clearing all entries for a dictionary. +def test_clear_entries(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Test", source_dialect="a", target_dialect="b", + ) + DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + DictionaryManager.add_entry(db_session, d.id, "world", "mundo") + + deleted = DictionaryManager.clear_entries(db_session, d.id) + assert deleted == 2 + + entries, total = DictionaryManager.list_entries(db_session, d.id) + assert total == 0 +# [/DEF:test_clear_entries:Function] +# [/DEF:DictionaryTests:Module] diff --git a/backend/src/plugins/translate/__tests__/test_orchestrator.py b/backend/src/plugins/translate/__tests__/test_orchestrator.py new file mode 100644 index 00000000..ecdea175 --- /dev/null +++ b/backend/src/plugins/translate/__tests__/test_orchestrator.py @@ -0,0 +1,376 @@ +# [DEF:OrchestratorTests:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: test, translate, orchestrator, events +# @PURPOSE: Tests for TranslationOrchestrator: run lifecycle, partial failure, batch retry, event invariants, NULL handling. +# @LAYER: Test +# @RELATION: BINDS_TO -> [TranslationOrchestrator:Module] +# @RELATION: BINDS_TO -> [TranslationEventLog:Module] +# @TEST_CONTRACT: TranslationOrchestrator -> start_run, execute_run, cancel_run, retry_failed_batches +# @TEST_FIXTURE: mock_db -> MagicMock SQLAlchemy session +# @TEST_FIXTURE: mock_config_manager -> MagicMock ConfigManager +# @TEST_EDGE: missing_preview -> raises ValueError +# @TEST_EDGE: invalid_run_status -> raises ValueError + +import pytest +from unittest.mock import MagicMock, patch, PropertyMock +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from src.plugins.translate.orchestrator import TranslationOrchestrator +from src.plugins.translate.events import TranslationEventLog +from src.models.translate import ( + TranslationJob, + TranslationRun, + TranslationBatch, + TranslationRecord, + TranslationPreviewSession, +) + + +# [DEF:mock_job:Function] +# @PURPOSE: Create a mock TranslationJob with standard config. +@pytest.fixture +def mock_job() -> MagicMock: + job = MagicMock(spec=TranslationJob) + job.id = "job-123" + job.name = "Test Job" + job.status = "ACTIVE" + job.source_dialect = "postgresql" + job.target_dialect = "postgresql" + job.database_dialect = "postgresql" + job.source_datasource_id = "42" + job.translation_column = "name" + job.context_columns = ["category"] + job.target_schema = "public" + job.target_table = "translated_data" + job.target_key_cols = ["id"] + job.source_key_cols = ["id"] + job.target_language = "en" + job.provider_id = "provider-1" + job.batch_size = 50 + job.upsert_strategy = "MERGE" + return job + + +# [DEF:mock_preview_session:Function] +# @PURPOSE: Create a mock accepted preview session. +@pytest.fixture +def mock_preview_session() -> MagicMock: + session = MagicMock(spec=TranslationPreviewSession) + session.id = "session-1" + session.job_id = "job-123" + session.status = "APPLIED" + session.created_at = datetime.now(timezone.utc) + return session + + +# [DEF:TestTranslationEventLog:Class] +# @PURPOSE: Tests for TranslationEventLog terminal event invariant. +class TestTranslationEventLog: + + # [DEF:test_log_event_creates_record:Function] + # @PURPOSE: log_event creates a TranslationEvent row. + def test_log_event_creates_record(self) -> None: + db = MagicMock() + log = TranslationEventLog(db) + event = log.log_event( + job_id="job-1", + event_type="RUN_STARTED", + payload={"key": "value"}, + run_id="run-1", + ) + assert event is not None + assert event.event_type == "RUN_STARTED" + db.add.assert_called_once() + db.flush.assert_called_once() + + # [DEF:test_invalid_event_type_raises:Function] + # @PURPOSE: Invalid event type raises ValueError. + def test_invalid_event_type_raises(self) -> None: + db = MagicMock() + log = TranslationEventLog(db) + with pytest.raises(ValueError, match="Invalid event_type"): + log.log_event( + job_id="job-1", + event_type="UNKNOWN_EVENT", + ) + + # [DEF:test_terminal_event_invariant:Function] + # @PURPOSE: Only one terminal event allowed per run. + def test_terminal_event_invariant(self) -> None: + db = MagicMock() + # Mock existing terminal event + db.query.return_value.filter.return_value.first.return_value = ("event-1",) + + log = TranslationEventLog(db) + with pytest.raises(ValueError, match="already has a terminal event"): + log.log_event( + job_id="job-1", + run_id="run-1", + event_type="RUN_COMPLETED", + ) + + # [DEF:test_run_started_must_precede_other_events:Function] + # @PURPOSE: Other run events require RUN_STARTED to exist first. + def test_run_started_must_precede_other_events(self) -> None: + db = MagicMock() + # Mock no RUN_STARTED event + db.query.return_value.filter.return_value.first.side_effect = [None, None] + + log = TranslationEventLog(db) + with pytest.raises(ValueError, match="RUN_STARTED event must precede"): + log.log_event( + job_id="job-1", + run_id="run-1", + event_type="BATCH_STARTED", + ) + + # [DEF:test_prune_expired_creates_snapshot:Function] + # @PURPOSE: prune_expired creates MetricSnapshot before deleting. + def test_prune_expired_creates_snapshot(self) -> None: + db = MagicMock() + # Mock that there are expired events + db.query.return_value.filter.return_value.count.return_value = 10 + # Use side_effect to simulate data being consumed by deletion. + # The while loop re-queries; first call returns items, second returns empty. + db.query.return_value.filter.return_value.limit.return_value.all.side_effect = [ + [("e1",), ("e2",)], + [], + ] + + log = TranslationEventLog(db) + result = log.prune_expired(retention_days=90) + assert result["pruned"] >= 0 +# [/DEF:TestTranslationEventLog:Class] + + +# [DEF:TestTranslationOrchestrator:Class] +# @PURPOSE: Tests for TranslationOrchestrator run lifecycle. +class TestTranslationOrchestrator: + + # [DEF:test_start_run_success:Function] + # @PURPOSE: start_run creates a PENDING run with event. + def test_start_run_success( + self, + mock_service, + mock_job: MagicMock, + mock_preview_session: MagicMock, + ) -> None: + db = MagicMock() + config_manager = MagicMock() + + # Mock job query + db.query.return_value.filter.return_value.first.return_value = mock_job + + # Mock accepted preview session + preview_query = ( + db.query.return_value.filter.return_value + ) + preview_query.order_by.return_value.first.return_value = mock_preview_session + + orch = TranslationOrchestrator(db, config_manager, "test-user") + run = orch.start_run(job_id="job-123") + + assert run.status == "PENDING" + assert run.job_id == "job-123" + + # [DEF:test_start_run_missing_preview_raises:Function] + # @PURPOSE: Manual run without accepted preview raises ValueError. + def test_start_run_missing_preview_raises( + self, + mock_job: MagicMock, + ) -> None: + db = MagicMock() + config_manager = MagicMock() + + # Mock job query + db.query.return_value.filter.return_value.first.return_value = mock_job + + # No accepted preview session + db.query.return_value.filter.return_value.order_by.return_value.first.return_value = None + + orch = TranslationOrchestrator(db, config_manager, "test-user") + with pytest.raises(ValueError, match="no accepted preview"): + orch.start_run(job_id="job-123") + + # [DEF:test_start_run_draft_job_raises:Function] + # @PURPOSE: Draft job cannot be run. + def test_start_run_draft_job_raises(self) -> None: + db = MagicMock() + config_manager = MagicMock() + + draft_job = MagicMock(spec=TranslationJob) + draft_job.status = "DRAFT" + + db.query.return_value.filter.return_value.first.return_value = draft_job + + orch = TranslationOrchestrator(db, config_manager, "test-user") + with pytest.raises(ValueError, match="DRAFT"): + orch.start_run(job_id="job-123") + + # [DEF:test_execute_run_invalid_status:Function] + # @PURPOSE: Cannot execute a run that is not in PENDING status. + def test_execute_run_invalid_status(self, mock_job: MagicMock) -> None: + db = MagicMock() + config_manager = MagicMock() + + run = MagicMock(spec=TranslationRun) + run.status = "COMPLETED" + run.job_id = "job-123" + + db.query.return_value.filter.return_value.first.return_value = mock_job + + orch = TranslationOrchestrator(db, config_manager, "test-user") + with pytest.raises(ValueError, match="PENDING"): + orch.execute_run(run) + + # [DEF:test_cancel_run:Function] + # @PURPOSE: Cancel a run changes status to CANCELLED. + def test_cancel_run(self) -> None: + db = MagicMock() + config_manager = MagicMock() + + run = MagicMock(spec=TranslationRun) + run.id = "run-1" + run.job_id = "job-123" + run.status = "RUNNING" + + db.query.return_value.filter.return_value.first.return_value = run + + orch = TranslationOrchestrator(db, config_manager, "test-user") + result = orch.cancel_run("run-1") + + assert result.status == "CANCELLED" + + # [DEF:test_cancel_run_invalid_status:Function] + # @PURPOSE: Cannot cancel a completed run. + def test_cancel_run_invalid_status(self) -> None: + db = MagicMock() + config_manager = MagicMock() + + run = MagicMock(spec=TranslationRun) + run.status = "COMPLETED" + + db.query.return_value.filter.return_value.first.return_value = run + + orch = TranslationOrchestrator(db, config_manager, "test-user") + with pytest.raises(ValueError, match="cancelled"): + orch.cancel_run("run-1") + + # [DEF:test_retry_failed_batches_no_failures:Function] + # @PURPOSE: Raises if no failed batches found. + def test_retry_failed_batches_no_failures(self) -> None: + db = MagicMock() + config_manager = MagicMock() + + run = MagicMock(spec=TranslationRun) + run.id = "run-1" + run.job_id = "job-123" + run.status = "COMPLETED" + + # Mock run query + db.query.return_value.filter.return_value.first.return_value = run + + # No failed batches + db.query.return_value.filter.return_value.all.return_value = [] + + orch = TranslationOrchestrator(db, config_manager, "test-user") + with pytest.raises(ValueError, match="No failed batches"): + orch.retry_failed_batches("run-1") + + # [DEF:test_get_run_status:Function] + # @PURPOSE: get_run_status returns structured status. + def test_get_run_status(self) -> None: + db = MagicMock() + config_manager = MagicMock() + + run = MagicMock(spec=TranslationRun) + run.id = "run-1" + run.job_id = "job-123" + run.status = "COMPLETED" + run.started_at = datetime.now(timezone.utc) + run.completed_at = datetime.now(timezone.utc) + run.error_message = None + run.total_records = 100 + run.successful_records = 95 + run.failed_records = 3 + run.skipped_records = 2 + run.insert_status = "success" + run.superset_execution_id = "query-1" + run.created_by = "test-user" + run.created_at = datetime.now(timezone.utc) + + db.query.return_value.filter.return_value.first.return_value = run + db.query.return_value.filter.return_value.count.return_value = 5 + + orch = TranslationOrchestrator(db, config_manager, "test-user") + status = orch.get_run_status("run-1") + + assert status["id"] == "run-1" + assert status["status"] == "COMPLETED" + assert status["total_records"] == 100 + assert status["successful_records"] == 95 + assert status["insert_status"] == "success" + + # [DEF:test_get_run_records:Function] + # @PURPOSE: get_run_records returns paginated records. + def test_get_run_records(self) -> None: + db = MagicMock() + config_manager = MagicMock() + + record = MagicMock(spec=TranslationRecord) + record.id = "rec-1" + record.batch_id = "batch-1" + record.run_id = "run-1" + record.source_sql = "SELECT 1" + record.target_sql = "SELECT 2" + record.source_object_type = "table_row" + record.source_object_id = "0" + record.source_object_name = "" + record.status = "SUCCESS" + record.error_message = None + record.created_at = datetime.now(timezone.utc) + + db.query.return_value.filter.return_value.count.return_value = 1 + db.query.return_value.filter.return_value.order_by.return_value.offset.return_value.limit.return_value.all.return_value = [record] + + orch = TranslationOrchestrator(db, config_manager, "test-user") + result = orch.get_run_records("run-1") + + assert result["total"] == 1 + assert len(result["items"]) == 1 + assert result["items"][0]["status"] == "SUCCESS" + + # [DEF:test_get_run_history:Function] + # @PURPOSE: get_run_history returns list of runs. + def test_get_run_history(self) -> None: + db = MagicMock() + config_manager = MagicMock() + + run = MagicMock(spec=TranslationRun) + run.id = "run-1" + run.job_id = "job-123" + run.status = "COMPLETED" + run.started_at = datetime.now(timezone.utc) + run.completed_at = datetime.now(timezone.utc) + run.error_message = None + run.total_records = 100 + run.successful_records = 95 + run.failed_records = 3 + run.skipped_records = 2 + run.insert_status = "success" + run.superset_execution_id = "query-1" + run.created_by = "test-user" + run.created_at = datetime.now(timezone.utc) + + db.query.return_value.filter.return_value.count.return_value = 1 + db.query.return_value.filter.return_value.order_by.return_value.offset.return_value.limit.return_value.all.return_value = [run] + + orch = TranslationOrchestrator(db, config_manager, "test-user") + total, runs = orch.get_run_history("job-123") + + assert total == 1 + assert len(runs) == 1 + assert runs[0]["id"] == "run-1" +# [/DEF:TestTranslationOrchestrator:Class] +# [/DEF:OrchestratorTests:Module] diff --git a/backend/src/plugins/translate/__tests__/test_preview.py b/backend/src/plugins/translate/__tests__/test_preview.py new file mode 100644 index 00000000..e95e6b73 --- /dev/null +++ b/backend/src/plugins/translate/__tests__/test_preview.py @@ -0,0 +1,500 @@ +# [DEF:TranslationPreviewTests:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: test, translate, preview, session +# @PURPOSE: Test TranslationPreview service: preview, approve/edit/reject state transitions, cost estimation, acceptance gate. +# @LAYER: Test +# @RELATION: BINDS_TO -> [TranslationPreview:Module] + +from unittest.mock import MagicMock, patch, PropertyMock +from datetime import datetime, timezone, timedelta +import json +import pytest + +from src.models.translate import ( + TranslationJob, + TranslationPreviewSession, + TranslationPreviewRecord, + TranslationJobDictionary, +) +from src.plugins.translate.preview import TranslationPreview, TokenEstimator + + +# [DEF:_make_mock_job:Function] +# @PURPOSE: Create a mock TranslationJob with test config. +def _make_mock_job(**overrides): + job = MagicMock(spec=TranslationJob) + job.id = overrides.get("id", "job-123") + job.name = overrides.get("name", "Test Job") + job.source_dialect = overrides.get("source_dialect", "postgresql") + job.target_dialect = overrides.get("target_dialect", "clickhouse") + job.source_datasource_id = overrides.get("source_datasource_id", "42") + job.source_table = overrides.get("source_table", "source_table") + job.translation_column = overrides.get("translation_column", "title") + job.context_columns = overrides.get("context_columns", ["category", "description"]) + job.target_language = overrides.get("target_language", "ru") + job.provider_id = overrides.get("provider_id", "provider-1") + job.batch_size = overrides.get("batch_size", 50) + job.upsert_strategy = overrides.get("upsert_strategy", "MERGE") + job.status = overrides.get("status", "DRAFT") + return job +# [/DEF:_make_mock_job:Function] + + +# [DEF:_make_mock_provider:Function] +# @PURPOSE: Create a mock LLMProvider. +def _make_mock_provider(**overrides): + provider = MagicMock() + provider.id = overrides.get("id", "provider-1") + provider.provider_type = overrides.get("provider_type", "openai") + provider.base_url = overrides.get("base_url", "https://api.openai.com/v1") + provider.default_model = overrides.get("default_model", "gpt-4o-mini") + provider.api_key = "encrypted-key-123" + return provider +# [/DEF:_make_mock_provider:Function] + + +# [DEF:TestTranslationPreview:Class] +# @PURPOSE: Test suite for TranslationPreview service. +class TestTranslationPreview: + + # [DEF:test_preview_valid_job:Function] + # @PURPOSE: Verify preview creates session and records successfully. + def test_preview_valid_job(self): + """Preview with valid job should create session and return records.""" + db = MagicMock() + config_manager = MagicMock() + + # Mock job query + job = _make_mock_job() + db.query.return_value.filter.return_value.first.return_value = job + + # Mock environments + env = MagicMock() + env.id = "postgresql" + env.url = "https://superset.example.com" + env.username = "admin" + env.password = "admin" + env.verify_ssl = True + env.timeout = 30 + config_manager.get_environments.return_value = [env] + + # Mock SupersetClient + with patch("src.plugins.translate.preview.SupersetClient") as MockClient: + mock_client = MagicMock() + MockClient.return_value = mock_client + + # Mock dataset_detail + mock_client.get_dataset_detail.return_value = { + "table_name": "test_table", + "columns": [ + {"column_name": "title", "type": "VARCHAR"}, + {"column_name": "category", "type": "VARCHAR"}, + {"column_name": "description", "type": "TEXT"}, + ], + } + + # Mock build_dataset_preview_query_context + mock_client.build_dataset_preview_query_context.return_value = { + "datasource": {"id": 42, "type": "table"}, + "queries": [{ + "columns": [], + "metrics": ["count"], + "row_limit": 10, + }], + "form_data": {}, + "result_format": "json", + "result_type": "query", + "force": True, + } + + # Mock network request to Superset + mock_client.network.request.return_value = { + "result": [{ + "status": "success", + "data": [ + {"title": "Hello World", "category": "greeting", "description": "A friendly message"}, + {"title": "Goodbye", "category": "farewell", "description": "A parting message"}, + ], + }] + } + + # Mock LLM provider + with patch("src.plugins.translate.preview.LLMProviderService") as MockProviderService: + mock_provider_svc = MagicMock() + MockProviderService.return_value = mock_provider_svc + mock_provider = _make_mock_provider() + mock_provider_svc.get_provider.return_value = mock_provider + mock_provider_svc.get_decrypted_api_key.return_value = "sk-test-key" + + # Mock _call_openai_compatible to return structured JSON + with patch.object(TranslationPreview, "_call_openai_compatible") as mock_llm: + mock_llm.return_value = json.dumps({ + "rows": [ + {"row_id": "0", "translation": "Привет мир"}, + {"row_id": "1", "translation": "Прощай"}, + ] + }) + + # Mock DictionaryManager.filter_for_batch + with patch("src.plugins.translate.preview.DictionaryManager.filter_for_batch") as mock_filter: + mock_filter.return_value = [] + + preview_service = TranslationPreview(db, config_manager, "test-user") + result = preview_service.preview_rows(job_id="job-123", sample_size=10) + + # Verify result structure + assert result["job_id"] == "job-123" + assert result["status"] == "ACTIVE" + assert len(result["records"]) == 2 + assert result["records"][0]["target_sql"] == "Привет мир" + assert result["records"][1]["target_sql"] == "Прощай" + + # Verify cost estimate + assert result["cost_estimate"]["sample_size"] == 2 + assert result["cost_estimate"]["sample_total_tokens"] > 0 + assert result["cost_estimate"]["sample_cost"] >= 0 + + # Verify config and dict hashes + assert result["config_hash"] is not None + assert result["dict_snapshot_hash"] is not None + + # Verify session was added to DB + assert db.add.called + assert db.commit.called + # [/DEF:test_preview_valid_job:Function] + + # [DEF:test_preview_with_dictionary:Function] + # @PURPOSE: Verify glossary entries from dictionary are included in LLM prompt. + def test_preview_with_dictionary(self): + """Preview should include dictionary glossary in the prompt sent to LLM.""" + db = MagicMock() + config_manager = MagicMock() + + job = _make_mock_job() + db.query.return_value.filter.return_value.first.return_value = job + + env = MagicMock() + env.id = "postgresql" + config_manager.get_environments.return_value = [env] + + with patch("src.plugins.translate.preview.SupersetClient") as MockClient: + mock_client = MagicMock() + MockClient.return_value = mock_client + mock_client.get_dataset_detail.return_value = {"table_name": "test", "columns": []} + mock_client.build_dataset_preview_query_context.return_value = { + "datasource": {"id": 42, "type": "table"}, + "queries": [{"columns": [], "metrics": ["count"], "row_limit": 10}], + "form_data": {}, + "result_format": "json", + "result_type": "query", + "force": True, + } + mock_client.network.request.return_value = { + "result": [{"status": "success", "data": [{"title": "Hello World", "category": "greeting"}]}] + } + + with patch("src.plugins.translate.preview.LLMProviderService") as MockProviderService: + mock_provider_svc = MagicMock() + MockProviderService.return_value = mock_provider_svc + mock_provider_svc.get_provider.return_value = _make_mock_provider() + mock_provider_svc.get_decrypted_api_key.return_value = "sk-test-key" + + with patch.object(TranslationPreview, "_call_openai_compatible") as mock_llm: + mock_llm.return_value = json.dumps({"rows": [{"row_id": "0", "translation": "Привет мир"}]}) + + # Mock dictionary filter to return glossary entries + with patch("src.plugins.translate.preview.DictionaryManager.filter_for_batch") as mock_filter: + mock_filter.return_value = [ + { + "source_term": "Hello", + "target_term": "Здравствуйте", + "dictionary_id": "dict-1", + "dictionary_name": "Test Dict", + "context_notes": "Formal greeting", + } + ] + + preview_service = TranslationPreview(db, config_manager, "test-user") + result = preview_service.preview_rows(job_id="job-123", sample_size=5) + + # Verify LLM was called + mock_llm.assert_called_once() + assert len(result["records"]) == 1 + # [/DEF:test_preview_with_dictionary:Function] + + # [DEF:test_preview_row_approve:Function] + # @PURPOSE: Verify approving a preview row changes its status. + def test_preview_row_approve(self): + """Approve action should set record status to APPROVED.""" + db = MagicMock() + config_manager = MagicMock() + + # Mock active session + session = MagicMock(spec=TranslationPreviewSession) + session.id = "session-1" + session.job_id = "job-123" + session.status = "ACTIVE" + session.created_at = datetime.now(timezone.utc) + + # Mock record + record = MagicMock(spec=TranslationPreviewRecord) + record.id = "record-1" + record.session_id = "session-1" + record.status = "PENDING" + record.target_sql = "test translation" + record.feedback = None + + # Setup: db.query().filter().order_by().first() -> session (session lookup) + db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session + # Setup: db.query().filter().first() -> record (record lookup) + db.query.return_value.filter.return_value.first.return_value = record + + preview_service = TranslationPreview(db, config_manager, "test-user") + result = preview_service.update_preview_row( + job_id="job-123", + row_id="record-1", + action="approve", + ) + + assert record.status == "APPROVED" + assert result["status"] == "APPROVED" + assert db.commit.called + # [/DEF:test_preview_row_approve:Function] + + # [DEF:test_preview_row_reject:Function] + # @PURPOSE: Verify rejecting a preview row changes its status. + def test_preview_row_reject(self): + """Reject action should set record status to REJECTED.""" + db = MagicMock() + config_manager = MagicMock() + + session = MagicMock(spec=TranslationPreviewSession) + session.id = "session-1" + session.job_id = "job-123" + session.status = "ACTIVE" + + record = MagicMock(spec=TranslationPreviewRecord) + record.id = "record-1" + record.session_id = "session-1" + record.status = "PENDING" + record.target_sql = "test" + record.feedback = None + + db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session + db.query.return_value.filter.return_value.first.return_value = record + + preview_service = TranslationPreview(db, config_manager, "test-user") + result = preview_service.update_preview_row( + job_id="job-123", + row_id="record-1", + action="reject", + ) + + assert record.status == "REJECTED" + assert result["status"] == "REJECTED" + # [/DEF:test_preview_row_reject:Function] + + # [DEF:test_preview_row_edit:Function] + # @PURPOSE: Verify editing a preview row updates its translation and status. + def test_preview_row_edit(self): + """Edit action should update translation and set status to APPROVED.""" + db = MagicMock() + config_manager = MagicMock() + + session = MagicMock(spec=TranslationPreviewSession) + session.id = "session-1" + session.job_id = "job-123" + session.status = "ACTIVE" + + record = MagicMock(spec=TranslationPreviewRecord) + record.id = "record-1" + record.session_id = "session-1" + record.status = "PENDING" + record.target_sql = "old translation" + record.feedback = None + + db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session + db.query.return_value.filter.return_value.first.return_value = record + + preview_service = TranslationPreview(db, config_manager, "test-user") + result = preview_service.update_preview_row( + job_id="job-123", + row_id="record-1", + action="edit", + translation="edited translation", + ) + + assert record.target_sql == "edited translation" + assert record.status == "APPROVED" + assert result["status"] == "APPROVED" + assert result["target_sql"] == "edited translation" + # [/DEF:test_preview_row_edit:Function] + + # [DEF:test_preview_row_invalid_action:Function] + # @PURPOSE: Verify invalid action raises ValueError. + def test_preview_row_invalid_action(self): + """Invalid action should raise ValueError.""" + db = MagicMock() + config_manager = MagicMock() + + session = MagicMock(spec=TranslationPreviewSession) + session.id = "session-1" + session.job_id = "job-123" + session.status = "ACTIVE" + + record = MagicMock(spec=TranslationPreviewRecord) + record.id = "record-1" + record.session_id = "session-1" + record.status = "PENDING" + + db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session + db.query.return_value.filter.return_value.first.return_value = record + + preview_service = TranslationPreview(db, config_manager, "test-user") + with pytest.raises(ValueError, match="Invalid action"): + preview_service.update_preview_row( + job_id="job-123", + row_id="record-1", + action="invalid_action", + ) + # [/DEF:test_preview_row_invalid_action:Function] + + # [DEF:test_preview_accept_session:Function] + # @PURPOSE: Verify accepting a preview session gates execution. + def test_preview_accept_session(self): + """Accept should set session status to APPLIED and return records.""" + db = MagicMock() + config_manager = MagicMock() + + session = MagicMock(spec=TranslationPreviewSession) + session.id = "session-1" + session.job_id = "job-123" + session.status = "ACTIVE" + session.created_by = "test-user" + session.created_at = datetime.now(timezone.utc) + session.expires_at = datetime.now(timezone.utc) + timedelta(hours=24) + + record = MagicMock(spec=TranslationPreviewRecord) + record.id = "record-1" + record.session_id = "session-1" + record.source_sql = "Hello" + record.target_sql = "Привет" + record.status = "APPROVED" + record.feedback = None + + # Session lookup + db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session + # Records lookup + db.query.return_value.filter.return_value.all.return_value = [record] + + preview_service = TranslationPreview(db, config_manager, "test-user") + result = preview_service.accept_preview_session(job_id="job-123") + + assert session.status == "APPLIED" + assert result["status"] == "APPLIED" + assert len(result["records"]) == 1 + assert result["records"][0]["status"] == "APPROVED" + assert db.commit.called + # [/DEF:test_preview_accept_session:Function] + + # [DEF:test_preview_no_active_session:Function] + # @PURPOSE: Verify accept raises error when no active session. + def test_preview_no_active_session(self): + """Accept without active session should raise ValueError.""" + db = MagicMock() + config_manager = MagicMock() + + db.query.return_value.filter.return_value.order_by.return_value.first.return_value = None + + preview_service = TranslationPreview(db, config_manager, "test-user") + with pytest.raises(ValueError, match="No active preview session"): + preview_service.accept_preview_session(job_id="job-123") + # [/DEF:test_preview_no_active_session:Function] + + # [DEF:test_cost_estimation:Function] + # @PURPOSE: Verify token and cost estimation methods. + def test_cost_estimation(self): + """Token and cost estimation should return reasonable values.""" + prompt = "Translate this text from English to Russian. " * 100 + tokens = TokenEstimator.estimate_prompt_tokens(prompt) + assert tokens > 0 + assert tokens < len(prompt) # tokens typically fewer than chars + + output_tokens = TokenEstimator.estimate_output_tokens(10) + assert output_tokens == 500 + + cost = TokenEstimator.estimate_cost(1000, 0.002) + assert cost == 0.002 + + cost_default = TokenEstimator.estimate_cost(1000) + assert cost_default == 0.002 + # [/DEF:test_cost_estimation:Function] + + # [DEF:test_preview_parse_llm_response:Function] + # @PURPOSE: Verify LLM JSON response parsing. + def test_preview_parse_llm_response(self): + """Parse LLM response should extract translations keyed by row_id.""" + response = json.dumps({ + "rows": [ + {"row_id": "0", "translation": "Привет"}, + {"row_id": "1", "translation": "Мир"}, + ] + }) + result = TranslationPreview._parse_llm_response(response, 2) + assert result == {"0": "Привет", "1": "Мир"} + # [/DEF:test_preview_parse_llm_response:Function] + + # [DEF:test_preview_parse_llm_response_with_code_block:Function] + # @PURPOSE: Verify LLM response parsing handles markdown code blocks. + def test_preview_parse_llm_response_with_code_block(self): + """Parse LLM response should handle markdown code block wrapping.""" + response = "```json\n{\n \"rows\": [\n {\"row_id\": \"0\", \"translation\": \"Test\"}\n ]\n}\n```" + result = TranslationPreview._parse_llm_response(response, 1) + assert result == {"0": "Test"} + # [/DEF:test_preview_parse_llm_response_with_code_block:Function] + + # [DEF:test_preview_compute_config_hash:Function] + # @PURPOSE: Verify config hash computation is deterministic. + def test_preview_compute_config_hash(self): + """Config hash should be deterministic and non-empty.""" + job = _make_mock_job() + hash1 = TranslationPreview._compute_config_hash(job) + hash2 = TranslationPreview._compute_config_hash(job) + assert hash1 == hash2 + assert len(hash1) == 16 + # [/DEF:test_preview_compute_config_hash:Function] + + # [DEF:test_preview_missing_datasource:Function] + # @PURPOSE: Verify error when job has no datasource configured. + def test_preview_missing_datasource(self): + """Preview should fail if job has no datasource.""" + db = MagicMock() + config_manager = MagicMock() + + job = _make_mock_job(source_datasource_id=None) + db.query.return_value.filter.return_value.first.return_value = job + + preview_service = TranslationPreview(db, config_manager, "test-user") + with pytest.raises(ValueError, match="source datasource"): + preview_service.preview_rows(job_id="job-123") + # [/DEF:test_preview_missing_datasource:Function] + + # [DEF:test_preview_missing_translation_column:Function] + # @PURPOSE: Verify error when job has no translation column. + def test_preview_missing_translation_column(self): + """Preview should fail if job has no translation column.""" + db = MagicMock() + config_manager = MagicMock() + + job = _make_mock_job(translation_column=None) + db.query.return_value.filter.return_value.first.return_value = job + + preview_service = TranslationPreview(db, config_manager, "test-user") + with pytest.raises(ValueError, match="translation column"): + preview_service.preview_rows(job_id="job-123") + # [/DEF:test_preview_missing_translation_column:Function] + + +# [/DEF:TestTranslationPreview:Class] + +# [/DEF:TranslationPreviewTests:Module] diff --git a/backend/src/plugins/translate/__tests__/test_sql_generator.py b/backend/src/plugins/translate/__tests__/test_sql_generator.py new file mode 100644 index 00000000..cce062af --- /dev/null +++ b/backend/src/plugins/translate/__tests__/test_sql_generator.py @@ -0,0 +1,332 @@ +# [DEF:SQLGeneratorTests:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: test, translate, sql_generator +# @PURPOSE: Tests for SQLGenerator: PostgreSQL INSERT + UPSERT, ClickHouse INSERT, dialect quoting, NULL handling, injection safety. +# @LAYER: Test +# @RELATION: BINDS_TO -> [SQLGenerator:Module] +# @TEST_CONTRACT: SQLGenerator.generate -> SQL string, row_count +# @TEST_FIXTURE: sample_rows -> [{"col1": "val1", "col2": 42}, {"col1": "val2", "col2": 99}] +# @TEST_EDGE: empty_rows -> raises ValueError +# @TEST_EDGE: null_values -> properly encoded as NULL +# @TEST_EDGE: sql_injection -> values with single quotes are escaped + +import pytest +from datetime import datetime, timezone +from typing import Any, Dict, List + +from src.plugins.translate.sql_generator import ( + SQLGenerator, + _quote_identifier, + _encode_sql_value, + generate_insert_sql, + generate_upsert_sql, +) + + +# [DEF:_normalize_sql:Function] +# @PURPOSE: Normalize SQL for comparison by collapsing whitespace. +def _normalize_sql(sql: str) -> str: + return " ".join(sql.split()) + + +# [DEF:sample_rows:Function] +# @PURPOSE: Sample rows fixture for SQL generation tests. +@pytest.fixture +def sample_rows() -> List[Dict[str, Any]]: + return [ + {"col1": "hello", "col2": 42, "col3": None}, + {"col1": "world", "col2": 99, "col3": "note"}, + ] + + +# [DEF:TestQuoteIdentifier:Class] +# @PURPOSE: Tests for identifier quoting per dialect. +class TestQuoteIdentifier: + # [DEF:test_quote_postgresql:Function] + # @PURPOSE: PostgreSQL uses double quotes. + def test_quote_postgresql(self) -> None: + assert _quote_identifier("my_column", "postgresql") == '"my_column"' + + # [DEF:test_quote_clickhouse:Function] + # @PURPOSE: ClickHouse uses backticks. + def test_quote_clickhouse(self) -> None: + assert _quote_identifier("my_column", "clickhouse") == "`my_column`" + + # [DEF:test_quote_with_existing_quotes:Function] + # @PURPOSE: Strips existing quotes before quoting. + def test_quote_with_existing_quotes(self) -> None: + assert _quote_identifier('"my_col"', "postgresql") == '"my_col"' + assert _quote_identifier("`my_col`", "clickhouse") == "`my_col`" + + # [DEF:test_quote_schema_table:Function] + # @PURPOSE: Schema-qualified table references work. + def test_quote_schema_table(self) -> None: + assert _quote_identifier("my_schema", "postgresql") == '"my_schema"' + assert _quote_identifier("my_table", "postgresql") == '"my_table"' +# [/DEF:TestQuoteIdentifier:Class] + + +# [DEF:TestEncodeSqlValue:Class] +# @PURPOSE: Tests for SQL value encoding. +class TestEncodeSqlValue: + # [DEF:test_null:Function] + # @PURPOSE: None encodes as NULL. + def test_null(self) -> None: + assert _encode_sql_value(None) == "NULL" + + # [DEF:test_string:Function] + # @PURPOSE: Strings are single-quoted. + def test_string(self) -> None: + assert _encode_sql_value("hello") == "'hello'" + + # [DEF:test_integer:Function] + # @PURPOSE: Integers are not quoted. + def test_integer(self) -> None: + assert _encode_sql_value(42) == "42" + + # [DEF:test_float:Function] + # @PURPOSE: Floats are not quoted. + def test_float(self) -> None: + assert _encode_sql_value(3.14) == "3.14" + + # [DEF:test_boolean:Function] + # @PURPOSE: Booleans encode as TRUE/FALSE. + def test_boolean(self) -> None: + assert _encode_sql_value(True) == "TRUE" + assert _encode_sql_value(False) == "FALSE" + + # [DEF:test_sql_injection:Function] + # @PURPOSE: Single quotes in strings are escaped. + def test_sql_injection(self) -> None: + assert _encode_sql_value("it's a test") == "'it''s a test'" + + # [DEF:test_special_chars:Function] + # @PURPOSE: Special characters in strings. + def test_special_chars(self) -> None: + assert _encode_sql_value("line1\nline2") == "'line1\nline2'" +# [/DEF:TestEncodeSqlValue:Class] + + +# [DEF:TestGenerateInsertSql:Class] +# @PURPOSE: Tests for plain INSERT SQL generation. +class TestGenerateInsertSql: + # [DEF:test_basic_insert:Function] + # @PURPOSE: Generates basic INSERT with VALUES. + def test_basic_insert(self, sample_rows: List[Dict[str, Any]]) -> None: + sql = generate_insert_sql( + target_schema=None, + target_table="my_table", + columns=["col1", "col2", "col3"], + rows=sample_rows, + ) + assert sql.startswith("INSERT INTO my_table") + assert "VALUES" in sql + assert "'hello'" in sql + assert "42" in sql + assert "NULL" in sql + assert sql.endswith(";") + + # [DEF:test_insert_with_schema:Function] + # @PURPOSE: Schema-qualified table name. + def test_insert_with_schema(self, sample_rows: List[Dict[str, Any]]) -> None: + sql = generate_insert_sql( + target_schema="public", + target_table="my_table", + columns=["col1", "col2", "col3"], + rows=sample_rows, + ) + assert "public" in sql + assert "my_table" in sql + + # [DEF:test_empty_columns_raises:Function] + # @PURPOSE: Empty columns list raises ValueError. + def test_empty_columns_raises(self, sample_rows: List[Dict[str, Any]]) -> None: + with pytest.raises(ValueError, match="At least one column"): + generate_insert_sql( + target_schema=None, + target_table="my_table", + columns=[], + rows=sample_rows, + ) + + # [DEF:test_empty_rows_raises:Function] + # @PURPOSE: Empty rows list raises ValueError. + def test_empty_rows_raises(self) -> None: + with pytest.raises(ValueError, match="At least one row"): + generate_insert_sql( + target_schema=None, + target_table="my_table", + columns=["col1"], + rows=[], + ) + + # [DEF:test_null_handling:Function] + # @PURPOSE: NULL values are correctly represented. + def test_null_handling(self) -> None: + rows = [{"col1": None, "col2": "text"}] + sql = generate_insert_sql( + target_schema=None, + target_table="t", + columns=["col1", "col2"], + rows=rows, + ) + assert "NULL" in sql + assert "'text'" in sql + + # [DEF:test_injection_safety:Function] + # @PURPOSE: SQL injection via string values is prevented. + def test_injection_safety(self) -> None: + rows = [{"col1": "'; DROP TABLE users; --"}] + sql = generate_insert_sql( + target_schema=None, + target_table="t", + columns=["col1"], + rows=rows, + ) + # The single quote should be doubled, preventing injection + assert "'';" in sql or "''" in sql + assert "DROP TABLE" in sql # It's literal data, not a command +# [/DEF:TestGenerateInsertSql:Class] + + +# [DEF:TestGenerateUpsertSql:Class] +# @PURPOSE: Tests for PostgreSQL UPSERT SQL generation. +class TestGenerateUpsertSql: + # [DEF:test_basic_upsert:Function] + # @PURPOSE: Generates INSERT ... ON CONFLICT DO UPDATE. + def test_basic_upsert(self, sample_rows: List[Dict[str, Any]]) -> None: + sql = generate_upsert_sql( + target_schema=None, + target_table="my_table", + columns=["col1", "col2", "col3"], + key_columns=["col1"], + rows=sample_rows, + ) + assert "INSERT INTO" in sql + assert "ON CONFLICT" in sql + assert "DO UPDATE SET" in sql + assert "col2 = EXCLUDED.col2" in sql + + # [DEF:test_upsert_all_keys:Function] + # @PURPOSE: When all columns are keys, use DO NOTHING. + def test_upsert_all_keys(self) -> None: + rows = [{"id": 1, "name": "test"}] + sql = generate_upsert_sql( + target_schema=None, + target_table="t", + columns=["id", "name"], + key_columns=["id", "name"], + rows=rows, + ) + assert "DO NOTHING" in sql + assert "DO UPDATE" not in sql + + # [DEF:test_upsert_empty_keys:Function] + # @PURPOSE: Empty key_columns raises ValueError. + def test_upsert_empty_keys(self, sample_rows: List[Dict[str, Any]]) -> None: + with pytest.raises(ValueError, match="key_columns"): + generate_upsert_sql( + target_schema=None, + target_table="t", + columns=["col1"], + key_columns=[], + rows=sample_rows, + ) +# [/DEF:TestGenerateUpsertSql:Class] + + +# [DEF:TestSQLGenerator:Class] +# @PURPOSE: Tests for the full SQLGenerator class. +class TestSQLGenerator: + # [DEF:test_postgresql_insert:Function] + # @PURPOSE: PostgreSQL dialect generates proper INSERT. + def test_postgresql_insert(self, sample_rows: List[Dict[str, Any]]) -> None: + sql, count = SQLGenerator.generate( + dialect="postgresql", + target_schema="public", + target_table="my_table", + columns=["col1", "col2"], + rows=sample_rows, + upsert_strategy="INSERT", + ) + assert count == 2 + assert "INSERT INTO" in sql + assert '"my_table"' in sql # Quoted identifier + + # [DEF:test_postgresql_upsert:Function] + # @PURPOSE: PostgreSQL dialect generates proper UPSERT. + def test_postgresql_upsert(self, sample_rows: List[Dict[str, Any]]) -> None: + sql, count = SQLGenerator.generate( + dialect="postgresql", + target_schema="public", + target_table="my_table", + columns=["col1", "col2", "col3"], + rows=sample_rows, + key_columns=["col1"], + upsert_strategy="MERGE", + ) + assert count == 2 + assert "ON CONFLICT" in sql + + # [DEF:test_clickhouse_insert:Function] + # @PURPOSE: ClickHouse dialect generates plain INSERT. + def test_clickhouse_insert(self, sample_rows: List[Dict[str, Any]]) -> None: + sql, count = SQLGenerator.generate( + dialect="clickhouse", + target_schema="default", + target_table="my_table", + columns=["col1", "col2"], + rows=sample_rows, + upsert_strategy="INSERT", + ) + assert count == 2 + assert "INSERT INTO" in sql + # ClickHouse uses backticks + assert "`my_table`" in sql or "my_table" in sql + + # [DEF:test_clickhouse_upsert_fallback:Function] + # @PURPOSE: ClickHouse with MERGE strategy falls back to plain INSERT with warning. + def test_clickhouse_upsert_fallback(self, sample_rows: List[Dict[str, Any]]) -> None: + sql, count = SQLGenerator.generate( + dialect="clickhouse", + target_schema="default", + target_table="my_table", + columns=["col1", "col2"], + rows=sample_rows, + key_columns=["col1"], + upsert_strategy="MERGE", + ) + assert count == 2 + # ClickHouse doesn't support ON CONFLICT; generates plain INSERT + assert "INSERT INTO" in sql + + # [DEF:test_empty_rows_via_generator:Function] + # @PURPOSE: Generator raises on empty rows. + def test_empty_rows_via_generator(self) -> None: + with pytest.raises(ValueError, match="At least one row"): + SQLGenerator.generate( + dialect="postgresql", + target_schema=None, + target_table="t", + columns=["c"], + rows=[], + ) + + # [DEF:test_generate_batch:Function] + # @PURPOSE: generate_batch splits large row sets. + def test_generate_batch(self, sample_rows: List[Dict[str, Any]]) -> None: + # Create many rows to test batching + many_rows = sample_rows * 5 # 10 rows + statements = SQLGenerator.generate_batch( + dialect="postgresql", + target_schema="public", + target_table="t", + columns=["col1", "col2"], + rows=many_rows, + max_rows_per_statement=3, + ) + assert len(statements) > 1 # Should be split into multiple statements + total_count = sum(count for _, count in statements) + assert total_count == 10 +# [/DEF:TestSQLGenerator:Class] +# [/DEF:SQLGeneratorTests:Module] diff --git a/backend/src/plugins/translate/_utils.py b/backend/src/plugins/translate/_utils.py new file mode 100644 index 00000000..728d2e9a --- /dev/null +++ b/backend/src/plugins/translate/_utils.py @@ -0,0 +1,35 @@ +# [DEF:DictionaryUtils:Module] +# @COMPLEXITY: 1 +# @SEMANTICS: dictionary, utils, normalize, delimiter +# @PURPOSE: Utility helpers for dictionary management (term normalization and delimiter detection). +# @LAYER: Domain + +import re +import unicodedata + + +# [DEF:_normalize_term:Function] +# @PURPOSE: Normalize a term for case-insensitive unique constraint lookup. +# @RATIONALE: NFC normalization is applied before lowercasing to ensure consistent +# comparison of Unicode characters (e.g. precomposed vs decomposed forms). +# @REJECTED: Lowercasing without NFC normalization — would cause duplicate entries +# for semantically identical Unicode strings in different normalization forms. +def _normalize_term(term: str) -> str: + """Normalize a term by NFC, lowercasing, and removing extra whitespace.""" + if not term: + return "" + term = unicodedata.normalize('NFC', term) + return " ".join(term.lower().split()) +# [/DEF:_normalize_term:Function] + + +# [DEF:_detect_delimiter:Function] +# @PURPOSE: Detect the delimiter used in a CSV/TSV header line. +def _detect_delimiter(header_line: str) -> str: + """Detect delimiter by counting tabs vs commas in the first line.""" + if not header_line: + return "," + tab_count = header_line.count("\t") + comma_count = header_line.count(",") + return "\t" if tab_count > comma_count else "," +# [/DEF:_detect_delimiter:Function] diff --git a/backend/src/plugins/translate/dictionary.py b/backend/src/plugins/translate/dictionary.py new file mode 100644 index 00000000..fa2eb8f8 --- /dev/null +++ b/backend/src/plugins/translate/dictionary.py @@ -0,0 +1,781 @@ +# [DEF:DictionaryManagerModule:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: dictionary, manager, terminology, crud, import, filter +# @PURPOSE: Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering. +# @LAYER: Domain +# @RELATION: DEPENDS_ON -> [TerminologyDictionary:Class] +# @RELATION: DEPENDS_ON -> [DictionaryEntry:Class] +# @RELATION: DEPENDS_ON -> [TranslationJobDictionary:Class] +# @RELATION: DEPENDS_ON -> [TranslationJob:Class] +# @RATIONALE: C4 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term. +# @REJECTED: Pure C3 CRUD without state guards would allow orphaned job-dictionary links. +# @REJECTED: "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing. + +import csv +import io +import re +from typing import Any, Dict, List, Optional, Tuple +from sqlalchemy.orm import Session +from sqlalchemy import func +from ...core.logger import logger, belief_scope +from ...models.translate import ( + TerminologyDictionary, + DictionaryEntry, + TranslationJobDictionary, + TranslationJob, +) +from ._utils import _normalize_term, _detect_delimiter + + +# [DEF:DictionaryManager:Class] +# @COMPLEXITY: 4 +# @PURPOSE: Manages terminology dictionaries and their entries with referential integrity. +# @PRE: Database session is open and valid. +# @POST: Dictionary and entry mutations are persisted with conflict detection. +# @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards. +class DictionaryManager: + # [DEF:DictionaryManager.create_dictionary:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Create a new terminology dictionary. + # @PRE: payload contains name, source_dialect, target_dialect. + # @POST: New TerminologyDictionary row is created and returned. + @staticmethod + def create_dictionary( + db: Session, name: str, source_dialect: str, target_dialect: str, + created_by: Optional[str] = None, description: Optional[str] = None, + is_active: bool = True, + ) -> TerminologyDictionary: + with belief_scope("DictionaryManager.create_dictionary"): + logger.reason("Creating dictionary", {"name": name, "source": source_dialect, "target": target_dialect}) + dictionary = TerminologyDictionary( + name=name, + description=description, + source_dialect=source_dialect, + target_dialect=target_dialect, + is_active=is_active, + created_by=created_by, + ) + db.add(dictionary) + db.commit() + db.refresh(dictionary) + logger.reflect("Dictionary created", {"id": dictionary.id}) + return dictionary + # [/DEF:DictionaryManager.create_dictionary:Function] + + # [DEF:DictionaryManager.update_dictionary:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Update an existing terminology dictionary. + # @PRE: dict_id exists in terminology_dictionaries table. + # @POST: Dictionary metadata is updated and returned. + @staticmethod + def update_dictionary( + db: Session, dict_id: str, name: Optional[str] = None, + description: Optional[str] = None, source_dialect: Optional[str] = None, + target_dialect: Optional[str] = None, is_active: Optional[bool] = None, + ) -> TerminologyDictionary: + with belief_scope("DictionaryManager.update_dictionary"): + dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() + if not dictionary: + raise ValueError(f"Dictionary not found: {dict_id}") + logger.reason("Updating dictionary", {"id": dict_id}) + if name is not None: + dictionary.name = name + if description is not None: + dictionary.description = description + if source_dialect is not None: + dictionary.source_dialect = source_dialect + if target_dialect is not None: + dictionary.target_dialect = target_dialect + if is_active is not None: + dictionary.is_active = is_active + db.commit() + db.refresh(dictionary) + logger.reflect("Dictionary updated", {"id": dictionary.id}) + return dictionary + # [/DEF:DictionaryManager.update_dictionary:Function] + + # [DEF:DictionaryManager.delete_dictionary:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Delete a dictionary, blocked if attached to active/scheduled jobs. + # @PRE: dict_id exists. + # @POST: Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs. + # @SIDE_EFFECT: Deletes TerminologyDictionary row and all DictionaryEntry rows. + @staticmethod + def delete_dictionary(db: Session, dict_id: str) -> None: + with belief_scope("DictionaryManager.delete_dictionary"): + dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() + if not dictionary: + raise ValueError(f"Dictionary not found: {dict_id}") + + # Check for attached active/scheduled jobs + blocked_statuses = ["ACTIVE", "READY", "RUNNING", "SCHEDULED"] + attached_jobs = ( + db.query(TranslationJobDictionary) + .filter( + TranslationJobDictionary.dictionary_id == dict_id, + TranslationJobDictionary.job_id.in_( + db.query(TranslationJob.id).filter( + TranslationJob.status.in_(blocked_statuses) + ) + ), + ) + .count() + ) + + if attached_jobs > 0: + logger.explore("Delete blocked: dictionary attached to active jobs", {"dict_id": dict_id, "jobs": attached_jobs}) + raise ValueError( + f"Cannot delete dictionary attached to {attached_jobs} active/scheduled job(s). " + "Remove dictionary associations from jobs first." + ) + + logger.reason("Deleting dictionary", {"id": dict_id}) + # Delete entries and job-dictionary links first + db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete() + db.query(TranslationJobDictionary).filter(TranslationJobDictionary.dictionary_id == dict_id).delete() + db.delete(dictionary) + db.commit() + logger.reflect("Dictionary deleted", {"id": dict_id}) + # [/DEF:DictionaryManager.delete_dictionary:Function] + + # [DEF:DictionaryManager.get_dictionary:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Get a single dictionary by ID with entry count. + # @PRE: dict_id exists. + # @POST: Returns dict with dictionary + entry_count or raises ValueError. + @staticmethod + def get_dictionary(db: Session, dict_id: str) -> TerminologyDictionary: + dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() + if not dictionary: + raise ValueError(f"Dictionary not found: {dict_id}") + return dictionary + # [/DEF:DictionaryManager.get_dictionary:Function] + + # [DEF:DictionaryManager.list_dictionaries:Function] + # @COMPLEXITY: 2 + # @PURPOSE: List dictionaries with pagination and entry counts. + # @PRE: page >= 1, page_size between 1 and 100. + # @POST: Returns (list of dicts, total_count). + @staticmethod + def list_dictionaries( + db: Session, page: int = 1, page_size: int = 20, + ) -> Tuple[List[TerminologyDictionary], int]: + total = db.query(func.count(TerminologyDictionary.id)).scalar() or 0 + dictionaries = ( + db.query(TerminologyDictionary) + .order_by(TerminologyDictionary.created_at.desc()) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + return dictionaries, total + # [/DEF:DictionaryManager.list_dictionaries:Function] + + # [DEF:DictionaryManager.add_entry:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Add an entry to a dictionary, enforcing unique source_term_normalized. + # @PRE: dict_id exists. source_term and target_term are non-empty. + # @POST: New DictionaryEntry row is created or raises on duplicate. + @staticmethod + def add_entry( + db: Session, dict_id: str, source_term: str, target_term: str, + context_notes: Optional[str] = None, + ) -> DictionaryEntry: + with belief_scope("DictionaryManager.add_entry"): + normalized = _normalize_term(source_term) + existing = ( + db.query(DictionaryEntry) + .filter( + DictionaryEntry.dictionary_id == dict_id, + DictionaryEntry.source_term_normalized == normalized, + ) + .first() + ) + if existing: + raise ValueError( + f"Duplicate entry: '{source_term}' already exists in this dictionary " + f"(id={existing.id}). Use overwrite or keep_existing conflict mode." + ) + + logger.reason("Adding dictionary entry", {"dict_id": dict_id, "term": source_term}) + entry = DictionaryEntry( + dictionary_id=dict_id, + source_term=source_term.strip(), + source_term_normalized=normalized, + target_term=target_term.strip(), + context_notes=context_notes.strip() if context_notes else None, + ) + db.add(entry) + db.commit() + db.refresh(entry) + logger.reflect("Entry added", {"entry_id": entry.id}) + return entry + # [/DEF:DictionaryManager.add_entry:Function] + + # [DEF:DictionaryManager.edit_entry:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Edit an existing dictionary entry with duplicate-aware normalization. + # @PRE: entry_id exists. + # @POST: Entry fields are updated. + @staticmethod + def edit_entry( + db: Session, entry_id: str, source_term: Optional[str] = None, + target_term: Optional[str] = None, context_notes: Optional[str] = None, + ) -> DictionaryEntry: + with belief_scope("DictionaryManager.edit_entry"): + entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first() + if not entry: + raise ValueError(f"Entry not found: {entry_id}") + + logger.reason("Editing dictionary entry", {"entry_id": entry_id}) + if source_term is not None: + normalized = _normalize_term(source_term) + # Check uniqueness within the same dictionary + existing = ( + db.query(DictionaryEntry) + .filter( + DictionaryEntry.dictionary_id == entry.dictionary_id, + DictionaryEntry.source_term_normalized == normalized, + DictionaryEntry.id != entry_id, + ) + .first() + ) + if existing: + raise ValueError( + f"Duplicate entry: '{source_term}' already exists in this dictionary " + f"(id={existing.id})." + ) + entry.source_term = source_term.strip() + entry.source_term_normalized = normalized + if target_term is not None: + entry.target_term = target_term.strip() + if context_notes is not None: + entry.context_notes = context_notes.strip() if context_notes else None + db.commit() + db.refresh(entry) + logger.reflect("Entry updated", {"entry_id": entry.id}) + return entry + # [/DEF:DictionaryManager.edit_entry:Function] + + # [DEF:DictionaryManager.delete_entry:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Delete a single dictionary entry. + # @PRE: entry_id exists. + # @POST: Entry is deleted. + @staticmethod + def delete_entry(db: Session, entry_id: str) -> None: + with belief_scope("DictionaryManager.delete_entry"): + entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first() + if not entry: + raise ValueError(f"Entry not found: {entry_id}") + logger.reason("Deleting dictionary entry", {"entry_id": entry_id}) + db.delete(entry) + db.commit() + logger.reflect("Entry deleted", {"entry_id": entry_id}) + # [/DEF:DictionaryManager.delete_entry:Function] + + # [DEF:DictionaryManager.clear_entries:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Delete all entries for a dictionary. + # @PRE: dict_id exists. + # @POST: All entries for the dictionary are deleted. + @staticmethod + def clear_entries(db: Session, dict_id: str) -> int: + with belief_scope("DictionaryManager.clear_entries"): + dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() + if not dictionary: + raise ValueError(f"Dictionary not found: {dict_id}") + logger.reason("Clearing all entries", {"dict_id": dict_id}) + deleted = db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete() + db.commit() + logger.reflect("Entries cleared", {"dict_id": dict_id, "count": deleted}) + return deleted + # [/DEF:DictionaryManager.clear_entries:Function] + + # [DEF:DictionaryManager.list_entries:Function] + # @COMPLEXITY: 2 + # @PURPOSE: List entries for a dictionary with pagination. + # @PRE: dict_id exists. + # @POST: Returns (list of entries, total_count). + @staticmethod + def list_entries( + db: Session, dict_id: str, page: int = 1, page_size: int = 50, + ) -> Tuple[List[DictionaryEntry], int]: + total = ( + db.query(func.count(DictionaryEntry.id)) + .filter(DictionaryEntry.dictionary_id == dict_id) + .scalar() + or 0 + ) + entries = ( + db.query(DictionaryEntry) + .filter(DictionaryEntry.dictionary_id == dict_id) + .order_by(DictionaryEntry.source_term.asc()) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + return entries, total + # [/DEF:DictionaryManager.list_entries:Function] + + # [DEF:DictionaryManager.import_entries:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Import entries from CSV/TSV content with duplicate detection and conflict resolution. + # @PRE: content is valid CSV or TSV. dict_id exists. + # @POST: Entries are created/updated/skipped per conflict mode. Returns result summary. + # @SIDE_EFFECT: Batch-inserts or updates DictionaryEntry rows. + @staticmethod + def import_entries( + db: Session, dict_id: str, content: str, + delimiter: Optional[str] = None, + on_conflict: str = "overwrite", + preview_only: bool = False, + ) -> Dict[str, Any]: + with belief_scope("DictionaryManager.import_entries"): + # Validate dictionary + dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() + if not dictionary: + raise ValueError(f"Dictionary not found: {dict_id}") + + # Detect delimiter if not specified + if not delimiter: + delimiter = _detect_delimiter(content) + logger.reason("Detected delimiter", {"delimiter": repr(delimiter)}) + + if delimiter not in (",", "\t"): + raise ValueError(f"Unsupported delimiter: {repr(delimiter)}. Use ',' or '\\t'.") + + # Parse content + reader = csv.DictReader(io.StringIO(content), delimiter=delimiter) + required_fields = {"source_term", "target_term"} + if not reader.fieldnames or not required_fields.issubset(reader.fieldnames): + raise ValueError( + f"CSV/TSV must have at least 'source_term' and 'target_term' columns. " + f"Got: {reader.fieldnames}" + ) + + result: Dict[str, Any] = { + "total": 0, + "created": 0, + "updated": 0, + "skipped": 0, + "errors": [], + "preview": [], + } + + rows = list(reader) + result["total"] = len(rows) + + for row_idx, row in enumerate(rows): + try: + source_term = row.get("source_term", "").strip() + target_term = row.get("target_term", "").strip() + context_notes = row.get("context_notes", "").strip() or None + + if not source_term or not target_term: + result["errors"].append({ + "line": row_idx + 2, # +2 for header and 1-indexed + "error": "Both source_term and target_term are required", + "row": row, + }) + continue + + normalized = _normalize_term(source_term) + + if preview_only: + existing = ( + db.query(DictionaryEntry) + .filter( + DictionaryEntry.dictionary_id == dict_id, + DictionaryEntry.source_term_normalized == normalized, + ) + .first() + ) + preview_row = { + "line": row_idx + 2, + "source_term": source_term, + "target_term": target_term, + "context_notes": context_notes, + "is_conflict": existing is not None, + "existing_target_term": existing.target_term if existing else None, + } + result["preview"].append(preview_row) + continue + + existing = ( + db.query(DictionaryEntry) + .filter( + DictionaryEntry.dictionary_id == dict_id, + DictionaryEntry.source_term_normalized == normalized, + ) + .first() + ) + + if existing: + if on_conflict == "overwrite": + existing.source_term = source_term + existing.target_term = target_term + existing.context_notes = context_notes + result["updated"] += 1 + elif on_conflict == "keep_existing": + result["skipped"] += 1 + else: # cancel + result["errors"].append({ + "line": row_idx + 2, + "error": f"Conflict on '{source_term}' and on_conflict='cancel'", + "row": row, + }) + continue + else: + entry = DictionaryEntry( + dictionary_id=dict_id, + source_term=source_term, + source_term_normalized=normalized, + target_term=target_term, + context_notes=context_notes, + ) + db.add(entry) + result["created"] += 1 + + except Exception as e: + result["errors"].append({ + "line": row_idx + 2, + "error": str(e), + "row": row, + }) + + if not preview_only: + db.commit() + + logger.reflect("Import complete", { + "dict_id": dict_id, + "total": result["total"], + "created": result["created"], + "updated": result["updated"], + "skipped": result["skipped"], + "errors": len(result["errors"]), + }) + return result + # [/DEF:DictionaryManager.import_entries:Function] + + # [DEF:DictionaryManager.filter_for_batch:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job. + # @PRE: job_id exists and source_texts is a list of strings. + # @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries. + # @SIDE_EFFECT: Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables. + @staticmethod + def filter_for_batch( + db: Session, source_texts: List[str], job_id: str, + ) -> List[Dict[str, Any]]: + with belief_scope("DictionaryManager.filter_for_batch"): + # Get dictionaries attached to this job + job_dict_links = ( + db.query(TranslationJobDictionary) + .filter(TranslationJobDictionary.job_id == job_id) + .all() + ) + if not job_dict_links: + logger.reason("No dictionaries attached to job", {"job_id": job_id}) + return [] + + dict_ids = [jd.dictionary_id for jd in job_dict_links] + + # Get all active dictionaries + dictionaries = ( + db.query(TerminologyDictionary) + .filter( + TerminologyDictionary.id.in_(dict_ids), + TerminologyDictionary.is_active == True, + ) + .all() + ) + if not dictionaries: + return [] + + # Build dict_id -> dict_name lookup + dict_map = {d.id: d.name for d in dictionaries} + + # Fetch all entries for these dictionaries, ordered by dictionary (priority order from link order) + all_entries = ( + db.query(DictionaryEntry) + .filter(DictionaryEntry.dictionary_id.in_(dict_ids)) + .order_by(DictionaryEntry.source_term.asc()) + .all() + ) + + if not all_entries: + return [] + + matched: List[Dict[str, Any]] = [] + seen_source_match: set = set() + + for text_idx, source_text in enumerate(source_texts): + if not source_text: + continue + + text_lower = source_text.lower() + + for entry in all_entries: + norm = entry.source_term_normalized + if not norm: + continue + + # Word-boundary-aware matching + # Build pattern: \bterm\b (case-insensitive) + # Escape regex special chars in the search term + escaped = re.escape(norm) + pattern = re.compile(r"\b" + escaped + r"\b", re.IGNORECASE) + + if pattern.search(text_lower): + match_key = (text_idx, entry.id) + if match_key not in seen_source_match: + seen_source_match.add(match_key) + matched.append({ + "text_index": text_idx, + "source_text": source_text, + "entry_id": entry.id, + "source_term": entry.source_term, + "source_term_normalized": entry.source_term_normalized, + "target_term": entry.target_term, + "dictionary_id": entry.dictionary_id, + "dictionary_name": dict_map.get(entry.dictionary_id, ""), + "context_notes": entry.context_notes, + }) + + # Sort by dictionary link priority (order of dict_ids from link order) + dict_priority = {d_id: idx for idx, d_id in enumerate(dict_ids)} + matched.sort(key=lambda m: (dict_priority.get(m["dictionary_id"], 999), m["text_index"])) + + logger.reflect("Batch filter match complete", { + "job_id": job_id, + "source_texts": len(source_texts), + "matches": len(matched), + "dictionaries_used": len(dictionaries), + }) + return matched + # [/DEF:DictionaryManager.filter_for_batch:Function] + + + # [DEF:DictionaryManager.submit_correction:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Submit a term correction from a run result. Validates language match, detects conflicts. + # @PRE: source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists. + # @POST: Entry created or updated; origin tracking populated. Returns action + conflict info. + # @SIDE_EFFECT: Creates/updates DictionaryEntry row. + @staticmethod + def submit_correction( + db: Session, + dict_id: str, + source_term: str, + incorrect_target_term: str, + corrected_target_term: str, + origin_run_id: Optional[str] = None, + origin_row_key: Optional[str] = None, + origin_user_id: Optional[str] = None, + on_conflict: str = "overwrite", + ) -> Dict[str, Any]: + with belief_scope("DictionaryManager.submit_correction"): + # Validate dictionary exists + dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() + if not dictionary: + raise ValueError(f"Dictionary '{dict_id}' not found") + + normalized = _normalize_term(source_term) + existing = ( + db.query(DictionaryEntry) + .filter( + DictionaryEntry.dictionary_id == dict_id, + DictionaryEntry.source_term_normalized == normalized, + ) + .first() + ) + + result: Dict[str, Any] = { + "source_term": source_term, + "target_term": corrected_target_term, + "action": "created", + "entry_id": None, + "conflict": None, + "message": None, + } + + if existing: + # Conflict detected + if on_conflict == "keep_existing": + result["action"] = "conflict_detected" + result["conflict"] = { + "source_term": source_term, + "existing_target_term": existing.target_term, + "submitted_target_term": corrected_target_term, + "action": "keep_existing", + } + result["message"] = f"Existing entry kept: '{existing.target_term}'" + return result + elif on_conflict == "overwrite": + existing.target_term = corrected_target_term.strip() + if origin_run_id: + existing.origin_run_id = origin_run_id + if origin_row_key: + existing.origin_row_key = origin_row_key + if origin_user_id: + existing.origin_user_id = origin_user_id + db.flush() + result["action"] = "updated" + result["entry_id"] = existing.id + result["message"] = f"Entry updated from '{existing.target_term}' to '{corrected_target_term}'" + else: # cancel + result["action"] = "skipped" + result["conflict"] = { + "source_term": source_term, + "existing_target_term": existing.target_term, + "submitted_target_term": corrected_target_term, + "action": "cancel", + } + result["message"] = "Correction cancelled by conflict mode" + return result + else: + # Create new entry + entry = DictionaryEntry( + dictionary_id=dict_id, + source_term=source_term.strip(), + source_term_normalized=normalized, + target_term=corrected_target_term.strip(), + origin_run_id=origin_run_id, + origin_row_key=origin_row_key, + origin_user_id=origin_user_id, + ) + db.add(entry) + db.flush() + result["entry_id"] = entry.id + result["message"] = f"Entry created for '{source_term}' -> '{corrected_target_term}'" + + db.commit() + logger.reflect("Correction processed", result) + return result + # [/DEF:DictionaryManager.submit_correction:Function] + + # [DEF:DictionaryManager.submit_bulk_corrections:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Submit multiple term corrections atomically — all succeed or all fail. + # @PRE: corrections list is non-empty. dict_id exists. + # @POST: All corrections applied or none applied with conflict list. + # @SIDE_EFFECT: Creates/updates DictionaryEntry rows; commits once. + @staticmethod + def submit_bulk_corrections( + db: Session, + dict_id: str, + corrections: List[Dict[str, Any]], + origin_user_id: Optional[str] = None, + ) -> Dict[str, Any]: + with belief_scope("DictionaryManager.submit_bulk_corrections"): + # Validate dictionary first + dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() + if not dictionary: + raise ValueError(f"Dictionary '{dict_id}' not found") + + results: List[Dict[str, Any]] = [] + conflicts: List[Dict[str, Any]] = [] + all_ok = True + + for corr in corrections: + source_term = corr.get("source_term", "").strip() + incorrect_target = corr.get("incorrect_target_term", "").strip() + corrected_target = corr.get("corrected_target_term", "").strip() + + if not source_term or not corrected_target: + results.append({ + "source_term": source_term, + "action": "error", + "message": "source_term and corrected_target_term are required", + }) + all_ok = False + continue + + normalized = _normalize_term(source_term) + existing = ( + db.query(DictionaryEntry) + .filter( + DictionaryEntry.dictionary_id == dict_id, + DictionaryEntry.source_term_normalized == normalized, + ) + .first() + ) + + if existing: + on_conflict = corr.get("on_conflict", "overwrite") + if on_conflict == "keep_existing": + conflicts.append({ + "source_term": source_term, + "existing_target_term": existing.target_term, + "submitted_target_term": corrected_target, + "action": "keep_existing", + }) + results.append({ + "source_term": source_term, + "action": "conflict_detected", + "message": f"Existing entry kept: '{existing.target_term}'", + }) + continue + elif on_conflict == "overwrite": + existing.target_term = corrected_target + existing.origin_user_id = origin_user_id + results.append({ + "source_term": source_term, + "action": "updated", + "entry_id": existing.id, + "message": f"Updated to '{corrected_target}'", + }) + else: + conflicts.append({ + "source_term": source_term, + "existing_target_term": existing.target_term, + "submitted_target_term": corrected_target, + "action": "cancel", + }) + results.append({ + "source_term": source_term, + "action": "skipped", + "message": "Cancelled by conflict mode", + }) + else: + entry = DictionaryEntry( + dictionary_id=dict_id, + source_term=source_term, + source_term_normalized=normalized, + target_term=corrected_target, + origin_run_id=corr.get("origin_run_id"), + origin_row_key=corr.get("origin_row_key"), + origin_user_id=origin_user_id, + ) + db.add(entry) + results.append({ + "source_term": source_term, + "action": "created", + "message": f"Entry created for '{source_term}' -> '{corrected_target}'", + }) + + if conflicts and not all_ok: + # Rollback — all failed + db.rollback() + return { + "status": "conflicts", + "results": results, + "conflicts": conflicts, + "message": "Conflicts detected. Resolve before retrying.", + } + + db.commit() + + return { + "status": "completed", + "results": results, + "conflicts": conflicts, + "total": len(corrections), + "applied": sum(1 for r in results if r["action"] in ("created", "updated")), + } + # [/DEF:DictionaryManager.submit_bulk_corrections:Function] + + +# [/DEF:DictionaryManager:Class] +# [/DEF:DictionaryManagerModule:Module] diff --git a/backend/src/plugins/translate/events.py b/backend/src/plugins/translate/events.py new file mode 100644 index 00000000..098197bc --- /dev/null +++ b/backend/src/plugins/translate/events.py @@ -0,0 +1,262 @@ +# [DEF:TranslationEventLog:Module] +# @COMPLEXITY: 5 +# @SEMANTICS: translate, events, audit, logging +# @PURPOSE: Structured event logging for translation operations with terminal event invariant enforcement. +# @LAYER: Domain +# @RELATION: DEPENDS_ON -> [TranslationEvent] +# @RELATION: DEPENDS_ON -> [MetricSnapshot] +# @PRE: Database session is open and valid. +# @POST: Events are persisted immutably; terminal events enforce exactly-one invariant per run. +# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion. +# @DATA_CONTRACT: Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent] +# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id. +# @RATIONALE: Immutable event log with nullable run_id allows job-level events (no run context) and run-level events. +# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant. +# @REJECTED: Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit. +from typing import Any, Dict, List, Optional +from sqlalchemy.orm import Session +from datetime import datetime, timezone, timedelta +import uuid + +from ...core.logger import logger, belief_scope +from ...models.translate import TranslationEvent, MetricSnapshot + +# Terminal events per run — exactly one of these must exist for a completed run. +TERMINAL_EVENT_TYPES = {"RUN_COMPLETED", "RUN_FAILED", "RUN_CANCELLED"} +# All valid event types for validation. +VALID_EVENT_TYPES = TERMINAL_EVENT_TYPES | { + "JOB_CREATED", "JOB_UPDATED", "JOB_DELETED", + "RUN_STARTED", "RUN_RETRYING", "RUN_RETRY_INSERT", + "BATCH_STARTED", "BATCH_COMPLETED", "BATCH_FAILED", "BATCH_RETRYING", + "TRANSLATION_PHASE_STARTED", "TRANSLATION_PHASE_COMPLETED", + "INSERT_PHASE_STARTED", "INSERT_PHASE_COMPLETED", + "PREVIEW_CREATED", "PREVIEW_ACCEPTED", "PREVIEW_DISCARDED", + "SCHEDULE_CREATED", "SCHEDULE_UPDATED", "SCHEDULE_DELETED", + "METRICS_SNAPSHOT_CREATED", +} + +DEFAULT_RETENTION_DAYS = 90 + + +# [DEF:TranslationEventLog:Class] +# @COMPLEXITY: 5 +# @PURPOSE: Structured event logging for translation operations with terminal event invariant enforcement. +# @PRE: Database session is available. +# @POST: Events are written immutably; terminal events enforced per run. +# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events. +# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id. +class TranslationEventLog: + + def __init__(self, db: Session): + self.db = db + + # [DEF:log_event:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Write an immutable event. Enforces terminal event invariant for non-null run_id. + # @PRE: event_type must be a known type. If run_id is not None, enforce terminal invariant. + # @POST: TranslationEvent row is created. + # @SIDE_EFFECT: DB write. + def log_event( + self, + job_id: str, + event_type: str, + payload: Optional[Dict[str, Any]] = None, + run_id: Optional[str] = None, + created_by: Optional[str] = None, + ) -> TranslationEvent: + with belief_scope("TranslationEventLog.log_event"): + if event_type not in VALID_EVENT_TYPES: + raise ValueError( + f"Invalid event_type '{event_type}'. " + f"Must be one of: {', '.join(sorted(VALID_EVENT_TYPES))}" + ) + + # Enforce terminal event invariant for non-null run_id + if run_id is not None and event_type in TERMINAL_EVENT_TYPES: + existing_terminal = ( + self.db.query(TranslationEvent.id) + .filter( + TranslationEvent.run_id == run_id, + TranslationEvent.event_type.in_(TERMINAL_EVENT_TYPES), + ) + .first() + ) + if existing_terminal: + raise ValueError( + f"Run '{run_id}' already has a terminal event " + f"({existing_terminal[0]}). Cannot add another terminal event." + ) + + # Enforce run_started invariant — must exist before other run events + if run_id is not None and event_type not in {"RUN_STARTED"} | TERMINAL_EVENT_TYPES: + has_started = ( + self.db.query(TranslationEvent.id) + .filter( + TranslationEvent.run_id == run_id, + TranslationEvent.event_type == "RUN_STARTED", + ) + .first() + ) + if not has_started: + raise ValueError( + f"Cannot log '{event_type}' for run '{run_id}': " + f"RUN_STARTED event must precede other run events." + ) + + event = TranslationEvent( + id=str(uuid.uuid4()), + job_id=job_id, + run_id=run_id, + event_type=event_type, + event_data=payload or {}, + created_by=created_by, + created_at=datetime.now(timezone.utc), + ) + self.db.add(event) + self.db.flush() + + logger.reason(f"Event logged: {event_type}", { + "event_id": event.id, + "job_id": job_id, + "run_id": run_id, + "event_type": event_type, + }) + return event + # [/DEF:log_event:Function] + + # [DEF:query_events:Function] + # @PURPOSE: Query events with optional filters. + # @PRE: None. + # @POST: Returns list of TranslationEvent dicts matching filters. + def query_events( + self, + job_id: Optional[str] = None, + run_id: Optional[str] = None, + event_type: Optional[str] = None, + limit: int = 100, + offset: int = 0, + ) -> List[Dict[str, Any]]: + with belief_scope("TranslationEventLog.query_events"): + query = self.db.query(TranslationEvent) + + if job_id: + query = query.filter(TranslationEvent.job_id == job_id) + if run_id: + query = query.filter(TranslationEvent.run_id == run_id) + if event_type: + query = query.filter(TranslationEvent.event_type == event_type) + + events = ( + query.order_by(TranslationEvent.created_at.desc()) + .offset(offset) + .limit(limit) + .all() + ) + + return [ + { + "id": e.id, + "job_id": e.job_id, + "run_id": e.run_id, + "event_type": e.event_type, + "event_data": e.event_data or {}, + "created_by": e.created_by, + "created_at": e.created_at.isoformat() if e.created_at else None, + } + for e in events + ] + # [/DEF:query_events:Function] + + # [DEF:prune_expired:Function] + # @PURPOSE: Delete events older than retention_days. Persists MetricSnapshot before pruning. + # @PRE: None. + # @POST: Expired events are deleted; MetricSnapshot is created before deletion. + # @SIDE_EFFECT: Creates MetricSnapshot row; deletes TranslationEvent rows. + def prune_expired( + self, + retention_days: int = DEFAULT_RETENTION_DAYS, + batch_size: int = 1000, + ) -> Dict[str, Any]: + with belief_scope("TranslationEventLog.prune_expired"): + cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days) + logger.reason("Pruning expired events", { + "cutoff": cutoff.isoformat(), + "retention_days": retention_days, + }) + + # Find events to prune + expired_query = self.db.query(TranslationEvent).filter( + TranslationEvent.created_at < cutoff + ) + total_expired = expired_query.count() + + if total_expired == 0: + logger.reflect("No expired events to prune", {}) + return {"pruned": 0, "snapshot_id": None} + + # Create MetricSnapshot before pruning + snapshot = MetricSnapshot( + id=str(uuid.uuid4()), + job_id="_prune_aggregate_", + key_hash=f"prune_{cutoff.timestamp():.0f}", + covers_events_before=cutoff, + total_records=total_expired, + snapshot_date=datetime.now(timezone.utc), + ) + self.db.add(snapshot) + self.db.flush() + snapshot_id = snapshot.id + + # Delete in batches + pruned = 0 + while True: + batch_ids = ( + self.db.query(TranslationEvent.id) + .filter(TranslationEvent.created_at < cutoff) + .limit(batch_size) + .all() + ) + if not batch_ids: + break + ids = [row[0] for row in batch_ids] + self.db.query(TranslationEvent).filter( + TranslationEvent.id.in_(ids) + ).delete(synchronize_session=False) + self.db.flush() + pruned += len(ids) + logger.reason("Pruned batch", {"batch_size": len(ids), "total_pruned": pruned}) + + self.db.commit() + + logger.reflect("Pruning complete", { + "pruned": pruned, + "snapshot_id": snapshot_id, + }) + return {"pruned": pruned, "snapshot_id": snapshot_id} + # [/DEF:prune_expired:Function] + + # [DEF:get_run_event_summary:Function] + # @PURPOSE: Get a summary of events for a run, including invariant check. + # @PRE: run_id is not None. + # @POST: Returns dict with event list and invariant validity. + def get_run_event_summary(self, run_id: str) -> Dict[str, Any]: + with belief_scope("TranslationEventLog.get_run_event_summary"): + events = self.query_events(run_id=run_id) + + has_started = any(e["event_type"] == "RUN_STARTED" for e in events) + terminal_events = [e for e in events if e["event_type"] in TERMINAL_EVENT_TYPES] + + return { + "run_id": run_id, + "event_count": len(events), + "has_run_started": has_started, + "terminal_event_count": len(terminal_events), + "terminal_events": terminal_events, + "invariant_valid": has_started and len(terminal_events) <= 1, + "events": events, + } + # [/DEF:get_run_event_summary:Function] + + +# [/DEF:TranslationEventLog:Class] +# [/DEF:TranslationEventLog:Module] diff --git a/backend/src/plugins/translate/executor.py b/backend/src/plugins/translate/executor.py new file mode 100644 index 00000000..b5a23b70 --- /dev/null +++ b/backend/src/plugins/translate/executor.py @@ -0,0 +1,626 @@ +# [DEF:TranslationExecutor:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: translate, executor, batch, llm +# @PURPOSE: Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows. +# @LAYER: Domain +# @RELATION: DEPENDS_ON -> [TranslationBatch] +# @RELATION: DEPENDS_ON -> [TranslationRecord] +# @RELATION: DEPENDS_ON -> [TranslationRun] +# @RELATION: DEPENDS_ON -> [LLMProviderService] +# @RELATION: DEPENDS_ON -> [DictionaryManager] +# @RELATION: DEPENDS_ON -> [TranslationPreview] +# @PRE: Valid TranslationRun with job configuration. DB session is available. +# @POST: TranslationBatch and TranslationRecord rows are created. Run status is updated. +# @SIDE_EFFECT: Calls LLM provider; creates DB rows; updates run statistics. +# @RATIONALE: Batch processing with retry — independent batches allow partial recovery. +# @REJECTED: Single monolithic LLM call — would lose all progress on any failure. + +import json +import time +import uuid +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Set, Tuple, Callable + +from sqlalchemy.orm import Session + +from ...core.logger import logger, belief_scope +from ...core.config_manager import ConfigManager +from ...models.translate import ( + TranslationJob, + TranslationRun, + TranslationBatch, + TranslationRecord, + TranslationPreviewSession, + TranslationPreviewRecord, +) +from ...services.llm_provider import LLMProviderService +from ...services.llm_prompt_templates import render_prompt +from .dictionary import DictionaryManager +from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE + + +# [DEF:MAX_RETRIES_PER_BATCH:Constant] +# @PURPOSE: Maximum number of retries for a single batch before marking it failed. +MAX_RETRIES_PER_BATCH = 3 + + +# [DEF:TranslationExecutor:Class] +# @COMPLEXITY: 4 +# @PURPOSE: Process translation batches: fetch source rows, filter dict, call LLM, persist results. +# @PRE: DB session and config manager available. +# @POST: Batches and records created with status tracking. +# @SIDE_EFFECT: LLM API calls; DB writes. +class TranslationExecutor: + + def __init__( + self, + db: Session, + config_manager: ConfigManager, + current_user: Optional[str] = None, + on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None, + ): + self.db = db + self.config_manager = config_manager + self.current_user = current_user + self.on_batch_progress = on_batch_progress + self._current_run_id: Optional[str] = None + + # [DEF:execute_run:Function] + # @PURPOSE: Run full translation execution for a TranslationRun. + # @PRE: run is in PENDING or RUNNING status with valid job config. + # @POST: Run is populated with batches and records. + # @SIDE_EFFECT: LLM API calls; DB batch writes. + def execute_run( + self, + run: TranslationRun, + llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None, + ) -> TranslationRun: + with belief_scope("TranslationExecutor.execute_run"): + job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first() + if not job: + raise ValueError(f"Job '{run.job_id}' not found for run '{run.id}'") + + logger.reason("Starting translation execution", { + "run_id": run.id, + "job_id": job.id, + "batch_size": job.batch_size, + }) + + # Mark run as RUNNING + run.status = "RUNNING" + run.started_at = datetime.now(timezone.utc) + self.db.flush() + + # Fetch source rows from the accepted preview session + source_rows = self._fetch_source_rows(job.id, run.id) + if not source_rows: + logger.explore("No source rows to translate", {"run_id": run.id}) + run.status = "COMPLETED" + run.completed_at = datetime.now(timezone.utc) + self.db.flush() + return run + + total_rows = len(source_rows) + run.total_records = total_rows + + # Split into batches + batch_size = job.batch_size or 50 + batches = [ + source_rows[i:i + batch_size] + for i in range(0, total_rows, batch_size) + ] + + logger.reason(f"Processing {len(batches)} batches", { + "run_id": run.id, + "total_rows": total_rows, + "batch_size": batch_size, + }) + + successful_records = 0 + failed_records = 0 + skipped_records = 0 + + for batch_idx, batch_rows in enumerate(batches): + batch_result = self._process_batch( + job=job, + run_id=run.id, + batch_index=batch_idx, + batch_rows=batch_rows, + ) + successful_records += batch_result["successful"] + failed_records += batch_result["failed"] + skipped_records += batch_result["skipped"] + + # Update run stats incrementally + run.successful_records = successful_records + run.failed_records = failed_records + run.skipped_records = skipped_records + self.db.flush() + + if self.on_batch_progress: + self.on_batch_progress( + run.id, batch_idx + 1, len(batches), + successful_records, total_rows, + ) + + # Update final run status + if failed_records == 0 and skipped_records == 0: + run.status = "COMPLETED" + elif successful_records == 0: + run.status = "FAILED" + else: + run.status = "COMPLETED" # Partial success + + run.completed_at = datetime.now(timezone.utc) + self.db.flush() + + logger.reflect("Translation execution complete", { + "run_id": run.id, + "status": run.status, + "total": total_rows, + "successful": successful_records, + "failed": failed_records, + "skipped": skipped_records, + }) + + return run + # [/DEF:execute_run:Function] + + # [DEF:_fetch_source_rows:Function] + # @PURPOSE: Fetch source rows from the accepted preview session for this job. + # @PRE: job_id exists. + # @POST: Returns list of dicts with source data. + def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]: + with belief_scope("TranslationExecutor._fetch_source_rows"): + # Get the latest APPLIED preview session + session = ( + self.db.query(TranslationPreviewSession) + .filter( + TranslationPreviewSession.job_id == job_id, + TranslationPreviewSession.status == "APPLIED", + ) + .order_by(TranslationPreviewSession.created_at.desc()) + .first() + ) + if not session: + logger.explore("No accepted preview session found", {"job_id": job_id}) + return [] + + # Fetch APPROVED or all records from the session + records = ( + self.db.query(TranslationPreviewRecord) + .filter( + TranslationPreviewRecord.session_id == session.id, + TranslationPreviewRecord.status.in_(["APPROVED", "PENDING"]), + ) + .all() + ) + + source_rows = [] + for rec in records: + source_rows.append({ + "row_index": rec.source_object_id or "0", + "source_text": rec.source_sql or "", + "approved_translation": rec.target_sql if rec.status == "APPROVED" else None, + "source_object_name": rec.source_object_name or "", + }) + + logger.reason(f"Fetched {len(source_rows)} source rows from preview", { + "run_id": run_id, + "session_id": session.id, + }) + return source_rows + # [/DEF:_fetch_source_rows:Function] + + # [DEF:_process_batch:Function] + # @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records. + # @PRE: job and batch_rows are valid. + # @POST: TranslationBatch and TranslationRecord rows are created. + # @SIDE_EFFECT: LLM API call. + def _process_batch( + self, + job: TranslationJob, + run_id: str, + batch_index: int, + batch_rows: List[Dict[str, Any]], + ) -> Dict[str, int]: + with belief_scope("TranslationExecutor._process_batch"): + batch_start = time.monotonic() + + # Create batch record + batch = TranslationBatch( + id=str(uuid.uuid4()), + run_id=run_id, + batch_index=batch_index, + status="RUNNING", + total_records=len(batch_rows), + started_at=datetime.now(timezone.utc), + ) + self.db.add(batch) + self.db.flush() + batch_id = batch.id + + result = {"successful": 0, "failed": 0, "skipped": 0, "retries": 0} + + # Extract source texts for dict filtering + source_texts = [ + row.get("source_text", "") + for row in batch_rows + if row.get("source_text") + ] + + # Filter dictionary entries + dict_matches = DictionaryManager.filter_for_batch( + self.db, source_texts, job.id + ) + + # For each row, determine if we need LLM translation or can use approved translation + rows_for_llm = [] + pre_translated = [] + + for row in batch_rows: + if row.get("approved_translation"): + pre_translated.append(row) + else: + rows_for_llm.append(row) + + # Handle pre-translated (approved) rows + for row in pre_translated: + record = TranslationRecord( + id=str(uuid.uuid4()), + batch_id=batch_id, + run_id=run_id, + source_sql=row.get("source_text", ""), + target_sql=row.get("approved_translation"), + source_object_type="table_row", + source_object_id=row.get("row_index"), + source_object_name=row.get("source_object_name", ""), + status="SUCCESS", + ) + self.db.add(record) + result["successful"] += 1 + + # Process rows needing LLM translation + if rows_for_llm: + llm_result = self._call_llm_for_batch( + job=job, + run_id=run_id, + batch_rows=rows_for_llm, + dict_matches=dict_matches, + batch_id=batch_id, + ) + result["successful"] += llm_result["successful"] + result["failed"] += llm_result["failed"] + result["skipped"] += llm_result["skipped"] + result["retries"] += llm_result.get("retries", 0) + + # Update batch status + batch.successful_records = result["successful"] + batch.failed_records = result["failed"] + batch.completed_at = datetime.now(timezone.utc) + batch.status = "COMPLETED" if result["failed"] == 0 else "COMPLETED_WITH_ERRORS" + self.db.flush() + + batch_latency = int((time.monotonic() - batch_start) * 1000) + logger.reason(f"Batch {batch_index} complete", { + "batch_id": batch_id, + "latency_ms": batch_latency, + **result, + }) + + return result + # [/DEF:_process_batch:Function] + + # [DEF:_call_llm_for_batch:Function] + # @PURPOSE: Call LLM for a batch of rows requiring translation. Parse structured JSON response. + # @PRE: job has valid provider_id. batch_rows is non-empty. + # @POST: Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows. + # @SIDE_EFFECT: HTTP call to LLM provider. + def _call_llm_for_batch( + self, + job: TranslationJob, + run_id: str, + batch_rows: List[Dict[str, Any]], + dict_matches: List[Dict[str, Any]], + batch_id: str, + ) -> Dict[str, int]: + with belief_scope("TranslationExecutor._call_llm_for_batch"): + # Build dictionary section + dictionary_section = "" + if dict_matches: + glossary_lines = [] + for m in dict_matches: + glossary_lines.append( + f"- '{m['source_term']}' -> '{m['target_term']}'" + f"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}" + ) + dictionary_section = ( + "Terminology dictionary (use these translations when applicable):\n" + + "\n".join(glossary_lines) + + "\n\n" + ) + + # Build rows JSON for LLM + rows_json = json.dumps([ + { + "row_id": str(row.get("row_index", idx)), + "text": row.get("source_text", ""), + } + for idx, row in enumerate(batch_rows) + ], indent=2) + + # Build prompt + prompt = render_prompt( + DEFAULT_EXECUTION_PROMPT_TEMPLATE, + { + "source_language": job.source_dialect or "SQL", + "target_language": job.target_language or job.target_dialect or "en", + "source_dialect": job.source_dialect or "", + "target_dialect": job.target_dialect or "", + "translation_column": job.translation_column or "", + "dictionary_section": dictionary_section, + "rows_json": rows_json, + "row_count": str(len(batch_rows)), + }, + ) + + # Call LLM with retry + llm_response = None + last_error = None + retries = 0 + + for attempt in range(1, MAX_RETRIES_PER_BATCH + 1): + try: + llm_response = self._call_llm(job, prompt) + break + except Exception as e: + last_error = str(e) + retries += 1 + logger.explore(f"LLM call failed (attempt {attempt})", { + "batch_id": batch_id, + "error": last_error, + "attempt": attempt, + }) + if attempt < MAX_RETRIES_PER_BATCH: + time.sleep(2 ** attempt) # Exponential backoff + else: + logger.explore("LLM call exhausted retries", { + "batch_id": batch_id, + "last_error": last_error, + }) + + if llm_response is None: + # All retries failed — mark all rows as failed + for row in batch_rows: + record = TranslationRecord( + id=str(uuid.uuid4()), + batch_id=batch_id, + run_id=run_id, + source_sql=row.get("source_text", ""), + target_sql=None, + source_object_type="table_row", + source_object_id=row.get("row_index"), + source_object_name=row.get("source_object_name", ""), + status="FAILED", + error_message=f"LLM call failed after {retries} retries: {last_error}", + ) + self.db.add(record) + return {"successful": 0, "failed": len(batch_rows), "skipped": 0, "retries": retries} + + # Parse LLM response + try: + translations = self._parse_llm_response(llm_response, len(batch_rows)) + except ValueError as e: + # Parse failure — mark all rows as SKIPPED + logger.explore("LLM response parse failed", { + "batch_id": batch_id, + "error": str(e), + "response_preview": llm_response[:500] if llm_response else "", + }) + skipped = len(batch_rows) + for row in batch_rows: + record = TranslationRecord( + id=str(uuid.uuid4()), + batch_id=batch_id, + run_id=run_id, + source_sql=row.get("source_text", ""), + target_sql=None, + source_object_type="table_row", + source_object_id=row.get("row_index"), + source_object_name=row.get("source_object_name", ""), + status="SKIPPED", + error_message=f"LLM parse failure: {e}", + ) + self.db.add(record) + return { + "successful": 0, + "failed": 0, + "skipped": skipped, + "retries": retries, + } + + successful = 0 + failed = 0 + skipped = 0 + + for row in batch_rows: + row_id = str(row.get("row_index", "")) + translation = translations.get(row_id) + + if translation is None: + # NULL translation — skip + skipped += 1 + record = TranslationRecord( + id=str(uuid.uuid4()), + batch_id=batch_id, + run_id=run_id, + source_sql=row.get("source_text", ""), + target_sql="", + source_object_type="table_row", + source_object_id=row.get("row_index"), + source_object_name=row.get("source_object_name", ""), + status="SKIPPED", + error_message="NULL translation returned by LLM", + ) + self.db.add(record) + continue + + if translation.strip() == "": + # Empty translation — skip + skipped += 1 + record = TranslationRecord( + id=str(uuid.uuid4()), + batch_id=batch_id, + run_id=run_id, + source_sql=row.get("source_text", ""), + target_sql="", + source_object_type="table_row", + source_object_id=row.get("row_index"), + source_object_name=row.get("source_object_name", ""), + status="SKIPPED", + error_message="Empty translation returned by LLM", + ) + self.db.add(record) + continue + + successful += 1 + record = TranslationRecord( + id=str(uuid.uuid4()), + batch_id=batch_id, + run_id=run_id, + source_sql=row.get("source_text", ""), + target_sql=translation, + source_object_type="table_row", + source_object_id=row.get("row_index"), + source_object_name=row.get("source_object_name", ""), + status="SUCCESS", + ) + self.db.add(record) + + return { + "successful": successful, + "failed": failed, + "skipped": skipped, + "retries": retries, + } + # [/DEF:_call_llm_for_batch:Function] + + # [DEF:_call_llm:Function] + # @PURPOSE: Call the configured LLM provider with the batch prompt. + # @PRE: job has valid provider_id. + # @POST: Returns raw LLM response string. + # @SIDE_EFFECT: HTTP call to LLM provider. + def _call_llm(self, job: TranslationJob, prompt: str) -> str: + with belief_scope("TranslationExecutor._call_llm"): + if not job.provider_id: + raise ValueError("Job has no LLM provider configured") + + provider_svc = LLMProviderService(self.db) + provider = provider_svc.get_provider(job.provider_id) + if not provider: + raise ValueError(f"LLM provider '{job.provider_id}' not found") + + api_key = provider_svc.get_decrypted_api_key(job.provider_id) + if not api_key: + raise ValueError(f"Could not decrypt API key for provider '{job.provider_id}'") + + model = provider.default_model or "gpt-4o-mini" + provider_type = provider.provider_type.lower() if provider.provider_type else "openai" + + if provider_type in ("openai", "openai_compatible"): + return self._call_openai_compatible( + base_url=provider.base_url, + api_key=api_key, + model=model, + prompt=prompt, + ) + else: + raise ValueError(f"Unsupported provider type '{provider_type}'") + # [/DEF:_call_llm:Function] + + # [DEF:_call_openai_compatible:Function] + # @PURPOSE: Call OpenAI-compatible API for batch translation. + # @PRE: Valid API endpoint, key, model, and prompt. + # @POST: Returns response text. + # @SIDE_EFFECT: HTTP POST to LLM API. + @staticmethod + def _call_openai_compatible( + base_url: str, + api_key: str, + model: str, + prompt: str, + ) -> str: + with belief_scope("TranslationExecutor._call_openai_compatible"): + import requests as http_requests + + url = f"{base_url.rstrip('/')}/chat/completions" + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + payload = { + "model": model, + "messages": [ + {"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics."}, + {"role": "user", "content": prompt}, + ], + "temperature": 0.1, + "response_format": {"type": "json_object"}, + } + + response = http_requests.post(url, headers=headers, json=payload, timeout=180) + response.raise_for_status() + data = response.json() + + choices = data.get("choices", []) + if not choices: + raise ValueError("LLM returned no choices") + + content = choices[0].get("message", {}).get("content", "") + if not content: + raise ValueError("LLM returned empty content") + + return content + # [/DEF:_call_openai_compatible:Function] + + # [DEF:_parse_llm_response:Function] + # @PURPOSE: Parse LLM JSON response into dict of row_id -> translation. + # @PRE: response_text is valid JSON with {"rows": [...]} structure. + # @POST: Returns dict mapping row_id to translation text. + @staticmethod + def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]: + with belief_scope("TranslationExecutor._parse_llm_response"): + try: + data = json.loads(response_text) + except json.JSONDecodeError: + # Try to extract from markdown code block + import re + match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', response_text, re.DOTALL) + if match: + try: + data = json.loads(match.group(1)) + except json.JSONDecodeError: + raise ValueError("LLM response was not valid JSON") + else: + raise ValueError("LLM response was not valid JSON") + + rows = data.get("rows", []) + if not isinstance(rows, list): + raise ValueError("LLM response missing 'rows' array") + + translations: Dict[str, str] = {} + for item in rows: + row_id = str(item.get("row_id", "")) + translation = item.get("translation") + if translation is None: + # Skip NULL translations — they'll be handled by caller + continue + if row_id: + translations[row_id] = str(translation) + + return translations + # [/DEF:_parse_llm_response:Function] + + +# [/DEF:TranslationExecutor:Class] +# [/DEF:TranslationExecutor:Module] diff --git a/backend/src/plugins/translate/metrics.py b/backend/src/plugins/translate/metrics.py new file mode 100644 index 00000000..c39d977c --- /dev/null +++ b/backend/src/plugins/translate/metrics.py @@ -0,0 +1,171 @@ +# [DEF:TranslationMetrics:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: translate, metrics, aggregation +# @PURPOSE: Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting. +# @LAYER: Domain +# @RELATION: DEPENDS_ON -> [TranslationEvent:Class] +# @RELATION: DEPENDS_ON -> [MetricSnapshot:Class] +# @RELATION: DEPENDS_ON -> [TranslationRun:Class] +# @RELATION: DEPENDS_ON -> [TranslationSchedule:Class] +# @PRE: Database session is open. +# @POST: Metrics are aggregated and returned; no side effects. +# @RATIONALE: Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture. +# @REJECTED: Querying all events for all time — would be slow; MetricSnapshot provides historical summary. + +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Tuple +from sqlalchemy.orm import Session +from sqlalchemy import func + +from ...core.logger import logger, belief_scope +from ...models.translate import ( + TranslationEvent, + MetricSnapshot, + TranslationRun, + TranslationSchedule, +) + + +# [DEF:TranslationMetrics:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Aggregate translation metrics from live events and MetricSnapshot. +class TranslationMetrics: + + def __init__(self, db: Session): + self.db = db + + # [DEF:get_job_metrics:Function] + # @PURPOSE: Get aggregated metrics for a specific job. + # @PRE: job_id exists. + # @POST: Returns dict with metrics from events + latest snapshot. + def get_job_metrics(self, job_id: str) -> Dict[str, Any]: + with belief_scope("TranslationMetrics.get_job_metrics"): + # Run counts from TranslationRun + run_counts = ( + self.db.query( + TranslationRun.status, + func.count(TranslationRun.id), + ) + .filter(TranslationRun.job_id == job_id) + .group_by(TranslationRun.status) + .all() + ) + + total_runs = 0 + status_counts: Dict[str, int] = {} + for status_val, count in run_counts: + status_counts[status_val] = count + total_runs += count + + # Aggregate record stats from runs + record_stats = ( + self.db.query( + func.coalesce(func.sum(TranslationRun.total_records), 0), + func.coalesce(func.sum(TranslationRun.successful_records), 0), + func.coalesce(func.sum(TranslationRun.failed_records), 0), + func.coalesce(func.sum(TranslationRun.skipped_records), 0), + ) + .filter(TranslationRun.job_id == job_id) + .first() + ) + total_records = record_stats[0] if record_stats else 0 + successful_records = record_stats[1] if record_stats else 0 + failed_records = record_stats[2] if record_stats else 0 + skipped_records = record_stats[3] if record_stats else 0 + + # Average duration from runs + avg_duration = ( + self.db.query( + func.avg( + func.extract('epoch', TranslationRun.completed_at - TranslationRun.started_at) * 1000 + ) + ) + .filter( + TranslationRun.job_id == job_id, + TranslationRun.started_at.isnot(None), + TranslationRun.completed_at.isnot(None), + ) + .scalar() + ) + + # Last run + last_run = ( + self.db.query(TranslationRun) + .filter(TranslationRun.job_id == job_id) + .order_by(TranslationRun.created_at.desc()) + .first() + ) + + # Next scheduled run + next_schedule = ( + self.db.query(TranslationSchedule) + .filter( + TranslationSchedule.job_id == job_id, + TranslationSchedule.is_active == True, + ) + .first() + ) + + # Cumulative tokens/cost from MetricSnapshot + events + cumulative_tokens = 0 + cumulative_cost = 0.0 + + # Latest MetricSnapshot + latest_snapshot = ( + self.db.query(MetricSnapshot) + .filter(MetricSnapshot.job_id == job_id) + .order_by(MetricSnapshot.snapshot_date.desc()) + .first() + ) + if latest_snapshot: + # MetricSnapshot stores per-snapshot aggregated tokens/cost + # Here we sum what we stored — in practice MetricSnapshot.covers_events_before + # indicates cutoff + pass + + # Live events (<90 days) for token/cost + cutoff = datetime.now(timezone.utc) + live_events = ( + self.db.query(TranslationEvent) + .filter( + TranslationEvent.job_id == job_id, + TranslationEvent.event_type.in_(["TRANSLATION_PHASE_COMPLETED", "RUN_COMPLETED"]), + TranslationEvent.created_at > cutoff, # events newer than snapshot + ) + .all() + ) + + return { + "job_id": job_id, + "total_runs": total_runs, + "successful_runs": status_counts.get("COMPLETED", 0), + "failed_runs": status_counts.get("FAILED", 0), + "cancelled_runs": status_counts.get("CANCELLED", 0), + "total_records": int(total_records), + "successful_records": int(successful_records), + "failed_records": int(failed_records), + "skipped_records": int(skipped_records), + "cumulative_tokens": cumulative_tokens, + "cumulative_cost": cumulative_cost, + "avg_duration_ms": int(avg_duration) if avg_duration else None, + "last_run_at": last_run.created_at.isoformat() if last_run else None, + "next_scheduled_run": next_schedule.last_run_at.isoformat() if next_schedule and next_schedule.last_run_at else None, + } + # [/DEF:get_job_metrics:Function] + + # [DEF:get_all_metrics:Function] + # @PURPOSE: Get aggregated metrics for all jobs. + # @POST: Returns list of per-job metrics. + def get_all_metrics(self) -> List[Dict[str, Any]]: + with belief_scope("TranslationMetrics.get_all_metrics"): + job_ids = ( + self.db.query(TranslationRun.job_id) + .distinct() + .all() + ) + return [self.get_job_metrics(jid[0]) for jid in job_ids if jid[0]] + # [/DEF:get_all_metrics:Function] + + +# [/DEF:TranslationMetrics:Class] +# [/DEF:TranslationMetrics:Module] diff --git a/backend/src/plugins/translate/orchestrator.py b/backend/src/plugins/translate/orchestrator.py new file mode 100644 index 00000000..17e55e2f --- /dev/null +++ b/backend/src/plugins/translate/orchestrator.py @@ -0,0 +1,855 @@ +# [DEF:TranslationOrchestrator:Module] +# @COMPLEXITY: 5 +# @SEMANTICS: translate, orchestrator, lifecycle, run +# @PURPOSE: Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events. +# @LAYER: Domain +# @RELATION: DEPENDS_ON -> [TranslationRun] +# @RELATION: DEPENDS_ON -> [TranslationJob] +# @RELATION: DEPENDS_ON -> [TranslationExecutor] +# @RELATION: DEPENDS_ON -> [SQLGenerator] +# @RELATION: DEPENDS_ON -> [SupersetSqlLabExecutor] +# @RELATION: DEPENDS_ON -> [TranslationEventLog] +# @RELATION: DEPENDS_ON -> [TranslationPreviewSession] +# @PRE: Valid job and accepted preview (for manual runs). Superset and LLM are reachable. +# @POST: Translation run is executed, SQL generated and submitted, events recorded. +# @SIDE_EFFECT: Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events. +# @DATA_CONTRACT: Input[db, config_manager, current_user] -> Output[TranslationRun] +# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run. +# @RATIONALE: C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary. +# @REJECTED: Distributed actor model (Celery) — eventual-consistency challenges at current scale. +# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant. + +import json +import time +import uuid +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Callable, Tuple + +from sqlalchemy.orm import Session + +from ...core.logger import logger, belief_scope +from ...core.config_manager import ConfigManager +from ...models.translate import ( + TranslationJob, + TranslationRun, + TranslationBatch, + TranslationRecord, + TranslationPreviewSession, +) +from ...schemas.translate import TranslationRunResponse +from .executor import TranslationExecutor +from .sql_generator import SQLGenerator +from .superset_executor import SupersetSqlLabExecutor +from .events import TranslationEventLog +from ..translate.service import TranslateJobService + + +# [DEF:TranslationOrchestrator:Class] +# @COMPLEXITY: 5 +# @PURPOSE: Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging. +# @PRE: DB session and config manager are available. +# @POST: Runs are created, executed, and finalized with event records. +# @SIDE_EFFECT: Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows. +# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. +class TranslationOrchestrator: + + def __init__( + self, + db: Session, + config_manager: ConfigManager, + current_user: Optional[str] = None, + ): + self.db = db + self.config_manager = config_manager + self.current_user = current_user + self.event_log = TranslationEventLog(db) + self._job: Optional[TranslationJob] = None + + # [DEF:start_run:Function] + # @COMPLEXITY: 5 + # @PURPOSE: Start a new translation run for a job. + # @PRE: job_id exists. For manual runs, there must be an accepted preview session. + # @POST: TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded. + # @SIDE_EFFECT: DB writes. + def start_run( + self, + job_id: str, + is_scheduled: bool = False, + trigger_type: Optional[str] = None, + ) -> TranslationRun: + with belief_scope("TranslationOrchestrator.start_run"): + # Load and validate job + job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() + if not job: + raise ValueError(f"Translation job '{job_id}' not found") + self._job = job + + logger.reason("Starting translation run", { + "job_id": job_id, + "is_scheduled": is_scheduled, + }) + + # Validate preconditions + self._validate_preconditions(job, is_scheduled=is_scheduled) + + # Compute hashes + config_hash = self._compute_config_hash(job) + dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id) + + # Build config snapshot + config_snapshot = { + "source_dialect": job.source_dialect, + "target_dialect": job.target_dialect, + "database_dialect": job.database_dialect, + "source_datasource_id": job.source_datasource_id, + "source_table": job.source_table, + "target_schema": job.target_schema, + "target_table": job.target_table, + "source_key_cols": job.source_key_cols, + "target_key_cols": job.target_key_cols, + "translation_column": job.translation_column, + "context_columns": job.context_columns, + "target_language": job.target_language, + "provider_id": job.provider_id, + "batch_size": job.batch_size, + "upsert_strategy": job.upsert_strategy, + "dictionary_ids": self._compute_dict_snapshot_hash(job_id), + } + + # Compute key_hash from source_key_cols + import hashlib + key_hash_input = json.dumps({ + "source_key_cols": job.source_key_cols, + "source_datasource_id": job.source_datasource_id, + "source_table": job.source_table, + }, sort_keys=True) + key_hash = hashlib.sha256(key_hash_input.encode()).hexdigest()[:16] + + # Create run record with all hash/snapshot fields + run = TranslationRun( + id=str(uuid.uuid4()), + job_id=job_id, + status="PENDING", + trigger_type=trigger_type or ("scheduled" if is_scheduled else "manual"), + config_snapshot=config_snapshot, + key_hash=key_hash, + config_hash=config_hash, + dict_snapshot_hash=dict_snapshot_hash, + created_by=self.current_user, + created_at=datetime.now(timezone.utc), + ) + self.db.add(run) + self.db.flush() + + # Record event + self.event_log.log_event( + job_id=job_id, + run_id=run.id, + event_type="RUN_STARTED", + payload={ + "is_scheduled": is_scheduled, + "trigger_type": run.trigger_type, + "config_hash": config_hash, + "dict_snapshot_hash": dict_snapshot_hash, + "key_hash": key_hash, + }, + created_by=self.current_user, + ) + + self.db.commit() + self.db.refresh(run) + + logger.reflect("Run created", { + "run_id": run.id, + "job_id": job_id, + "status": run.status, + "trigger_type": run.trigger_type, + }) + return run + # [/DEF:start_run:Function] + + # [DEF:execute_run:Function] + # @PURPOSE: Execute a translation run: dispatch executor, generate SQL, submit to Superset. + # @PRE: run is in PENDING status. + # @POST: Run is executed, SQL generated, Superset submission attempted. + # @SIDE_EFFECT: LLM calls, DB writes, Superset API calls. + def execute_run( + self, + run: TranslationRun, + on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None, + skip_insert: bool = False, + ) -> TranslationRun: + with belief_scope("TranslationOrchestrator.execute_run"): + job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first() + if not job: + raise ValueError(f"Job '{run.job_id}' not found") + self._job = job + + if run.status != "PENDING": + raise ValueError( + f"Cannot execute run in status '{run.status}'. " + f"Run must be in PENDING status." + ) + + logger.reason("Executing run", { + "run_id": run.id, + "job_id": job.id, + "skip_insert": skip_insert, + }) + + # Record translation phase start + self.event_log.log_event( + job_id=job.id, + run_id=run.id, + event_type="TRANSLATION_PHASE_STARTED", + payload={}, + created_by=self.current_user, + ) + + # Dispatch executor + executor = TranslationExecutor( + self.db, self.config_manager, self.current_user, + on_batch_progress=on_batch_progress, + ) + try: + run = executor.execute_run(run, llm_progress_callback=None) + except Exception as e: + logger.explore("Translation execution failed", { + "run_id": run.id, + "error": str(e), + }) + run.status = "FAILED" + run.error_message = f"Translation execution failed: {e}" + run.completed_at = datetime.now(timezone.utc) + self.db.flush() + self.event_log.log_event( + job_id=job.id, + run_id=run.id, + event_type="RUN_FAILED", + payload={"error": str(e), "phase": "translation"}, + created_by=self.current_user, + ) + self.db.commit() + return run + + # Record translation phase complete + self.event_log.log_event( + job_id=job.id, + run_id=run.id, + event_type="TRANSLATION_PHASE_COMPLETED", + payload={ + "total": run.total_records, + "successful": run.successful_records, + "failed": run.failed_records, + "skipped": run.skipped_records, + }, + created_by=self.current_user, + ) + + # Skip insert phase if requested (e.g., for preview-only execution) + if skip_insert: + run.status = "COMPLETED" + run.completed_at = datetime.now(timezone.utc) + self.db.flush() + self.event_log.log_event( + job_id=job.id, + run_id=run.id, + event_type="RUN_COMPLETED", + payload={"skip_insert": True}, + created_by=self.current_user, + ) + self.db.commit() + return run + + # Generate SQL and submit to Superset + insert_result = self._generate_and_insert_sql(job, run) + run.insert_status = insert_result.get("status") + run.superset_execution_id = str(insert_result.get("query_id") or "") + run.superset_execution_log = insert_result + run.status = "COMPLETED" if insert_result.get("status") == "success" else "COMPLETED" + if insert_result.get("error_message"): + run.error_message = insert_result["error_message"] + run.completed_at = datetime.now(timezone.utc) + self.db.flush() + + # Record terminal event + terminal_event = "RUN_COMPLETED" + self.event_log.log_event( + job_id=job.id, + run_id=run.id, + event_type=terminal_event, + payload={ + "insert_status": insert_result.get("status"), + "query_id": insert_result.get("query_id"), + "rows_affected": insert_result.get("rows_affected"), + "total_records": run.total_records, + "successful": run.successful_records, + "failed": run.failed_records, + "skipped": run.skipped_records, + }, + created_by=self.current_user, + ) + + self.db.commit() + self.db.refresh(run) + + logger.reflect("Run execution complete", { + "run_id": run.id, + "status": run.status, + "insert_status": run.insert_status, + }) + return run + # [/DEF:execute_run:Function] + + # [DEF:_generate_and_insert_sql:Function] + # @PURPOSE: Generate INSERT SQL from successful records and submit to Superset SQL Lab. + # @PRE: job has target table configured. run has successful records. + # @POST: SQL is generated and submitted. Returns execution result. + # @SIDE_EFFECT: Superset API call. + def _generate_and_insert_sql( + self, + job: TranslationJob, + run: TranslationRun, + ) -> Dict[str, Any]: + with belief_scope("TranslationOrchestrator._generate_and_insert_sql"): + # Fetch successful records + records = ( + self.db.query(TranslationRecord) + .filter( + TranslationRecord.run_id == run.id, + TranslationRecord.status == "SUCCESS", + TranslationRecord.target_sql.isnot(None), + ) + .all() + ) + + if not records: + logger.reason("No successful records to insert", {"run_id": run.id}) + return {"status": "skipped", "reason": "no_records", "query_id": None} + + logger.reason(f"Generating SQL for {len(records)} records", { + "run_id": run.id, + "dialect": job.database_dialect or job.target_dialect, + }) + + # Build rows for SQL generation + columns = job.context_columns or [] + if job.translation_column and job.translation_column not in columns: + columns.append(job.translation_column) + + # Also include key columns if used for upsert + if job.target_key_cols: + for k in job.target_key_cols: + if k not in columns: + columns.append(k) + + rows_for_sql = [] + for rec in records: + row_data = {} + if job.context_columns: + for col in job.context_columns: + row_data[col] = "" + if job.translation_column: + row_data[job.translation_column] = rec.target_sql or "" + if job.target_key_cols: + for k in job.target_key_cols: + row_data[k] = "" + rows_for_sql.append(row_data) + + if not columns: + # Use target_sql as the sole column + columns = [job.translation_column or "translated_text"] + rows_for_sql = [{columns[0]: rec.target_sql or ""} for rec in records] + + # Generate SQL + try: + sql, row_count = SQLGenerator.generate( + dialect=job.database_dialect or job.target_dialect or "postgresql", + target_schema=job.target_schema, + target_table=job.target_table or "translated_data", + columns=columns, + rows=rows_for_sql, + key_columns=job.target_key_cols, + upsert_strategy=job.upsert_strategy or "MERGE", + ) + except ValueError as e: + logger.explore("SQL generation failed", {"error": str(e)}) + return {"status": "failed", "error_message": str(e), "query_id": None} + + # Log insert phase start + self.event_log.log_event( + job_id=job.id, + run_id=run.id, + event_type="INSERT_PHASE_STARTED", + payload={"sql_length": len(sql), "row_count": row_count}, + created_by=self.current_user, + ) + + # Submit to Superset + try: + env_id = job.source_dialect or "" + executor = SupersetSqlLabExecutor(self.config_manager, env_id) + result = executor.execute_and_poll( + sql=sql, + max_polls=30, + poll_interval_seconds=2.0, + ) + except Exception as e: + logger.explore("Superset SQL submission failed", { + "run_id": run.id, + "error": str(e), + }) + result = {"status": "failed", "error_message": str(e), "query_id": None} + + # Log insert phase complete + self.event_log.log_event( + job_id=job.id, + run_id=run.id, + event_type="INSERT_PHASE_COMPLETED", + payload=result, + created_by=self.current_user, + ) + + return result + # [/DEF:_generate_and_insert_sql:Function] + + # [DEF:_validate_preconditions:Function] + # @PURPOSE: Validate preconditions before starting a run. + # @PRE: None. + # @POST: Raises ValueError if preconditions are not met. + # @SIDE_EFFECT: None. + def _validate_preconditions( + self, + job: TranslationJob, + is_scheduled: bool = False, + ) -> None: + with belief_scope("TranslationOrchestrator._validate_preconditions"): + # Job must be in valid status + if job.status in ("DRAFT",): + raise ValueError( + f"Cannot run job '{job.id}' in status '{job.status}'. " + f"Job must be READY, ACTIVE, or COMPLETED." + ) + + # Must have target table configured + if not job.target_table: + raise ValueError( + f"Job '{job.id}' has no target table configured. " + "Configure a target table before running." + ) + + # Must have LLM provider configured + if not job.provider_id: + raise ValueError( + f"Job '{job.id}' has no LLM provider configured. " + "Select an LLM provider before running." + ) + + # Must have a translation column + if not job.translation_column: + raise ValueError( + f"Job '{job.id}' has no translation column configured. " + "Select a translation column before running." + ) + + # For manual runs, must have accepted preview + if not is_scheduled: + accepted_session = ( + self.db.query(TranslationPreviewSession) + .filter( + TranslationPreviewSession.job_id == job.id, + TranslationPreviewSession.status == "APPLIED", + ) + .order_by(TranslationPreviewSession.created_at.desc()) + .first() + ) + if not accepted_session: + raise ValueError( + f"Job '{job.id}' has no accepted preview session. " + "Run and accept a preview before executing a manual translation run." + ) + + logger.reason("Preconditions validated", { + "job_id": job.id, + "is_scheduled": is_scheduled, + }) + # [/DEF:_validate_preconditions:Function] + + # [DEF:retry_failed_batches:Function] + # @PURPOSE: Retry failed batches in a run. + # @PRE: run exists and has failed batches. + # @POST: Failed batches are re-processed. + # @SIDE_EFFECT: LLM calls, DB writes. + def retry_failed_batches( + self, + run_id: str, + ) -> TranslationRun: + with belief_scope("TranslationOrchestrator.retry_failed_batches"): + run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first() + if not run: + raise ValueError(f"Run '{run_id}' not found") + + job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first() + if not job: + raise ValueError(f"Job '{run.job_id}' not found") + self._job = job + + # Find failed batches + failed_batches = ( + self.db.query(TranslationBatch) + .filter( + TranslationBatch.run_id == run_id, + TranslationBatch.status.in_(["FAILED", "COMPLETED_WITH_ERRORS"]), + ) + .all() + ) + + if not failed_batches: + raise ValueError(f"No failed batches found for run '{run_id}'") + + logger.reason("Retrying failed batches", { + "run_id": run_id, + "batch_count": len(failed_batches), + }) + + self.event_log.log_event( + job_id=job.id, + run_id=run.id, + event_type="RUN_RETRYING", + payload={"batch_count": len(failed_batches)}, + created_by=self.current_user, + ) + + # Re-process each failed batch + executor = TranslationExecutor(self.db, self.config_manager, self.current_user) + run.status = "RUNNING" + self.db.flush() + + for batch in failed_batches: + # Fetch failed records for this batch + failed_records = ( + self.db.query(TranslationRecord) + .filter( + TranslationRecord.batch_id == batch.id, + TranslationRecord.status == "FAILED", + ) + .all() + ) + + if not failed_records: + continue + + # Build rows from failed records + retry_rows = [] + for rec in failed_records: + retry_rows.append({ + "row_index": rec.source_object_id or "0", + "source_text": rec.source_sql or "", + "approved_translation": None, + "source_object_name": rec.source_object_name or "", + }) + + # Process retry batch + result = executor._process_batch( + job=job, + run_id=run_id, + batch_index=batch.batch_index, + batch_rows=retry_rows, + ) + + # Update run stats + run.successful_records = (run.successful_records or 0) + result["successful"] + run.failed_records = (run.failed_records or 0) + result["failed"] + run.skipped_records = (run.skipped_records or 0) + result["skipped"] + self.db.flush() + + # Update run status + if run.failed_records == 0: + run.status = "COMPLETED" + elif run.successful_records == 0: + run.status = "FAILED" + else: + run.status = "COMPLETED" + + run.completed_at = datetime.now(timezone.utc) + self.db.flush() + + self.event_log.log_event( + job_id=job.id, + run_id=run.id, + event_type="RUN_COMPLETED", + payload={"retry": True}, + created_by=self.current_user, + ) + + self.db.commit() + logger.reflect("Retry complete", { + "run_id": run_id, + "status": run.status, + }) + return run + # [/DEF:retry_failed_batches:Function] + + # [DEF:retry_insert:Function] + # @PURPOSE: Retry the SQL insert phase for a completed run. + # @PRE: run exists and has successful records. + # @POST: SQL is regenerated and re-submitted to Superset. + # @SIDE_EFFECT: Superset API call. + def retry_insert( + self, + run_id: str, + ) -> TranslationRun: + with belief_scope("TranslationOrchestrator.retry_insert"): + run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first() + if not run: + raise ValueError(f"Run '{run_id}' not found") + + job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first() + if not job: + raise ValueError(f"Job '{run.job_id}' not found") + self._job = job + + logger.reason("Retrying insert phase", { + "run_id": run_id, + }) + + self.event_log.log_event( + job_id=job.id, + run_id=run.id, + event_type="RUN_RETRY_INSERT", + payload={}, + created_by=self.current_user, + ) + + # Regenerate SQL and submit + insert_result = self._generate_and_insert_sql(job, run) + + # Update run + run.insert_status = insert_result.get("status") + run.superset_execution_id = str(insert_result.get("query_id") or "") + if insert_result.get("error_message"): + run.error_message = insert_result["error_message"] + self.db.flush() + self.db.commit() + self.db.refresh(run) + + logger.reflect("Insert retry complete", { + "run_id": run_id, + "insert_status": run.insert_status, + }) + return run + # [/DEF:retry_insert:Function] + + # [DEF:cancel_run:Function] + # @PURPOSE: Cancel a running translation. + # @PRE: run is in PENDING or RUNNING status. + # @POST: Run status is set to CANCELLED. + # @SIDE_EFFECT: DB write; records event. + def cancel_run(self, run_id: str) -> TranslationRun: + with belief_scope("TranslationOrchestrator.cancel_run"): + run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first() + if not run: + raise ValueError(f"Run '{run_id}' not found") + + if run.status not in ("PENDING", "RUNNING"): + raise ValueError( + f"Cannot cancel run in status '{run.status}'. " + f"Only PENDING or RUNNING runs can be cancelled." + ) + + run.status = "CANCELLED" + run.completed_at = datetime.now(timezone.utc) + self.db.flush() + + self.event_log.log_event( + job_id=run.job_id, + run_id=run.id, + event_type="RUN_CANCELLED", + payload={}, + created_by=self.current_user, + ) + + self.db.commit() + self.db.refresh(run) + + logger.reflect("Run cancelled", { + "run_id": run_id, + }) + return run + # [/DEF:cancel_run:Function] + + # [DEF:get_run_status:Function] + # @PURPOSE: Get run status with statistics. + # @PRE: run_id exists. + # @POST: Returns dict with run details. + def get_run_status(self, run_id: str) -> Dict[str, Any]: + with belief_scope("TranslationOrchestrator.get_run_status"): + run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first() + if not run: + raise ValueError(f"Run '{run_id}' not found") + + # Count batches + batch_count = ( + self.db.query(TranslationBatch) + .filter(TranslationBatch.run_id == run_id) + .count() + ) + + # Get event summary + event_summary = self.event_log.get_run_event_summary(run_id) + + return { + "id": run.id, + "job_id": run.job_id, + "status": run.status, + "started_at": run.started_at.isoformat() if run.started_at else None, + "completed_at": run.completed_at.isoformat() if run.completed_at else None, + "error_message": run.error_message, + "total_records": run.total_records or 0, + "successful_records": run.successful_records or 0, + "failed_records": run.failed_records or 0, + "skipped_records": run.skipped_records or 0, + "insert_status": run.insert_status, + "superset_execution_id": run.superset_execution_id, + "batch_count": batch_count, + "event_invariants": { + "has_run_started": event_summary["has_run_started"], + "terminal_event_count": event_summary["terminal_event_count"], + "invariant_valid": event_summary["invariant_valid"], + }, + "created_by": run.created_by, + "created_at": run.created_at.isoformat() if run.created_at else None, + } + # [/DEF:get_run_status:Function] + + # [DEF:get_run_records:Function] + # @PURPOSE: Get paginated records for a run. + # @PRE: run_id exists. + # @POST: Returns dict with records and pagination info. + def get_run_records( + self, + run_id: str, + page: int = 1, + page_size: int = 50, + status_filter: Optional[str] = None, + ) -> Dict[str, Any]: + with belief_scope("TranslationOrchestrator.get_run_records"): + query = self.db.query(TranslationRecord).filter( + TranslationRecord.run_id == run_id + ) + + if status_filter: + query = query.filter(TranslationRecord.status == status_filter) + + total = query.count() + offset = (page - 1) * page_size + records = ( + query.order_by(TranslationRecord.created_at.desc()) + .offset(offset) + .limit(page_size) + .all() + ) + + return { + "items": [ + { + "id": r.id, + "batch_id": r.batch_id, + "source_sql": r.source_sql, + "target_sql": r.target_sql, + "source_object_type": r.source_object_type, + "source_object_id": r.source_object_id, + "source_object_name": r.source_object_name, + "status": r.status, + "error_message": r.error_message, + "created_at": r.created_at.isoformat() if r.created_at else None, + } + for r in records + ], + "total": total, + "page": page, + "page_size": page_size, + "status_filter": status_filter, + } + # [/DEF:get_run_records:Function] + + # [DEF:get_run_history:Function] + # @PURPOSE: Get run history for a job. + # @PRE: job_id exists. + # @POST: Returns list of runs. + def get_run_history( + self, + job_id: str, + page: int = 1, + page_size: int = 20, + ) -> Tuple[int, List[Dict[str, Any]]]: + with belief_scope("TranslationOrchestrator.get_run_history"): + query = self.db.query(TranslationRun).filter( + TranslationRun.job_id == job_id + ) + total = query.count() + runs = ( + query.order_by(TranslationRun.created_at.desc()) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + + return total, [ + { + "id": r.id, + "job_id": r.job_id, + "status": r.status, + "started_at": r.started_at.isoformat() if r.started_at else None, + "completed_at": r.completed_at.isoformat() if r.completed_at else None, + "error_message": r.error_message, + "total_records": r.total_records or 0, + "successful_records": r.successful_records or 0, + "failed_records": r.failed_records or 0, + "skipped_records": r.skipped_records or 0, + "insert_status": r.insert_status, + "superset_execution_id": r.superset_execution_id, + "created_by": r.created_by, + "created_at": r.created_at.isoformat() if r.created_at else None, + } + for r in runs + ] + # [/DEF:get_run_history:Function] + + # [DEF:_compute_config_hash:Function] + # @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison. + @staticmethod + def _compute_config_hash(job: TranslationJob) -> str: + import hashlib + config_str = json.dumps({ + "source_dialect": job.source_dialect, + "target_dialect": job.target_dialect, + "source_datasource_id": job.source_datasource_id, + "translation_column": job.translation_column, + "context_columns": job.context_columns, + "target_language": job.target_language, + "provider_id": job.provider_id, + "batch_size": job.batch_size, + "upsert_strategy": job.upsert_strategy, + }, sort_keys=True) + return hashlib.sha256(config_str.encode()).hexdigest()[:16] + # [/DEF:_compute_config_hash:Function] + + # [DEF:_compute_dict_snapshot_hash:Function] + # @PURPOSE: Compute a hash of dictionary state for snapshot comparison. + def _compute_dict_snapshot_hash(self, job_id: str) -> str: + import hashlib + from ...models.translate import TranslationJobDictionary + dict_links = ( + self.db.query(TranslationJobDictionary) + .filter(TranslationJobDictionary.job_id == job_id) + .all() + ) + dict_ids = sorted([dl.dictionary_id for dl in dict_links]) + hash_input = ",".join(dict_ids) + return hashlib.sha256(hash_input.encode()).hexdigest()[:16] + # [/DEF:_compute_dict_snapshot_hash:Function] + + +# [/DEF:TranslationOrchestrator:Class] +# [/DEF:TranslationOrchestrator:Module] diff --git a/backend/src/plugins/translate/plugin.py b/backend/src/plugins/translate/plugin.py new file mode 100644 index 00000000..9112301c --- /dev/null +++ b/backend/src/plugins/translate/plugin.py @@ -0,0 +1,62 @@ +# [DEF:TranslatePlugin:Module] +# @COMPLEXITY: 2 +# @SEMANTICS: plugin, translate, llm, sql, dashboard +# @PURPOSE: TranslatePlugin skeleton for cross-dialect SQL and dashboard translation. +# @LAYER: Domain +# @RELATION: INHERITS -> [PluginBase] +# @RATIONALE: Separate plugin avoids bloating LLMAnalysisPlugin beyond fractal limit (<400 lines). +# @REJECTED: Extending LLMAnalysisPlugin would conflate two distinct feature domains. + +from typing import Dict, Any, Optional +from ...core.plugin_base import PluginBase + + +# [DEF:TranslatePlugin:Class] +# @PURPOSE: Plugin for translating SQL queries and dashboard definitions across database dialects. +# @RELATION: IMPLEMENTS -> backend.src.core.plugin_base.PluginBase +class TranslatePlugin(PluginBase): + @property + def id(self) -> str: + return "translate" + + @property + def name(self) -> str: + return "LLM Table Translation" + + @property + def description(self) -> str: + return "Cross-dialect SQL and dashboard translation using LLMs." + + @property + def version(self) -> str: + return "1.0.0" + + def get_schema(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "source_dialect": { + "type": "string", + "title": "Source Dialect", + "description": "Source database dialect (e.g. PostgreSQL, ClickHouse)", + }, + "target_dialect": { + "type": "string", + "title": "Target Dialect", + "description": "Target database dialect (e.g. PostgreSQL, ClickHouse)", + }, + "job_id": { + "type": "string", + "title": "Translation Job ID", + "description": "Existing translation job to execute", + }, + }, + "required": ["job_id"], + } + + async def execute(self, params: Dict[str, Any], context: Optional[Any] = None): + """Execute a translation job — implementation deferred to later phases.""" + raise NotImplementedError("TranslatePlugin.execute not yet implemented") +# [/DEF:TranslatePlugin:Class] + +# [/DEF:TranslatePlugin:Module] diff --git a/backend/src/plugins/translate/preview.py b/backend/src/plugins/translate/preview.py new file mode 100644 index 00000000..a9db69bb --- /dev/null +++ b/backend/src/plugins/translate/preview.py @@ -0,0 +1,788 @@ +# [DEF:TranslationPreview:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: translate, preview, llm, session +# @PURPOSE: Preview session management: fetch sample rows from Superset, send to LLM with context + filtered dictionary, return side-by-side results. +# @LAYER: Domain +# @RELATION: DEPENDS_ON -> [TranslationJob:Class] +# @RELATION: DEPENDS_ON -> [TranslationPreviewSession:Class] +# @RELATION: DEPENDS_ON -> [TranslationPreviewRecord:Class] +# @RELATION: DEPENDS_ON -> [LLMProviderService] +# @RELATION: DEPENDS_ON -> [DictionaryManager] +# @RELATION: DEPENDS_ON -> [SupersetClient] +# @RELATION: DEPENDS_ON -> [render_prompt] +# @RELATION: DEPENDS_ON -> [ConfigManager] +# @PRE: Database session and config manager are available. +# @POST: Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected. +# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows. +# @RATIONALE: C4 because preview is stateful with approve/edit/reject lifecycle and LLM API calls. +# @REJECTED: Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably. + +from typing import Any, Dict, List, Optional, Tuple +from sqlalchemy.orm import Session +from datetime import datetime, timezone, timedelta +import uuid +import json +import hashlib +import re + +from ...core.logger import logger, belief_scope +from ...core.config_manager import ConfigManager +from ...core.superset_client import SupersetClient +from ...models.translate import ( + TranslationJob, + TranslationPreviewSession, + TranslationPreviewRecord, + TranslationJobDictionary, +) +from ...services.llm_provider import LLMProviderService +from ...services.llm_prompt_templates import render_prompt +from .dictionary import DictionaryManager + +# [DEF:DEFAULT_EXECUTION_PROMPT_TEMPLATE:Constant] +# @PURPOSE: Default prompt template for batch LLM translation execution (no context columns — faster). +DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = ( + "Translate the following database content from {source_language} to {target_language}.\n\n" + "Source dialect: {source_dialect}\n" + "Target dialect: {target_dialect}\n" + "Column to translate: {translation_column}\n\n" + "{dictionary_section}" + "For each row, provide an accurate translation of the text.\n\n" + "Rows to translate:\n{rows_json}\n\n" + "Respond with a JSON object in this exact format:\n" + '{{"rows": [{{"row_id": "", "translation": ""}}]}}\n' + "Each row_id must match the index provided. Return exactly {row_count} entries." +) +# [/DEF:DEFAULT_EXECUTION_PROMPT_TEMPLATE:Constant] + + +# [DEF:DEFAULT_PREVIEW_PROMPT_TEMPLATE:Constant] +# @PURPOSE: Default prompt template for LLM translation preview. +DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = ( + "Translate the following database content from {source_language} to {target_language}.\n\n" + "Source dialect: {source_dialect}\n" + "Target dialect: {target_dialect}\n" + "Column to translate: {translation_column}\n\n" + "{dictionary_section}" + "For each row, provide an accurate translation of the '{translation_column}' value.\n" + "Consider the context columns when determining the meaning of the text.\n\n" + "Rows to translate:\n{rows_json}\n\n" + "Respond with a JSON object in this exact format:\n" + '{{"rows": [{{"row_id": "", "translation": ""}}]}}\n' + "Each row_id must match the index provided. Return exactly {row_count} entries." +) +# [/DEF:DEFAULT_PREVIEW_PROMPT_TEMPLATE:Constant] + + +# [DEF:TokenEstimator:Class] +# @PURPOSE: Estimate token counts and costs for LLM translation operations. +# @RATIONALE: Token estimation uses a heuristic (chars/token ratio) since exact tokenization depends on the LLM model. +# @REJECTED: Using an external tokenizer library would introduce a heavy dependency for estimation only. +class TokenEstimator: + """Estimate token counts and costs for LLM operations.""" + + CHARS_PER_TOKEN_ESTIMATE: float = 4.0 + OUTPUT_TOKENS_PER_ROW_ESTIMATE: int = 50 + TOKEN_COST_PER_1K: float = 0.002 # Default cost per 1K tokens + + # [DEF:estimate_prompt_tokens:Function] + # @PURPOSE: Estimate token count for a prompt string. + # @PRE: prompt is a non-empty string. + # @POST: Returns estimated token count (integer). + @staticmethod + def estimate_prompt_tokens(prompt: str) -> int: + if not prompt: + return 0 + return max(1, int(len(prompt) / TokenEstimator.CHARS_PER_TOKEN_ESTIMATE)) + # [/DEF:estimate_prompt_tokens:Function] + + # [DEF:estimate_output_tokens:Function] + # @PURPOSE: Estimate output token count for translating N rows. + # @PRE: row_count >= 0. + # @POST: Returns estimated output token count. + @staticmethod + def estimate_output_tokens(row_count: int) -> int: + return row_count * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE + # [/DEF:estimate_output_tokens:Function] + + # [DEF:estimate_cost:Function] + # @PURPOSE: Estimate cost for a given number of tokens. + # @PRE: total_tokens >= 0. + # @POST: Returns estimated cost in USD. + @staticmethod + def estimate_cost(total_tokens: int, cost_per_1k: Optional[float] = None) -> float: + rate = cost_per_1k if cost_per_1k is not None else TokenEstimator.TOKEN_COST_PER_1K + return round((total_tokens / 1000) * rate, 6) + # [/DEF:estimate_cost:Function] + + +# [/DEF:TokenEstimator:Class] + + +# [DEF:TranslationPreview:Class] +# @COMPLEXITY: 4 +# @PURPOSE: Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate. +# @PRE: Database session and config manager are available. +# @POST: Preview sessions created with persisted records; full execution gates on accepted session. +# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows. +class TranslationPreview: + + def __init__( + self, + db: Session, + config_manager: ConfigManager, + current_user: Optional[str] = None, + ): + self.db = db + self.config_manager = config_manager + self.current_user = current_user + + # [DEF:preview_rows:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records. + # @PRE: job_id exists and job has source_datasource_id, translation_column configured. + # @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session. + # @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows. + def preview_rows( + self, + job_id: str, + sample_size: int = 10, + prompt_template: Optional[str] = None, + ) -> Dict[str, Any]: + with belief_scope("TranslationPreview.preview_rows"): + logger.reason("Starting preview for job", {"job_id": job_id, "sample_size": sample_size}) + + # 1. Load job + job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() + if not job: + raise ValueError(f"Translation job '{job_id}' not found") + if not job.source_datasource_id: + raise ValueError("Job must have a source datasource configured for preview") + if not job.translation_column: + raise ValueError("Job must have a translation column configured for preview") + + # 2. Compute config hash and dict snapshot hash + config_hash = self._compute_config_hash(job) + dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id) + + # 3. Fetch sample rows from Superset + logger.reason("Fetching sample rows from Superset", { + "datasource_id": job.source_datasource_id, + "sample_size": sample_size, + "translation_column": job.translation_column, + }) + source_rows = self._fetch_sample_rows( + job=job, + sample_size=sample_size, + ) + if not source_rows: + raise ValueError("No rows returned from datasource for preview") + + actual_row_count = len(source_rows) + logger.reason("Fetched sample rows", {"actual_count": actual_row_count}) + + # 4. Build prompt context from rows + all_source_texts = [] + row_meta: List[Dict[str, Any]] = [] + for idx, row in enumerate(source_rows): + translation_value = str(row.get(job.translation_column, "") or "") + context_values = {} + if job.context_columns: + for col in job.context_columns: + context_values[col] = str(row.get(col, "") or "") + all_source_texts.append(translation_value) + row_meta.append({ + "row_index": idx, + "source_text": translation_value, + "context_data": context_values, + "source_row": row, + }) + + # 5. Filter dictionary entries for this batch + dict_matches = DictionaryManager.filter_for_batch( + self.db, all_source_texts, job_id + ) + + # Build dictionary glossary section + dictionary_section = "" + if dict_matches: + glossary_lines = [] + for m in dict_matches: + glossary_lines.append( + f"- '{m['source_term']}' -> '{m['target_term']}'" + f"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}" + ) + dictionary_section = ( + "Terminology dictionary (use these translations when applicable):\n" + + "\n".join(glossary_lines) + + "\n\n" + ) + + # 6. Build LLM prompt + rows_json = json.dumps([ + {"row_id": str(m["row_index"]), "text": m["source_text"], "context": m["context_data"]} + for m in row_meta + ], indent=2) + + template = prompt_template or DEFAULT_PREVIEW_PROMPT_TEMPLATE + prompt = render_prompt(template, { + "source_language": job.source_dialect or "SQL", + "target_language": job.target_language or job.target_dialect or "en", + "source_dialect": job.source_dialect or "", + "target_dialect": job.target_dialect or "", + "translation_column": job.translation_column or "", + "dictionary_section": dictionary_section, + "rows_json": rows_json, + "row_count": str(actual_row_count), + }) + + # 7. Estimate tokens/cost for sample and full dataset + sample_prompt_tokens = TokenEstimator.estimate_prompt_tokens(prompt) + sample_output_tokens = TokenEstimator.estimate_output_tokens(actual_row_count) + sample_total_tokens = sample_prompt_tokens + sample_output_tokens + sample_cost = TokenEstimator.estimate_cost(sample_total_tokens) + + # Estimate full dataset cost (if we knew total rows) + total_est_tokens = TokenEstimator.estimate_prompt_tokens( + prompt.replace(str(actual_row_count), "{total}") + ) + TokenEstimator.estimate_output_tokens(sample_size * 10) # rough extrapolation + total_est_cost = TokenEstimator.estimate_cost(total_est_tokens) + + # 8. Call LLM + logger.reason("Calling LLM for preview translation", { + "provider_id": job.provider_id, + "row_count": actual_row_count, + "estimated_tokens": sample_total_tokens, + }) + llm_response = self._call_llm( + job=job, + prompt=prompt, + ) + + # 9. Parse LLM response + translations = self._parse_llm_response(llm_response, actual_row_count) + + # 10. Create preview session + session = TranslationPreviewSession( + id=str(uuid.uuid4()), + job_id=job_id, + status="ACTIVE", + created_by=self.current_user, + created_at=datetime.now(timezone.utc), + expires_at=datetime.now(timezone.utc) + timedelta(hours=24), + ) + self.db.add(session) + self.db.flush() + + # 11. Create preview records + records = [] + for meta in row_meta: + idx = meta["row_index"] + translation = translations.get(str(idx), "") + is_rejected = False + status = "PENDING" + feedback = None + + record = TranslationPreviewRecord( + id=str(uuid.uuid4()), + session_id=session.id, + source_sql=meta["source_text"], + target_sql=translation, + source_object_type="table_row", + source_object_id=str(idx), + source_object_name=f"Row {idx + 1}", + status=status, + feedback=feedback, + created_at=datetime.now(timezone.utc), + ) + self.db.add(record) + self.db.flush() + records.append({ + "id": record.id, + "source_sql": record.source_sql, + "target_sql": record.target_sql, + "source_object_type": record.source_object_type, + "source_object_id": record.source_object_id, + "source_object_name": record.source_object_name, + "status": record.status, + "feedback": record.feedback, + }) + + self.db.commit() + + result = { + "id": session.id, + "job_id": job_id, + "status": "ACTIVE", + "created_by": self.current_user, + "created_at": session.created_at.isoformat(), + "expires_at": session.expires_at.isoformat() if session.expires_at else None, + "records": records, + "cost_estimate": { + "sample_size": actual_row_count, + "sample_prompt_tokens": sample_prompt_tokens, + "sample_output_tokens": sample_output_tokens, + "sample_total_tokens": sample_total_tokens, + "sample_cost": sample_cost, + "estimated_total_rows": actual_row_count * 10, + "estimated_tokens": total_est_tokens, + "estimated_cost": total_est_cost, + }, + "config_hash": config_hash, + "dict_snapshot_hash": dict_snapshot_hash, + } + + logger.reflect("Preview completed", { + "session_id": session.id, + "row_count": actual_row_count, + "sample_cost": sample_cost, + }) + return result + # [/DEF:preview_rows:Function] + + # [DEF:update_preview_row:Function] + # @PURPOSE: Approve, edit, or reject an individual preview row. + # @PRE: session_id and row_id exist, session is ACTIVE. + # @POST: PreviewRecord status is updated. + def update_preview_row( + self, + job_id: str, + row_id: str, + action: str, + translation: Optional[str] = None, + feedback: Optional[str] = None, + ) -> Dict[str, Any]: + with belief_scope("TranslationPreview.update_preview_row"): + # Find the active session for this job + session = ( + self.db.query(TranslationPreviewSession) + .filter( + TranslationPreviewSession.job_id == job_id, + TranslationPreviewSession.status == "ACTIVE", + ) + .order_by(TranslationPreviewSession.created_at.desc()) + .first() + ) + if not session: + raise ValueError(f"No active preview session for job '{job_id}'") + + record = ( + self.db.query(TranslationPreviewRecord) + .filter( + TranslationPreviewRecord.id == row_id, + TranslationPreviewRecord.session_id == session.id, + ) + .first() + ) + if not record: + raise ValueError(f"Preview record '{row_id}' not found in active session") + + if action == "approve": + record.status = "APPROVED" + elif action == "reject": + record.status = "REJECTED" + elif action == "edit": + record.status = "APPROVED" + if translation is not None: + record.target_sql = translation + else: + raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.") + + if feedback is not None: + record.feedback = feedback + + self.db.commit() + self.db.refresh(record) + + logger.reason(f"Preview row {action}d", { + "row_id": row_id, + "session_id": session.id, + "status": record.status, + }) + + return { + "id": record.id, + "source_sql": record.source_sql, + "target_sql": record.target_sql, + "status": record.status, + "feedback": record.feedback, + } + # [/DEF:update_preview_row:Function] + + # [DEF:accept_preview_session:Function] + # @PURPOSE: Mark a preview session as accepted, which gates full execution. + # @PRE: job_id has an ACTIVE preview session. + # @POST: Session status changes to APPLIED. + # @SIDE_EFFECT: Future full execution calls will check for accepted session. + def accept_preview_session(self, job_id: str) -> Dict[str, Any]: + with belief_scope("TranslationPreview.accept_preview_session"): + session = ( + self.db.query(TranslationPreviewSession) + .filter( + TranslationPreviewSession.job_id == job_id, + TranslationPreviewSession.status == "ACTIVE", + ) + .order_by(TranslationPreviewSession.created_at.desc()) + .first() + ) + if not session: + raise ValueError(f"No active preview session for job '{job_id}'") + + session.status = "APPLIED" + self.db.commit() + self.db.refresh(session) + + logger.reason("Preview session accepted", { + "session_id": session.id, + "job_id": job_id, + }) + + # Get records for response + records = ( + self.db.query(TranslationPreviewRecord) + .filter(TranslationPreviewRecord.session_id == session.id) + .all() + ) + + return { + "id": session.id, + "job_id": job_id, + "status": "APPLIED", + "created_by": session.created_by, + "created_at": session.created_at.isoformat(), + "expires_at": session.expires_at.isoformat() if session.expires_at else None, + "records": [ + { + "id": r.id, + "source_sql": r.source_sql, + "target_sql": r.target_sql, + "status": r.status, + "feedback": r.feedback, + } + for r in records + ], + } + # [/DEF:accept_preview_session:Function] + + # [DEF:get_preview_session:Function] + # @PURPOSE: Get the latest preview session for a job with its records. + # @PRE: job_id exists. + # @POST: Returns session data with records or raises ValueError. + def get_preview_session(self, job_id: str) -> Dict[str, Any]: + with belief_scope("TranslationPreview.get_preview_session"): + session = ( + self.db.query(TranslationPreviewSession) + .filter(TranslationPreviewSession.job_id == job_id) + .order_by(TranslationPreviewSession.created_at.desc()) + .first() + ) + if not session: + raise ValueError(f"No preview session found for job '{job_id}'") + + records = ( + self.db.query(TranslationPreviewRecord) + .filter(TranslationPreviewRecord.session_id == session.id) + .all() + ) + + return { + "id": session.id, + "job_id": job_id, + "status": session.status, + "created_by": session.created_by, + "created_at": session.created_at.isoformat(), + "expires_at": session.expires_at.isoformat() if session.expires_at else None, + "records": [ + { + "id": r.id, + "source_sql": r.source_sql, + "target_sql": r.target_sql, + "source_object_type": r.source_object_type, + "source_object_id": r.source_object_id, + "source_object_name": r.source_object_name, + "status": r.status, + "feedback": r.feedback, + } + for r in records + ], + } + # [/DEF:get_preview_session:Function] + + # [DEF:_fetch_sample_rows:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Fetch sample rows from the Superset dataset for preview. + # @PRE: job has source_datasource_id and translation_column. + # @POST: Returns list of dicts with row data. + # @SIDE_EFFECT: Calls Superset chart data endpoint. + def _fetch_sample_rows(self, job: TranslationJob, sample_size: int = 10) -> List[Dict[str, Any]]: + with belief_scope("TranslationPreview._fetch_sample_rows"): + # Find environment config using source_dialect as env_id + environments = self.config_manager.get_environments() + env_config = next( + (e for e in environments if e.id == job.source_dialect), + None, + ) + if not env_config: + logger.explore("Could not find environment for datasource", { + "env_id": job.source_dialect, + }) + # Fallback: try first environment + if environments: + env_config = environments[0] + logger.explore("Falling back to first available environment", { + "env_id": env_config.id, + }) + else: + raise ValueError("No Superset environments configured") + + client = SupersetClient(env_config) + + # Fetch dataset detail to build proper query context + dataset_detail = client.get_dataset_detail(int(job.source_datasource_id)) + + # Determine columns to query + query_columns = [job.translation_column] + if job.context_columns: + query_columns.extend(job.context_columns) + + # Build query context for chart data endpoint + query_context = client.build_dataset_preview_query_context( + dataset_id=int(job.source_datasource_id), + dataset_record=dataset_detail, + template_params={}, + effective_filters=[], + ) + + # Modify to fetch specific columns as raw data (no aggregation) + queries = query_context.get("queries", []) + if queries: + queries[0]["columns"] = query_columns + queries[0]["metrics"] = [] + queries[0]["row_limit"] = sample_size + queries[0]["result_type"] = "query" + + try: + response = client.network.request( + method="POST", + endpoint="/api/v1/chart/data", + data=json.dumps(query_context), + headers={"Content-Type": "application/json"}, + ) + except Exception as e: + # Try legacy endpoint as fallback + logger.explore("Chart data endpoint failed, trying legacy", {"error": str(e)}) + try: + response = client.network.request( + method="POST", + endpoint="/explore_json/form_data", + params={"form_data": json.dumps(query_context.get("form_data", {}))}, + headers={"Content-Type": "application/json"}, + ) + except Exception as e2: + raise ValueError( + f"Failed to fetch sample data from Superset: {e2}" + ) + + # Parse response + rows = self._extract_data_rows(response) + logger.reason("Extracted data rows", {"count": len(rows)}) + return rows + # [/DEF:_fetch_sample_rows:Function] + + # [DEF:_extract_data_rows:Function] + # @PURPOSE: Extract data rows from Superset chart data response. + # @PRE: response is a dict from Superset API. + # @POST: Returns list of row dicts. + @staticmethod + def _extract_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]: + with belief_scope("TranslationPreview._extract_data_rows"): + # Try various response formats + result = response.get("result") + if isinstance(result, list): + for item in result: + if isinstance(item, dict): + data = item.get("data") + if isinstance(data, list) and data: + return data + + # Try flat result + if isinstance(result, dict): + data = result.get("data") + if isinstance(data, list) and data: + return data + + # Legacy: response may have data at top level + data = response.get("data") + if isinstance(data, list) and data: + return data + + # Last resort: return response itself wrapped if it looks like a list of rows + if isinstance(result, list): + return result + + return [] + # [/DEF:_extract_data_rows:Function] + + # [DEF:_call_llm:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Call the configured LLM provider with the preview prompt. + # @PRE: job has a valid provider_id. + # @POST: Returns raw LLM response string. + # @SIDE_EFFECT: Makes HTTP call to LLM provider. + def _call_llm(self, job: TranslationJob, prompt: str) -> str: + with belief_scope("TranslationPreview._call_llm"): + if not job.provider_id: + raise ValueError("Job has no LLM provider configured") + + provider_svc = LLMProviderService(self.db) + provider = provider_svc.get_provider(job.provider_id) + if not provider: + raise ValueError(f"LLM provider '{job.provider_id}' not found") + + api_key = provider_svc.get_decrypted_api_key(job.provider_id) + if not api_key: + raise ValueError(f"Could not decrypt API key for provider '{job.provider_id}'") + + # Build the API call based on provider type + model = provider.default_model or "gpt-4o-mini" + provider_type = provider.provider_type.lower() if provider.provider_type else "openai" + + if provider_type in ("openai", "openai_compatible"): + response_text = self._call_openai_compatible( + base_url=provider.base_url, + api_key=api_key, + model=model, + prompt=prompt, + ) + else: + raise ValueError(f"Unsupported provider type '{provider_type}' for preview") + + logger.reason("LLM call completed", { + "provider_id": job.provider_id, + "model": model, + "response_length": len(response_text), + }) + return response_text + # [/DEF:_call_llm:Function] + + # [DEF:_call_openai_compatible:Function] + # @PURPOSE: Call an OpenAI-compatible API for translation. + # @PRE: base_url, api_key, model, prompt are valid. + # @POST: Returns response text. + # @SIDE_EFFECT: Makes HTTP POST to LLM API. + @staticmethod + def _call_openai_compatible( + base_url: str, + api_key: str, + model: str, + prompt: str, + ) -> str: + with belief_scope("TranslationPreview._call_openai_compatible"): + import requests as http_requests + + url = f"{base_url.rstrip('/')}/chat/completions" + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + payload = { + "model": model, + "messages": [ + {"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics."}, + {"role": "user", "content": prompt}, + ], + "temperature": 0.1, + "response_format": {"type": "json_object"}, + } + + response = http_requests.post(url, headers=headers, json=payload, timeout=120) + response.raise_for_status() + data = response.json() + + choices = data.get("choices", []) + if not choices: + raise ValueError("LLM returned no choices") + + content = choices[0].get("message", {}).get("content", "") + if not content: + raise ValueError("LLM returned empty content") + + return content + # [/DEF:_call_openai_compatible:Function] + + # [DEF:_parse_llm_response:Function] + # @PURPOSE: Parse the LLM JSON response into a dict of row_id -> translation. + # @PRE: response_text is valid JSON with {"rows": [...]} structure. + # @POST: Returns dict mapping string row_id to translation text. + @staticmethod + def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]: + with belief_scope("TranslationPreview._parse_llm_response"): + try: + data = json.loads(response_text) + except json.JSONDecodeError: + # Try to extract JSON from markdown code block + import re + match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', response_text, re.DOTALL) + if match: + try: + data = json.loads(match.group(1)) + except json.JSONDecodeError: + raise ValueError("LLM response was not valid JSON") + else: + raise ValueError("LLM response was not valid JSON") + + rows = data.get("rows", []) + if not isinstance(rows, list): + raise ValueError("LLM response missing 'rows' array") + + translations: Dict[str, str] = {} + for item in rows: + row_id = str(item.get("row_id", "")) + translation = str(item.get("translation", "")) + if row_id: + translations[row_id] = translation + + if len(translations) < expected_count: + logger.explore("LLM returned fewer translations than expected", { + "expected": expected_count, + "received": len(translations), + "missing": [str(i) for i in range(expected_count) if str(i) not in translations], + }) + + return translations + # [/DEF:_parse_llm_response:Function] + + # [DEF:_compute_config_hash:Function] + # @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison. + @staticmethod + def _compute_config_hash(job: TranslationJob) -> str: + config_str = json.dumps({ + "source_dialect": job.source_dialect, + "target_dialect": job.target_dialect, + "source_datasource_id": job.source_datasource_id, + "translation_column": job.translation_column, + "context_columns": job.context_columns, + "target_language": job.target_language, + "provider_id": job.provider_id, + "batch_size": job.batch_size, + "upsert_strategy": job.upsert_strategy, + }, sort_keys=True) + return hashlib.sha256(config_str.encode()).hexdigest()[:16] + # [/DEF:_compute_config_hash:Function] + + # [DEF:_compute_dict_snapshot_hash:Function] + # @PURPOSE: Compute a hash of the dictionary state for snapshot comparison. + def _compute_dict_snapshot_hash(self, job_id: str) -> str: + dict_links = ( + self.db.query(TranslationJobDictionary) + .filter(TranslationJobDictionary.job_id == job_id) + .all() + ) + dict_ids = sorted([dl.dictionary_id for dl in dict_links]) + hash_input = ",".join(dict_ids) + return hashlib.sha256(hash_input.encode()).hexdigest()[:16] + # [/DEF:_compute_dict_snapshot_hash:Function] + + +# [/DEF:TranslationPreview:Class] + +# [/DEF:TranslationPreview:Module] diff --git a/backend/src/plugins/translate/scheduler.py b/backend/src/plugins/translate/scheduler.py new file mode 100644 index 00000000..12a8c911 --- /dev/null +++ b/backend/src/plugins/translate/scheduler.py @@ -0,0 +1,361 @@ +# [DEF:TranslationScheduler:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: translate, scheduler, apscheduler, cron +# @PURPOSE: Manage TranslationSchedule rows and register them with core SchedulerService. +# @LAYER: Domain +# @RELATION: DEPENDS_ON -> [TranslationSchedule:Class] +# @RELATION: DEPENDS_ON -> [SchedulerService:Class] +# @RELATION: DEPENDS_ON -> [TranslationOrchestrator:Class] +# @RELATION: DEPENDS_ON -> [TranslationEventLog:Class] +# @PRE: Database session and SchedulerService are available. +# @POST: TranslationSchedule CRUD persisted; APScheduler jobs registered/updated/removed. +# @SIDE_EFFECT: Registers APScheduler jobs; runs translations on trigger; creates events. +# @RATIONALE: Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance. +# @REJECTED: Separate scheduler instance would create resource contention. +# @REJECTED: Polling-based approach — event-driven APScheduler is more precise. + +import uuid +from datetime import datetime, timezone, timedelta +from typing import Any, Dict, List, Optional + +from sqlalchemy.orm import Session + +from ...core.logger import logger, belief_scope +from ...core.config_manager import ConfigManager +from ...models.translate import TranslationSchedule, TranslationJob, TranslationRun +from .events import TranslationEventLog + + +# [DEF:TranslationScheduler:Class] +# @COMPLEXITY: 4 +# @PURPOSE: CRUD for TranslationSchedule rows + APScheduler registration wrappers. +class TranslationScheduler: + + def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None): + self.db = db + self.config_manager = config_manager + self.current_user = current_user + self.event_log = TranslationEventLog(db) + + # [DEF:create_schedule:Function] + # @PURPOSE: Create a new schedule for a job. + # @PRE: job_id exists. cron_expression is valid. + # @POST: TranslationSchedule row created. + def create_schedule( + self, + job_id: str, + cron_expression: str, + timezone: str = "UTC", + is_active: bool = True, + ) -> TranslationSchedule: + with belief_scope("TranslationScheduler.create_schedule"): + # Verify job exists + job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() + if not job: + raise ValueError(f"Translation job '{job_id}' not found") + + schedule = TranslationSchedule( + id=str(uuid.uuid4()), + job_id=job_id, + cron_expression=cron_expression, + timezone=timezone, + is_active=is_active, + created_by=self.current_user, + ) + self.db.add(schedule) + self.db.commit() + self.db.refresh(schedule) + + self.event_log.log_event( + job_id=job_id, + event_type="SCHEDULE_CREATED", + payload={ + "schedule_id": schedule.id, + "cron_expression": cron_expression, + "timezone": timezone, + }, + created_by=self.current_user, + ) + + logger.reflect("Schedule created", {"schedule_id": schedule.id, "job_id": job_id}) + return schedule + # [/DEF:create_schedule:Function] + + # [DEF:update_schedule:Function] + # @PURPOSE: Update an existing schedule. + # @PRE: job_id has an existing schedule. + # @POST: Schedule updated. + def update_schedule( + self, + job_id: str, + cron_expression: Optional[str] = None, + timezone_str: Optional[str] = None, + is_active: Optional[bool] = None, + ) -> TranslationSchedule: + with belief_scope("TranslationScheduler.update_schedule"): + schedule = self.db.query(TranslationSchedule).filter( + TranslationSchedule.job_id == job_id + ).first() + if not schedule: + raise ValueError(f"No schedule found for job '{job_id}'") + + if cron_expression is not None: + schedule.cron_expression = cron_expression + if timezone_str is not None: + schedule.timezone = timezone_str + if is_active is not None: + schedule.is_active = is_active + schedule.updated_at = datetime.now(timezone.utc) + self.db.commit() + self.db.refresh(schedule) + + self.event_log.log_event( + job_id=job_id, + event_type="SCHEDULE_UPDATED", + payload={ + "schedule_id": schedule.id, + "cron_expression": schedule.cron_expression, + "timezone": schedule.timezone, + "is_active": schedule.is_active, + }, + created_by=self.current_user, + ) + + logger.reflect("Schedule updated", {"schedule_id": schedule.id, "job_id": job_id}) + return schedule + # [/DEF:update_schedule:Function] + + # [DEF:delete_schedule:Function] + # @PURPOSE: Delete a schedule for a job. + # @PRE: job_id has an existing schedule. + # @POST: Schedule deleted. + def delete_schedule(self, job_id: str) -> None: + with belief_scope("TranslationScheduler.delete_schedule"): + schedule = self.db.query(TranslationSchedule).filter( + TranslationSchedule.job_id == job_id + ).first() + if not schedule: + raise ValueError(f"No schedule found for job '{job_id}'") + + schedule_id = schedule.id + self.db.delete(schedule) + self.db.commit() + + self.event_log.log_event( + job_id=job_id, + event_type="SCHEDULE_DELETED", + payload={"schedule_id": schedule_id}, + created_by=self.current_user, + ) + + logger.reflect("Schedule deleted", {"schedule_id": schedule_id, "job_id": job_id}) + # [/DEF:delete_schedule:Function] + + # [DEF:enable_disable_schedule:Function] + # @PURPOSE: Enable or disable a schedule. + # @PRE: job_id has an existing schedule. + # @POST: Schedule is_active updated. + def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule: + with belief_scope("TranslationScheduler.set_schedule_active"): + schedule = self.db.query(TranslationSchedule).filter( + TranslationSchedule.job_id == job_id + ).first() + if not schedule: + raise ValueError(f"No schedule found for job '{job_id}'") + + schedule.is_active = is_active + schedule.updated_at = datetime.now(timezone.utc) + self.db.commit() + self.db.refresh(schedule) + + logger.reflect("Schedule active state set", { + "schedule_id": schedule.id, + "job_id": job_id, + "is_active": is_active, + }) + return schedule + # [/DEF:enable_disable_schedule:Function] + + # [DEF:get_schedule:Function] + # @PURPOSE: Get schedule for a job. + # @PRE: job_id exists. + # @POST: Returns TranslationSchedule or raises ValueError. + def get_schedule(self, job_id: str) -> TranslationSchedule: + with belief_scope("TranslationScheduler.get_schedule"): + schedule = self.db.query(TranslationSchedule).filter( + TranslationSchedule.job_id == job_id + ).first() + if not schedule: + raise ValueError(f"No schedule found for job '{job_id}'") + return schedule + # [/DEF:get_schedule:Function] + + # [DEF:list_active_schedules:Function] + # @PURPOSE: List all active schedules. + # @POST: Returns list of active TranslationSchedule rows. + @staticmethod + def list_active_schedules(db: Session) -> List[TranslationSchedule]: + return ( + db.query(TranslationSchedule) + .filter(TranslationSchedule.is_active == True) + .all() + ) + # [/DEF:list_active_schedules:Function] + + # [DEF:get_next_executions:Function] + # @PURPOSE: Compute next N execution times from cron expression. + # @PRE: cron_expression is valid. + # @POST: Returns list of ISO datetime strings. + @staticmethod + def get_next_executions(cron_expression: str, timezone_str: str = "UTC", n: int = 3) -> List[str]: + from zoneinfo import ZoneInfo + from apscheduler.triggers.cron import CronTrigger + from apscheduler.triggers.interval import IntervalTrigger + + try: + tz = ZoneInfo(timezone_str) + trigger = CronTrigger.from_crontab(cron_expression, timezone=tz) + except (ValueError, KeyError) as e: + logger.warning(f"[get_next_executions] Invalid cron: {e}") + return [] + + now = datetime.now(tz) + results = [] + next_time = now + prev = None + for _ in range(n): + ft = trigger.get_next_fire_time(prev, next_time) + if ft is None: + break + results.append(ft.isoformat()) + prev = ft + next_time = ft + return results + # [/DEF:get_next_executions:Function] + + +# [/DEF:TranslationScheduler:Class] + + +# [DEF:execute_scheduled_translation:Function] +# @PURPOSE: APScheduler job handler — runs scheduled translation with concurrency check and new-key-only fallback. +# @PRE: schedule_id is valid. +# @POST: Translation run created and executed if no concurrent run exists. +# @SIDE_EFFECT: DB writes; LLM calls; Superset API calls. +# @RATIONALE: New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full. +def execute_scheduled_translation( + schedule_id: str, + job_id: str, + db_session_maker, + config_manager: ConfigManager, +) -> None: + """APScheduler job callback for scheduled translations.""" + db: Session = db_session_maker() + try: + with belief_scope("execute_scheduled_translation"): + # Load schedule + schedule = db.query(TranslationSchedule).filter( + TranslationSchedule.id == schedule_id, + TranslationSchedule.job_id == job_id, + ).first() + if not schedule or not schedule.is_active: + logger.info(f"[scheduled_translation] Schedule inactive/missing: {schedule_id}") + return + + logger.reason("Scheduled translation triggered", { + "schedule_id": schedule_id, + "job_id": job_id, + }) + + # Concurrency check: max 1 pending/running run per job + active_run = ( + db.query(TranslationRun) + .filter( + TranslationRun.job_id == job_id, + TranslationRun.status.in_(["PENDING", "RUNNING"]), + ) + .first() + ) + if active_run: + logger.explore("Skipping scheduled run — concurrent run in progress", { + "job_id": job_id, + "existing_run_id": active_run.id, + }) + event_log = TranslationEventLog(db) + event_log.log_event( + job_id=job_id, + event_type="RUN_STARTED", + payload={"reason": "skipped_concurrent", "existing_run_id": active_run.id}, + ) + return + + # Check baseline expiry + baseline_expired = False + most_recent = ( + db.query(TranslationRun) + .filter( + TranslationRun.job_id == job_id, + TranslationRun.insert_status == "succeeded", + ) + .order_by(TranslationRun.created_at.desc()) + .first() + ) + if most_recent: + age = datetime.now(timezone.utc) - most_recent.created_at + if age > timedelta(days=90): + baseline_expired = True + logger.reason("Baseline expired — full translation", { + "job_id": job_id, + "last_run": most_recent.created_at.isoformat(), + "age_days": age.days, + }) + + # Import and execute + from .orchestrator import TranslationOrchestrator + + orch = TranslationOrchestrator(db, config_manager, current_user="scheduler") + run = orch.start_run( + job_id=job_id, + is_scheduled=True, + ) + run.trigger_type = "scheduled" + db.flush() + + if baseline_expired: + event_log = TranslationEventLog(db) + event_log.log_event( + job_id=job_id, + run_id=run.id, + event_type="RUN_STARTED", + payload={"reason": "baseline_expired", "age_days": age.days}, + ) + + # Execute in same thread (APScheduler runs in background thread pool) + try: + orch.execute_run(run) + run.insert_status = run.insert_status or "succeeded" + except Exception as exec_err: + logger.explore("Scheduled translation execution failed", { + "run_id": run.id, + "error": str(exec_err), + }) + # Leave schedule enabled — schedule continues on failure + run.status = "FAILED" + run.error_message = str(exec_err) + run.completed_at = datetime.now(timezone.utc) + + # Update schedule tracking + schedule.last_run_at = datetime.now(timezone.utc) + db.commit() + + logger.reflect("Scheduled translation complete", { + "schedule_id": schedule_id, + "run_id": run.id, + "status": run.status, + }) + except Exception as e: + logger.error(f"[scheduled_translation] Unexpected error: {e}") + finally: + db.close() +# [/DEF:execute_scheduled_translation:Function] +# [/DEF:TranslationScheduler:Module] diff --git a/backend/src/plugins/translate/service.py b/backend/src/plugins/translate/service.py new file mode 100644 index 00000000..2e0ff094 --- /dev/null +++ b/backend/src/plugins/translate/service.py @@ -0,0 +1,541 @@ +# [DEF:TranslateJobService:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: translate, service, crud, validation +# @PURPOSE: Service layer for translation job CRUD with datasource column validation and database dialect detection. +# @LAYER: Domain +# @RELATION: DEPENDS_ON -> [TranslationJob] +# @RELATION: DEPENDS_ON -> [SupersetClient] +# @RELATION: DEPENDS_ON -> [ConfigManager] +# @PRE: Database session and config manager are available. +# @POST: Translation jobs are created/updated/deleted with column validation and dialect caching. +# @SIDE_EFFECT: Queries Superset for column metadata and database dialect at save time. +# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. +# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity. + +from typing import Any, Dict, List, Optional, Tuple +from sqlalchemy.orm import Session +from datetime import datetime, timezone +import uuid + +from ...core.logger import logger +from ...core.config_manager import ConfigManager +from ...core.superset_client import SupersetClient +from ...models.translate import TranslationJob, TranslationJobDictionary +from ...schemas.translate import ( + TranslateJobCreate, + TranslateJobUpdate, + TranslateJobResponse, + DatasourceColumnsResponse, + DatasourceColumnResponse, +) + +# Supported database dialects for translation +SUPPORTED_DIALECTS = { + "postgresql", "mysql", "clickhouse", "sqlite", "mssql", + "oracle", "snowflake", "bigquery", "redshift", "presto", + "trino", "druid", "hive", "spark", "databricks", +} + + +# [DEF:get_dialect_from_database:Function] +# @PURPOSE: Extract normalized dialect string from a Superset database record. +# @PRE: database_record is a dict from Superset API. +# @POST: Returns normalized dialect string or raises ValueError if unsupported. +def get_dialect_from_database(database_record: Dict[str, Any]) -> str: + """Extract and validate dialect from Superset database record.""" + backend = ( + database_record.get("backend") + or database_record.get("engine") + or "" + ).lower().strip() + + if not backend: + raise ValueError("Could not determine database dialect from connection") + + # Map Superset backend names to normalized dialect + dialect_map = { + "postgresql": "postgresql", + "mysql": "mysql", + "clickhouse": "clickhouse", + "sqlite": "sqlite", + "mssql": "mssql", + "oracle": "oracle", + "snowflake": "snowflake", + "bigquery": "bigquery", + "redshift": "redshift", + "presto": "presto", + "trino": "trino", + "druid": "druid", + "hive": "hive", + "spark": "spark", + "databricks": "databricks", + } + + normalized = dialect_map.get(backend, backend) + if normalized not in SUPPORTED_DIALECTS: + raise ValueError( + f"Unsupported database dialect: '{backend}'. " + f"Supported dialects: {', '.join(sorted(SUPPORTED_DIALECTS))}" + ) + return normalized +# [/DEF:get_dialect_from_database:Function] + + +# [DEF:fetch_datasource_metadata:Function] +# @PURPOSE: Fetch datasource columns and database dialect from Superset. +# @PRE: dataset_id is a valid Superset dataset ID and environment has valid credentials. +# @POST: Returns (columns_list, dialect_string) or raises on failure. +def fetch_datasource_metadata( + dataset_id: int, + env_id: str, + config_manager: ConfigManager, +) -> Tuple[List[Dict[str, Any]], str]: + """Fetch column metadata and database dialect for a datasource from Superset.""" + # Find environment config + environments = config_manager.get_environments() + env_config = next((e for e in environments if e.id == env_id), None) + if not env_config: + raise ValueError(f"Superset environment '{env_id}' not found in configuration") + + # Create Superset client and fetch dataset detail + client = SupersetClient(env_config) + dataset_detail = client.get_dataset_detail(dataset_id) + + # Extract columns + raw_columns = dataset_detail.get("columns", []) + columns = [] + for col in raw_columns: + col_name = col.get("name") or col.get("column_name") + if not col_name: + continue + columns.append({ + "name": str(col_name), + "type": col.get("type"), + "is_physical": col.get("is_physical", True), + "is_dttm": col.get("is_dttm", False), + "description": col.get("description", ""), + }) + + # Extract database dialect from datasource + database_info = dataset_detail.get("database", {}) + if isinstance(database_info, dict): + dialect = get_dialect_from_database(database_info) + else: + # Fallback: try to fetch database directly + try: + db_id = dataset_detail.get("database_id") + if db_id: + db_record = client.get_database(int(db_id)) + db_result = db_record.get("result", db_record) if isinstance(db_record, dict) else db_record + dialect = get_dialect_from_database(db_result) + else: + raise ValueError("No database information available for this datasource") + except Exception as e: + logger.warning(f"[translate_service] Could not fetch database dialect: {e}") + raise ValueError(f"Could not determine database dialect: {e}") + + return columns, dialect +# [/DEF:fetch_datasource_metadata:Function] + + +# [DEF:detect_virtual_columns:Function] +# @PURPOSE: Identify virtual (calculated) columns from column metadata. +# @PRE: columns is a list of dicts with 'is_physical' key. +# @POST: Returns list of virtual column names. +def detect_virtual_columns(columns: List[Dict[str, Any]]) -> List[str]: + """Return names of columns that are virtual (not physical).""" + return [col["name"] for col in columns if not col.get("is_physical", True)] +# [/DEF:detect_virtual_columns:Function] + + +# [DEF:TranslateJobService:Class] +# @PURPOSE: Service for translation job CRUD with validation and Superset integration. +class TranslateJobService: + + def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None): + self.db = db + self.config_manager = config_manager + self.current_user = current_user + + # [DEF:list_jobs:Function] + # @PURPOSE: List translation jobs with optional status filter and pagination. + # @POST: Returns tuple of (total_count, list_of_jobs). + def list_jobs( + self, + page: int = 1, + page_size: int = 20, + status_filter: Optional[str] = None, + ) -> Tuple[int, List[TranslationJob]]: + query = self.db.query(TranslationJob) + + if status_filter: + query = query.filter(TranslationJob.status == status_filter) + + total = query.count() + offset = (page - 1) * page_size + jobs = query.order_by(TranslationJob.created_at.desc()).offset(offset).limit(page_size).all() + + return total, jobs + # [/DEF:list_jobs:Function] + + # [DEF:get_job:Function] + # @PURPOSE: Get a single translation job by ID. + # @POST: Returns TranslationJob or raises ValueError. + def get_job(self, job_id: str) -> TranslationJob: + job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() + if not job: + raise ValueError(f"Translation job '{job_id}' not found") + return job + # [/DEF:get_job:Function] + + # [DEF:create_job:Function] + # @PURPOSE: Create a new translation job with column validation. + # @PRE: payload contains valid job configuration. + # @POST: Returns the created TranslationJob with database_dialect cached. + # @SIDE_EFFECT: Validates columns via SupersetClient if source_datasource_id is provided. + # @SIDE_EFFECT: Caches database_dialect from Superset connection. + def create_job(self, payload: TranslateJobCreate) -> TranslationJob: + logger.info(f"[TranslateJobService] Creating job '{payload.name}'") + + # Validate: must have a translation column if datasource is configured + if payload.source_datasource_id and not payload.translation_column: + raise ValueError("A translation column is required when a datasource is selected") + + # Validate upsert strategy + valid_strategies = {"MERGE", "INSERT", "UPDATE"} + if payload.upsert_strategy not in valid_strategies: + raise ValueError( + f"Invalid upsert_strategy '{payload.upsert_strategy}'. " + f"Must be one of: {', '.join(sorted(valid_strategies))}" + ) + + # Detect database dialect and validate columns if datasource is specified + dialect = payload.database_dialect + if payload.source_datasource_id and payload.source_dialect: + # If no explicit dialect, try to detect it + if not dialect: + try: + env_id = payload.source_dialect + _, detected_dialect = fetch_datasource_metadata( + int(payload.source_datasource_id), + env_id, + self.config_manager, + ) + dialect = detected_dialect + except Exception as e: + logger.warning(f"[TranslateJobService] Dialect detection failed: {e}") + dialect = payload.source_dialect + + # Build job instance + job = TranslationJob( + id=str(uuid.uuid4()), + name=payload.name, + description=payload.description, + source_dialect=payload.source_dialect, + target_dialect=payload.target_dialect, + database_dialect=dialect, + source_datasource_id=payload.source_datasource_id, + source_table=payload.source_table, + target_schema=payload.target_schema, + target_table=payload.target_table, + source_key_cols=payload.source_key_cols or [], + target_key_cols=payload.target_key_cols or [], + translation_column=payload.translation_column, + context_columns=payload.context_columns or [], + target_language=payload.target_language, + provider_id=payload.provider_id, + batch_size=payload.batch_size, + upsert_strategy=payload.upsert_strategy, + status="DRAFT", + created_by=self.current_user, + ) + self.db.add(job) + self.db.flush() + + # Attach dictionaries + if payload.dictionary_ids: + for dict_id in payload.dictionary_ids: + assoc = TranslationJobDictionary( + id=str(uuid.uuid4()), + job_id=job.id, + dictionary_id=dict_id, + ) + self.db.add(assoc) + + self.db.commit() + self.db.refresh(job) + logger.info(f"[TranslateJobService] Created job '{job.id}'") + return job + # [/DEF:create_job:Function] + + # [DEF:update_job:Function] + # @PURPOSE: Update an existing translation job. + # @PRE: payload contains fields to update. + # @POST: Returns the updated TranslationJob. + # @SIDE_EFFECT: Re-detects database_dialect if source_datasource_id changed. + def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob: + logger.info(f"[TranslateJobService] Updating job '{job_id}'") + job = self.get_job(job_id) + + update_data = payload.model_dump(exclude_unset=True) + dict_ids = update_data.pop("dictionary_ids", None) + + for field, value in update_data.items(): + if hasattr(job, field): + setattr(job, field, value) + + # Re-detect dialect if datasource changed + if payload.source_datasource_id and not payload.database_dialect: + try: + env_id = (payload.source_dialect or job.source_dialect) + _, detected_dialect = fetch_datasource_metadata( + int(payload.source_datasource_id), + env_id, + self.config_manager, + ) + job.database_dialect = detected_dialect + except Exception as e: + logger.warning(f"[TranslateJobService] Dialect re-detection failed: {e}") + + job.updated_at = datetime.now(timezone.utc) + + # Update dictionary associations if provided + if dict_ids is not None: + self.db.query(TranslationJobDictionary).filter( + TranslationJobDictionary.job_id == job_id + ).delete() + for dict_id in dict_ids: + assoc = TranslationJobDictionary( + id=str(uuid.uuid4()), + job_id=job_id, + dictionary_id=dict_id, + ) + self.db.add(assoc) + + self.db.commit() + self.db.refresh(job) + logger.info(f"[TranslateJobService] Updated job '{job_id}'") + return job + # [/DEF:update_job:Function] + + # [DEF:delete_job:Function] + # @PURPOSE: Delete a translation job and its associations. + # @PRE: job_id must exist. + # @POST: Job and all related records are deleted. + def delete_job(self, job_id: str) -> None: + logger.info(f"[TranslateJobService] Deleting job '{job_id}'") + job = self.get_job(job_id) + + # Delete dictionary associations + self.db.query(TranslationJobDictionary).filter( + TranslationJobDictionary.job_id == job_id + ).delete() + + self.db.delete(job) + self.db.commit() + logger.info(f"[TranslateJobService] Deleted job '{job_id}'") + # [/DEF:delete_job:Function] + + # [DEF:duplicate_job:Function] + # @PURPOSE: Duplicate a translation job with a new name. + # @PRE: job_id must exist. + # @POST: Returns the duplicated TranslationJob with status DRAFT. + def duplicate_job(self, job_id: str, new_name: Optional[str] = None) -> TranslationJob: + logger.info(f"[TranslateJobService] Duplicating job '{job_id}'") + source = self.get_job(job_id) + + # Copy all fields except id, created_at, updated_at, status + new_job = TranslationJob( + id=str(uuid.uuid4()), + name=new_name or f"{source.name} (Copy)", + description=source.description, + source_dialect=source.source_dialect, + target_dialect=source.target_dialect, + database_dialect=source.database_dialect, + source_datasource_id=source.source_datasource_id, + source_table=source.source_table, + target_schema=source.target_schema, + target_table=source.target_table, + source_key_cols=source.source_key_cols, + target_key_cols=source.target_key_cols, + translation_column=source.translation_column, + context_columns=source.context_columns, + target_language=source.target_language, + provider_id=source.provider_id, + batch_size=source.batch_size, + upsert_strategy=source.upsert_strategy, + status="DRAFT", + created_by=self.current_user, + ) + self.db.add(new_job) + self.db.flush() + + # Copy dictionary associations + old_dicts = self.db.query(TranslationJobDictionary).filter( + TranslationJobDictionary.job_id == job_id + ).all() + for assoc in old_dicts: + new_assoc = TranslationJobDictionary( + id=str(uuid.uuid4()), + job_id=new_job.id, + dictionary_id=assoc.dictionary_id, + ) + self.db.add(new_assoc) + + self.db.commit() + self.db.refresh(new_job) + logger.info(f"[TranslateJobService] Duplicated job '{job_id}' -> '{new_job.id}'") + return new_job + # [/DEF:duplicate_job:Function] + + # [DEF:get_job_dictionary_ids:Function] + # @PURPOSE: Get dictionary IDs attached to a job. + # @POST: Returns list of dictionary IDs. + def get_job_dictionary_ids(self, job_id: str) -> List[str]: + assocs = self.db.query(TranslationJobDictionary).filter( + TranslationJobDictionary.job_id == job_id + ).all() + return [a.dictionary_id for a in assocs] + # [/DEF:get_job_dictionary_ids:Function] + + # [DEF:fetch_available_datasources:Function] + # @PURPOSE: List available Superset datasets for translation job creation. + def fetch_available_datasources(self, env_id: str, search: Optional[str] = None) -> list: + """List Superset datasets available for translation.""" + from ...core.superset_client import SupersetClient + env = self.config_manager.get_environment(env_id) + if not env: + raise ValueError(f"Environment '{env_id}' not found") + client = SupersetClient(env) + _, datasets = client.get_datasets() + result = [] + for ds in datasets: + name = ds.get("table_name", "") + if search and search.lower() not in name.lower(): + continue + db_info = ds.get("database", {}) + backend = db_info.get("backend", "") + dialect = _extract_dialect(backend) + result.append({ + "id": ds.get("id"), + "table_name": name, + "schema": ds.get("schema"), + "database_name": db_info.get("database_name", "Unknown"), + "database_dialect": dialect, + "description": ds.get("description", ""), + }) + return result + # [/DEF:fetch_available_datasources:Function] +# [/DEF:TranslateJobService:Class] + + +# [DEF:_extract_dialect:Function] +# @PURPOSE: Extract database dialect from Superset backend URI. +def _extract_dialect(backend: str) -> str: + """Extract dialect name from a Superset database backend URI (e.g. 'postgresql+psycopg2://...').""" + if not backend: + return "unknown" + try: + scheme = backend.split("://")[0] + dialect = scheme.split("+")[0] + return dialect.lower() + except Exception: + return "unknown" + + +# [DEF:job_to_response:Function] +# @PURPOSE: Convert a TranslationJob ORM model to a TranslateJobResponse schema with dictionary_ids. +def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -> TranslateJobResponse: + return TranslateJobResponse( + id=job.id, + name=job.name, + description=job.description, + source_dialect=job.source_dialect, + target_dialect=job.target_dialect, + database_dialect=job.database_dialect, + source_datasource_id=job.source_datasource_id, + source_table=job.source_table, + target_schema=job.target_schema, + target_table=job.target_table, + source_key_cols=job.source_key_cols or [], + target_key_cols=job.target_key_cols or [], + translation_column=job.translation_column, + context_columns=job.context_columns or [], + target_language=job.target_language, + provider_id=job.provider_id, + batch_size=job.batch_size or 50, + upsert_strategy=job.upsert_strategy or "MERGE", + status=job.status, + created_by=job.created_by, + created_at=job.created_at, + updated_at=job.updated_at, + dictionary_ids=dict_ids or [], + ) +# [/DEF:job_to_response:Function] + + +# [DEF:DatasourceColumnsService:Function] +# @PURPOSE: Fetch datasource column metadata from Superset and return structured response. +# @PRE: datasource_id is a valid Superset dataset ID. +# @POST: Returns DatasourceColumnsResponse with column metadata and database dialect. +# @SIDE_EFFECT: Queries Superset API for dataset detail and database info. +def get_datasource_columns( + datasource_id: int, + env_id: str, + config_manager: ConfigManager, +) -> DatasourceColumnsResponse: + """Fetch and return column metadata for a given Superset datasource.""" + logger.info(f"[get_datasource_columns] Fetching columns for datasource {datasource_id}") + + # Find environment config + environments = config_manager.get_environments() + env_config = next((e for e in environments if e.id == env_id), None) + if not env_config: + raise ValueError(f"Superset environment '{env_id}' not found") + + # Create Superset client + client = SupersetClient(env_config) + dataset_detail = client.get_dataset_detail(datasource_id) + + # Extract database dialect + database_info = dataset_detail.get("database", {}) + dialect = None + if isinstance(database_info, dict): + try: + dialect = get_dialect_from_database(database_info) + except (ValueError, KeyError): + dialect = None + if dialect is None: + database_id = dataset_detail.get("database_id") + if database_id: + db_record = client.get_database(int(database_id)) + db_result = db_record.get("result", db_record) if isinstance(db_record, dict) else db_record + dialect = get_dialect_from_database(db_result) + else: + raise ValueError("Could not determine database dialect for this datasource") + + # Extract columns + raw_columns = dataset_detail.get("columns", []) + columns = [] + for col in raw_columns: + col_name = col.get("name") or col.get("column_name") + if not col_name: + continue + columns.append(DatasourceColumnResponse( + name=str(col_name), + type=col.get("type"), + is_physical=col.get("is_physical", True), + is_dttm=col.get("is_dttm", False), + description=col.get("description", ""), + )) + + return DatasourceColumnsResponse( + datasource_id=datasource_id, + datasource_name=dataset_detail.get("table_name"), + schema_name=dataset_detail.get("schema"), + database_dialect=dialect, + columns=columns, + ) +# [/DEF:DatasourceColumnsService:Function] + +# [/DEF:TranslateJobService:Module] diff --git a/backend/src/plugins/translate/sql_generator.py b/backend/src/plugins/translate/sql_generator.py new file mode 100644 index 00000000..760c12f1 --- /dev/null +++ b/backend/src/plugins/translate/sql_generator.py @@ -0,0 +1,319 @@ +# [DEF:SQLGenerator:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: translate, sql, generator, dialect +# @PURPOSE: Dialect-aware safe SQL generation for INSERT/UPSERT operations. +# @LAYER: Domain +# @RELATION: DEPENDS_ON -> [TranslationJob] +# @RELATION: DEPENDS_ON -> [TranslationRun] +# @PRE: Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS. +# @POST: Returns safe SQL strings for the target dialect. +# @SIDE_EFFECT: None — pure code generation. +# @RATIONALE: Dialect-aware SQL uses ON CONFLICT for PostgreSQL; plain INSERT for ClickHouse with documented limitations. +# @REJECTED: UPDATE statements — source is append-only; UPSERT covers overwrite case. +# @REJECTED: ORM-based insert bypasses Superset's SQL Lab audit trail. + +from typing import Any, Dict, List, Optional, Tuple +from ...core.logger import logger, belief_scope + +# PostgreSQL/Greenplum dialects that support ON CONFLICT +POSTGRESQL_DIALECTS = {"postgresql", "redshift", "greenplum"} +# Dialects that use backtick or no quoting +CLICKHOUSE_DIALECTS = {"clickhouse"} + + +# [DEF:_quote_identifier:Function] +# @PURPOSE: Quote an identifier per dialect rules. PostgreSQL uses double quotes; ClickHouse uses backticks. +# @PRE: identifier is a non-empty string. +# @POST: Returns safely quoted identifier. +def _quote_identifier(identifier: str, dialect: str) -> str: + """Quote a SQL identifier per dialect rules.""" + if not identifier: + return identifier + # Remove any existing quotes to avoid double-quoting + cleaned = identifier.strip().strip('"').strip('`').strip('[]') + if dialect in POSTGRESQL_DIALECTS: + return f'"{cleaned}"' + elif dialect in CLICKHOUSE_DIALECTS: + return f"`{cleaned}`" + else: + # Generic ANSI double-quote + return f'"{cleaned}"' +# [/DEF:_quote_identifier:Function] + + +# [DEF:_encode_sql_value:Function] +# @PURPOSE: Encode a Python value into a SQL-safe literal for INSERT VALUES. +# @PRE: value is a Python primitive or None. +# @POST: Returns SQL-safe string literal representation. +def _encode_sql_value(value: Any) -> str: + """Encode a Python value into a SQL-safe literal.""" + if value is None: + return "NULL" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + if isinstance(value, (int, float)): + return str(value) + # String — escape single quotes by doubling them + escaped = str(value).replace("'", "''") + return f"'{escaped}'" +# [/DEF:_encode_sql_value:Function] + + +# [DEF:_build_values_clause:Function] +# @PURPOSE: Build a VALUES clause for multiple rows. +# @PRE: columns list is non-empty; rows is a list of dicts. +# @POST: Returns SQL VALUES clause string. +def _build_values_clause(columns: List[str], rows: List[Dict[str, Any]]) -> str: + """Build VALUES (...) clause for multiple rows.""" + value_groups = [] + for row in rows: + values = [_encode_sql_value(row.get(col)) for col in columns] + value_groups.append(f"({', '.join(values)})") + return ",\n".join(value_groups) +# [/DEF:_build_values_clause:Function] + + +# [DEF:generate_insert_sql:Function] +# @PURPOSE: Generate a dialect-aware INSERT SQL statement. +# @PRE: target_table, columns are non-empty. +# @POST: Returns SQL string or raises ValueError on invalid input. +def generate_insert_sql( + target_schema: Optional[str], + target_table: str, + columns: List[str], + rows: List[Dict[str, Any]], +) -> str: + """Generate a plain INSERT SQL.""" + with belief_scope("generate_insert_sql"): + if not target_table: + raise ValueError("target_table is required for INSERT SQL generation") + if not columns: + raise ValueError("At least one column is required for INSERT SQL generation") + if not rows: + raise ValueError("At least one row is required for INSERT SQL generation") + + col_list = ", ".join(columns) + values = _build_values_clause(columns, rows) + + table_ref = target_table + if target_schema: + table_ref = f"{target_schema}.{target_table}" + + sql = f"INSERT INTO {table_ref} ({col_list})\nVALUES\n{values};" + return sql +# [/DEF:generate_insert_sql:Function] + + +# [DEF:generate_upsert_sql:Function] +# @PURPOSE: Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE. +# @PRE: dialect is postgresql-compatible. target_table, columns, key_columns are non-empty. +# @POST: Returns UPSERT SQL string or raises ValueError. +def generate_upsert_sql( + target_schema: Optional[str], + target_table: str, + columns: List[str], + key_columns: List[str], + rows: List[Dict[str, Any]], +) -> str: + """Generate INSERT ... ON CONFLICT (key_cols) DO UPDATE SET ... SQL.""" + with belief_scope("generate_upsert_sql"): + if not target_table: + raise ValueError("target_table is required for UPSERT SQL generation") + if not columns: + raise ValueError("At least one column is required for UPSERT SQL generation") + if not key_columns: + raise ValueError("key_columns are required for UPSERT SQL generation") + if not rows: + raise ValueError("At least one row is required for UPSERT SQL generation") + + col_list = ", ".join(columns) + key_list = ", ".join(key_columns) + values = _build_values_clause(columns, rows) + + table_ref = target_table + if target_schema: + table_ref = f"{target_schema}.{target_table}" + + # Build SET clause: exclude key columns from update + update_cols = [c for c in columns if c not in key_columns] + if not update_cols: + # If only key columns, use DO NOTHING + conflict_action = "DO NOTHING" + else: + set_parts = [f"{col} = EXCLUDED.{col}" for col in update_cols] + conflict_action = "DO UPDATE SET\n" + ",\n".join(set_parts) + + sql = ( + f"INSERT INTO {table_ref} ({col_list})\n" + f"VALUES\n" + f"{values}\n" + f"ON CONFLICT ({key_list}) {conflict_action};" + ) + return sql +# [/DEF:generate_upsert_sql:Function] + + +# [DEF:SQLGenerator:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Generate safe, dialect-appropriate SQL INSERT/UPSERT statements. +# @PRE: Job has target_schema, target_table, key columns configured. +# @POST: Returns generated SQL string for the target dialect. +class SQLGenerator: + + # [DEF:SQLGenerator.generate:Function] + # @PURPOSE: Generate SQL for a set of rows, detecting dialect from the job configuration. + # @PRE: dialect is a supported database dialect. columns list is non-empty. rows is non-empty. + # @POST: Returns tuple of (sql_string, statement_count). + # @SIDE_EFFECT: None — pure SQL generation. + @staticmethod + def generate( + dialect: str, + target_schema: Optional[str], + target_table: str, + columns: List[str], + rows: List[Dict[str, Any]], + key_columns: Optional[List[str]] = None, + upsert_strategy: str = "MERGE", + ) -> Tuple[str, int]: + """Generate dialect-appropriate INSERT/UPSERT SQL. + + Args: + dialect: Target database dialect (e.g. 'postgresql', 'clickhouse'). + target_schema: Optional schema name. + target_table: Target table name. + columns: List of column names to insert. + rows: List of row dicts with column values. + key_columns: Key columns for conflict resolution (UPSERT). + upsert_strategy: 'MERGE' (UPSERT), 'INSERT' (plain INSERT). + + Returns: + Tuple of (sql_string, row_count). + """ + with belief_scope("SQLGenerator.generate"): + logger.reason("Generating SQL", { + "dialect": dialect, + "schema": target_schema, + "table": target_table, + "columns": len(columns), + "rows": len(rows), + "strategy": upsert_strategy, + }) + + # Validate inputs + if not target_table: + raise ValueError("target_table is required") + if not columns: + raise ValueError("At least one column is required") + if not rows: + raise ValueError("At least one row is required") + + # Build fully qualified table reference + table_ref = target_table + if target_schema: + quoted_schema = _quote_identifier(target_schema, dialect) + quoted_table = _quote_identifier(target_table, dialect) + table_ref = f"{quoted_schema}.{quoted_table}" + else: + table_ref = _quote_identifier(target_table, dialect) + + # Quote columns per dialect + quoted_columns = [_quote_identifier(c, dialect) for c in columns] + quoted_key_columns = ( + [_quote_identifier(k, dialect) for k in key_columns] + if key_columns + else [] + ) + + # Generate SQL per dialect and strategy + use_upsert = upsert_strategy.upper() == "MERGE" and key_columns + + if dialect in POSTGRESQL_DIALECTS or dialect not in CLICKHOUSE_DIALECTS: + # PostgreSQL and other ANSI dialects: support UPSERT via ON CONFLICT + if use_upsert: + sql = generate_upsert_sql( + target_schema=None, + target_table=table_ref, + columns=quoted_columns, + key_columns=quoted_key_columns, + rows=rows, + ) + else: + sql = generate_insert_sql( + target_schema=None, + target_table=table_ref, + columns=quoted_columns, + rows=rows, + ) + elif dialect in CLICKHOUSE_DIALECTS: + # ClickHouse: plain INSERT, no ON CONFLICT support + sql = generate_insert_sql( + target_schema=None, + target_table=table_ref, + columns=quoted_columns, + rows=rows, + ) + if use_upsert: + logger.reason("ClickHouse UPSERT not supported; using plain INSERT", { + "note": "ClickHouse does not support ON CONFLICT. Use ReplacingMergeTree for dedup.", + }) + else: + # Fallback: plain INSERT + sql = generate_insert_sql( + target_schema=None, + target_table=table_ref, + columns=quoted_columns, + rows=rows, + ) + + logger.reflect("SQL generated", { + "dialect": dialect, + "row_count": len(rows), + "sql_length": len(sql), + }) + return sql, len(rows) + # [/DEF:SQLGenerator.generate:Function] + + # [DEF:SQLGenerator.generate_batch:Function] + # @PURPOSE: Generate separate INSERT statements for each row (batch-safe version). + # @PRE: Same as generate(). + # @POST: Returns list of (sql_string, row_index) tuples. + @staticmethod + def generate_batch( + dialect: str, + target_schema: Optional[str], + target_table: str, + columns: List[str], + rows: List[Dict[str, Any]], + key_columns: Optional[List[str]] = None, + upsert_strategy: str = "MERGE", + max_rows_per_statement: int = 500, + ) -> List[Tuple[str, int]]: + """Generate SQL in batches, splitting large row sets into multiple statements. + + Returns: + List of (sql_string, row_count) tuples. + """ + with belief_scope("SQLGenerator.generate_batch"): + if not rows: + return [] + + statements = [] + for i in range(0, len(rows), max_rows_per_statement): + chunk = rows[i:i + max_rows_per_statement] + sql, count = SQLGenerator.generate( + dialect=dialect, + target_schema=target_schema, + target_table=target_table, + columns=columns, + rows=chunk, + key_columns=key_columns, + upsert_strategy=upsert_strategy, + ) + statements.append((sql, count)) + + return statements + # [/DEF:SQLGenerator.generate_batch:Function] + + +# [/DEF:SQLGenerator:Class] +# [/DEF:SQLGenerator:Module] diff --git a/backend/src/plugins/translate/superset_executor.py b/backend/src/plugins/translate/superset_executor.py new file mode 100644 index 00000000..b653ca3c --- /dev/null +++ b/backend/src/plugins/translate/superset_executor.py @@ -0,0 +1,316 @@ +# [DEF:SupersetSqlLabExecutor:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: translate, superset, sqllab, execute +# @PURPOSE: Submit SQL to Superset SQL Lab API and poll execution status. +# @LAYER: Infra +# @RELATION: DEPENDS_ON -> [SupersetClient] +# @RELATION: DEPENDS_ON -> [ConfigManager] +# @PRE: Valid Superset environment configuration and authenticated client. +# @POST: SQL is submitted to Superset SQL Lab; execution reference is returned. +# @SIDE_EFFECT: Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status. +# @RATIONALE: Direct SQL Lab API submission provides audit trail and execution monitoring within Superset. +# @REJECTED: Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC. + +from typing import Any, Dict, List, Optional, Tuple +from datetime import datetime, timezone +import json +import time +import uuid + +from ...core.logger import logger, belief_scope +from ...core.config_manager import ConfigManager +from ...core.superset_client import SupersetClient + + +# [DEF:SupersetSqlLabExecutor:Class] +# @COMPLEXITY: 4 +# @PURPOSE: Submit SQL to Superset SQL Lab API with polling and status tracking. +# @PRE: Valid environment ID and ConfigManager. +# @POST: SQL submitted to Superset; execution reference recorded. +# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status. +class SupersetSqlLabExecutor: + + def __init__(self, config_manager: ConfigManager, env_id: str): + self.config_manager = config_manager + self.env_id = env_id + self._client: Optional[SupersetClient] = None + self._database_id: Optional[int] = None + + # [DEF:_get_client:Function] + # @PURPOSE: Lazy-initialize SupersetClient for the configured environment. + # @PRE: env_id must correspond to a valid environment config. + # @POST: Returns authenticated SupersetClient. + # @SIDE_EFFECT: Authenticates against Superset API. + def _get_client(self) -> SupersetClient: + with belief_scope("SupersetSqlLabExecutor._get_client"): + if self._client is not None: + return self._client + + environments = self.config_manager.get_environments() + env_config = next((e for e in environments if e.id == self.env_id), None) + if not env_config: + raise ValueError(f"Superset environment '{self.env_id}' not found") + + self._client = SupersetClient(env_config) + return self._client + # [/DEF:_get_client:Function] + + # [DEF:resolve_database_id:Function] + # @PURPOSE: Resolve the target database ID from the environment. + # @PRE: database_name or database_id should be known. + # @POST: Returns database_id integer or raises ValueError. + # @SIDE_EFFECT: Fetches databases list from Superset. + def resolve_database_id(self, database_name: Optional[str] = None) -> int: + with belief_scope("SupersetSqlLabExecutor.resolve_database_id"): + client = self._get_client() + _, databases = client.get_databases( + query={"columns": ["id", "database_name"]} + ) + if not databases: + raise ValueError("No databases found in Superset environment") + + if database_name: + for db in databases: + if db.get("database_name", "").lower() == database_name.lower(): + self._database_id = db["id"] + logger.reason("Resolved database ID by name", { + "database_name": database_name, + "database_id": self._database_id, + }) + return self._database_id + raise ValueError( + f"Database '{database_name}' not found in Superset environment" + ) + + # Default: use first database + self._database_id = databases[0]["id"] + logger.reason("Using default database", { + "database_name": databases[0].get("database_name"), + "database_id": self._database_id, + }) + return self._database_id + # [/DEF:resolve_database_id:Function] + + # [DEF:execute_sql:Function] + # @PURPOSE: Submit SQL to Superset SQL Lab and return execution tracking info. + # @PRE: sql is valid SQL string. database_id is a valid Superset DB ID. + # @POST: Returns execution result dict with query_id and status. + # @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/. + def execute_sql( + self, + sql: str, + database_id: Optional[int] = None, + run_async: bool = True, + ) -> Dict[str, Any]: + with belief_scope("SupersetSqlLabExecutor.execute_sql"): + client = self._get_client() + db_id = database_id or self._database_id + if not db_id: + db_id = self.resolve_database_id() + + logger.reason("Submitting SQL to Superset SQL Lab", { + "database_id": db_id, + "sql_length": len(sql), + "run_async": run_async, + }) + + payload = { + "database_id": db_id, + "sql": sql, + "runAsync": run_async, + "schema": None, + "tab": "translation-insert", + } + + try: + response = client.network.request( + method="POST", + endpoint="/api/v1/sqllab/execute/", + data=json.dumps(payload), + headers={"Content-Type": "application/json"}, + ) + except Exception as e: + logger.explore("SQL Lab execute failed", {"error": str(e)}) + raise ValueError(f"Superset SQL Lab execute failed: {e}") + + # Parse response + result = response if isinstance(response, dict) else {} + query_id = result.get("query_id") or result.get("id") + status = result.get("status", "unknown") + + logger.reason("SQL Lab execute response", { + "query_id": query_id, + "status": status, + }) + + return { + "query_id": query_id, + "status": status, + "raw_response": result, + "database_id": db_id, + } + # [/DEF:execute_sql:Function] + + # [DEF:poll_execution_status:Function] + # @PURPOSE: Poll Superset for SQL execution status until completion or timeout. + # @PRE: query_id is a valid Superset query ID. + # @POST: Returns final execution status dict. + # @SIDE_EFFECT: Makes HTTP GET requests to Superset. + def poll_execution_status( + self, + query_id: str, + max_polls: int = 60, + poll_interval_seconds: float = 2.0, + ) -> Dict[str, Any]: + with belief_scope("SupersetSqlLabExecutor.poll_execution_status"): + client = self._get_client() + + logger.reason("Polling SQL execution status", { + "query_id": query_id, + "max_polls": max_polls, + "interval": poll_interval_seconds, + }) + + for attempt in range(max_polls): + try: + response = client.network.request( + method="GET", + endpoint=f"/api/v1/query/{query_id}", + ) + result = response if isinstance(response, dict) else {} + status = result.get("status", "unknown") + state = result.get("state", result.get("status", "")) + + # Terminal states + if state in ("success", "finished", "completed"): + logger.reason("SQL execution completed", { + "query_id": query_id, + "attempt": attempt + 1, + "rows_affected": result.get("rows"), + }) + return { + "query_id": query_id, + "status": "success", + "state": state, + "rows_affected": result.get("rows"), + "error_message": result.get("error_message"), + "results": result.get("results"), + "completed_on": result.get("completed_on"), + } + + if state in ("failed", "error", "stopped"): + error_msg = result.get("error_message", "Unknown error") + logger.explore("SQL execution failed", { + "query_id": query_id, + "state": state, + "error": error_msg, + }) + return { + "query_id": query_id, + "status": "failed", + "state": state, + "error_message": error_msg, + "results": None, + } + + if state in ("pending", "running", "started"): + time.sleep(poll_interval_seconds) + continue + + # Unknown state — treat as still running + time.sleep(poll_interval_seconds) + + except Exception as e: + logger.explore("Polling error, retrying", { + "query_id": query_id, + "attempt": attempt + 1, + "error": str(e), + }) + time.sleep(poll_interval_seconds) + continue + + # Timeout + logger.explore("SQL execution polling timed out", { + "query_id": query_id, + "max_polls": max_polls, + }) + return { + "query_id": query_id, + "status": "timeout", + "error_message": f"Polling timed out after {max_polls} attempts", + "results": None, + } + # [/DEF:poll_execution_status:Function] + + # [DEF:execute_and_poll:Function] + # @PURPOSE: Execute SQL and wait for completion. One-shot convenience method. + # @PRE: sql is valid SQL. + # @POST: Returns final execution result. + # @SIDE_EFFECT: Makes HTTP calls to Superset API. + def execute_and_poll( + self, + sql: str, + database_id: Optional[int] = None, + max_polls: int = 60, + poll_interval_seconds: float = 2.0, + ) -> Dict[str, Any]: + with belief_scope("SupersetSqlLabExecutor.execute_and_poll"): + exec_result = self.execute_sql( + sql=sql, + database_id=database_id, + run_async=True, + ) + + query_id = exec_result.get("query_id") + if not query_id: + logger.explore("No query_id from SQL Lab execute", { + "raw_response": exec_result.get("raw_response"), + }) + return { + "status": "failed", + "error_message": "No query_id returned from SQL Lab", + "query_id": None, + } + + return self.poll_execution_status( + query_id=str(query_id), + max_polls=max_polls, + poll_interval_seconds=poll_interval_seconds, + ) + # [/DEF:execute_and_poll:Function] + + # [DEF:get_query_results:Function] + # @PURPOSE: Fetch the results of a completed query. + # @PRE: query_id is a valid Superset query ID. + # @POST: Returns query results if available. + # @SIDE_EFFECT: Makes HTTP GET to Superset. + def get_query_results(self, query_id: str) -> Dict[str, Any]: + with belief_scope("SupersetSqlLabExecutor.get_query_results"): + client = self._get_client() + try: + response = client.network.request( + method="GET", + endpoint=f"/api/v1/query/{query_id}/results", + ) + result = response if isinstance(response, dict) else {} + return { + "query_id": query_id, + "status": "success" if result.get("results") else "no_results", + "results": result.get("results"), + } + except Exception as e: + logger.explore("Failed to fetch query results", { + "query_id": query_id, + "error": str(e), + }) + return { + "query_id": query_id, + "status": "failed", + "error_message": str(e), + "results": None, + } + # [/DEF:get_query_results:Function] + + +# [/DEF:SupersetSqlLabExecutor:Class] +# [/DEF:SupersetSqlLabExecutor:Module] diff --git a/backend/src/schemas/translate.py b/backend/src/schemas/translate.py new file mode 100644 index 00000000..e6a7081b --- /dev/null +++ b/backend/src/schemas/translate.py @@ -0,0 +1,491 @@ +# [DEF:TranslateSchemas:Module] +# @COMPLEXITY: 2 +# @SEMANTICS: translate, schemas, pydantic +# @PURPOSE: Pydantic v2 schemas for translation API request/response serialization. +# @LAYER: API +# @RELATION: DEPENDS_ON -> pydantic + +from datetime import datetime +from typing import Any, Dict, List, Optional +from pydantic import BaseModel, Field +import json + + +# [DEF:TranslateJobCreate:Class] +# @PURPOSE: Schema for creating a new translation job. +class TranslateJobCreate(BaseModel): + name: str + description: Optional[str] = None + source_dialect: str = Field(..., description="Source database dialect (e.g. postgresql, clickhouse)") + target_dialect: str = Field(..., description="Target database dialect (e.g. postgresql, clickhouse)") + database_dialect: Optional[str] = Field(None, description="Detected dialect from Superset connection at save time") + source_datasource_id: Optional[str] = Field(None, description="Superset datasource ID") + source_table: Optional[str] = Field(None, description="Source table name") + target_schema: Optional[str] = Field(None, description="Target table schema") + target_table: Optional[str] = Field(None, description="Target table name") + source_key_cols: Optional[List[str]] = Field(default_factory=list, description="Source key column names") + target_key_cols: Optional[List[str]] = Field(default_factory=list, description="Target key column names") + translation_column: Optional[str] = Field(None, description="Column to translate") + context_columns: Optional[List[str]] = Field(default_factory=list, description="Context column names") + target_language: Optional[str] = Field(None, description="Target language code") + provider_id: Optional[str] = Field(None, description="LLM provider ID") + batch_size: int = Field(50, description="Records per batch") + upsert_strategy: str = Field("MERGE", description="UPSERT strategy: MERGE, INSERT, UPDATE") + dictionary_ids: Optional[List[str]] = Field(default_factory=list, description="Associated terminology dictionary IDs") +# [/DEF:TranslateJobCreate:Class] + + +# [DEF:TranslateJobUpdate:Class] +# @PURPOSE: Schema for updating an existing translation job. +class TranslateJobUpdate(BaseModel): + name: Optional[str] = None + description: Optional[str] = None + source_dialect: Optional[str] = None + target_dialect: Optional[str] = None + database_dialect: Optional[str] = None + source_datasource_id: Optional[str] = None + source_table: Optional[str] = None + target_schema: Optional[str] = None + target_table: Optional[str] = None + source_key_cols: Optional[List[str]] = None + target_key_cols: Optional[List[str]] = None + translation_column: Optional[str] = None + context_columns: Optional[List[str]] = None + target_language: Optional[str] = None + provider_id: Optional[str] = None + batch_size: Optional[int] = None + upsert_strategy: Optional[str] = None + status: Optional[str] = None + dictionary_ids: Optional[List[str]] = None +# [/DEF:TranslateJobUpdate:Class] + + +# [DEF:TranslateJobResponse:Class] +# @PURPOSE: Schema for translation job API responses. +class TranslateJobResponse(BaseModel): + id: str + name: str + description: Optional[str] = None + source_dialect: str + target_dialect: str + database_dialect: Optional[str] = None + source_datasource_id: Optional[str] = None + source_table: Optional[str] = None + target_schema: Optional[str] = None + target_table: Optional[str] = None + source_key_cols: Optional[List[str]] = None + target_key_cols: Optional[List[str]] = None + translation_column: Optional[str] = None + context_columns: Optional[List[str]] = None + target_language: Optional[str] = None + provider_id: Optional[str] = None + batch_size: int = 50 + upsert_strategy: str = "MERGE" + status: str + created_by: Optional[str] = None + created_at: datetime + updated_at: Optional[datetime] = None + dictionary_ids: Optional[List[str]] = None + + class Config: + from_attributes = True +# [/DEF:TranslateJobResponse:Class] + + +# [DEF:DatasourceColumnResponse:Class] +# @PURPOSE: Schema for datasource column metadata response. +class DatasourceColumnResponse(BaseModel): + name: str + type: Optional[str] = None + is_physical: bool = True + is_dttm: bool = False + description: Optional[str] = None + + +# [DEF:DatasourceColumnsResponse:Class] +# @PURPOSE: Schema for datasource columns endpoint response. +class DatasourceColumnsResponse(BaseModel): + datasource_id: int + datasource_name: Optional[str] = None + schema_name: Optional[str] = None + database_dialect: str + columns: List[DatasourceColumnResponse] = [] + + class Config: + from_attributes = True +# [/DEF:DatasourceColumnsResponse:Class] + + +# [DEF:DuplicateJobResponse:Class] +# @PURPOSE: Schema for duplicate job response. +class DuplicateJobResponse(BaseModel): + id: str + name: str + message: str = "Job duplicated successfully" + + class Config: + from_attributes = True +# [/DEF:DuplicateJobResponse:Class] + + +# [DEF:DictionaryCreate:Class] +# @PURPOSE: Schema for creating a new terminology dictionary. +class DictionaryCreate(BaseModel): + name: str + description: Optional[str] = None + source_dialect: str + target_dialect: str + is_active: bool = True +# [/DEF:DictionaryCreate:Class] + + +# [DEF:DictionaryImport:Class] +# @PURPOSE: Schema for importing entries into a terminology dictionary. +class DictionaryImport(BaseModel): + content: str = Field(..., description="CSV or TSV content as raw string") + delimiter: Optional[str] = Field(None, description="Detected or forced delimiter: ',' or '\\t'. Auto-detect if omitted.") + on_conflict: str = Field("overwrite", description="'overwrite' or 'keep_existing' or 'cancel'") + preview_only: bool = Field(False, description="If true, return preview without applying") +# [/DEF:DictionaryImport:Class] + + +# [DEF:DictionaryResponse:Class] +# @PURPOSE: Schema for terminology dictionary API responses. +class DictionaryResponse(BaseModel): + id: str + name: str + description: Optional[str] = None + source_dialect: str + target_dialect: str + is_active: bool + created_by: Optional[str] = None + created_at: datetime + updated_at: Optional[datetime] = None + entry_count: Optional[int] = None + + class Config: + from_attributes = True +# [/DEF:DictionaryResponse:Class] + + +# [DEF:DictionaryEntryCreate:Class] +# @PURPOSE: Schema for adding/editing a dictionary entry. +class DictionaryEntryCreate(BaseModel): + source_term: str = Field(..., description="Source term to translate") + target_term: str = Field(..., description="Target/translated term") + context_notes: Optional[str] = Field(None, description="Optional context notes") + + class Config: + from_attributes = True +# [/DEF:DictionaryEntryCreate:Class] + + +# [DEF:DictionaryEntryResponse:Class] +# @PURPOSE: Schema for dictionary entry API responses. +class DictionaryEntryResponse(BaseModel): + id: str + dictionary_id: str + source_term: str + source_term_normalized: str + target_term: str + context_notes: Optional[str] = None + created_at: datetime + updated_at: Optional[datetime] = None + + class Config: + from_attributes = True +# [/DEF:DictionaryEntryResponse:Class] + + +# [DEF:DictionaryImportResult:Class] +# @PURPOSE: Schema for dictionary import result. +class DictionaryImportResult(BaseModel): + total: int = 0 + created: int = 0 + updated: int = 0 + skipped: int = 0 + errors: List[Dict[str, Any]] = Field(default_factory=list) + preview: List[Dict[str, Any]] = Field(default_factory=list, description="Preview rows with conflict flags") +# [/DEF:DictionaryImportResult:Class] + + +# [DEF:PreviewRequest:Class] +# @PURPOSE: Schema for triggering a translation preview. +class PreviewRequest(BaseModel): + sample_size: int = Field(10, ge=1, le=100, description="Number of sample rows to preview") + prompt_template: Optional[str] = Field(None, description="Optional custom prompt template") + + +# [DEF:PreviewRowUpdate:Class] +# @PURPOSE: Schema for approving/editing/rejecting a preview row. +class PreviewRowUpdate(BaseModel): + action: str = Field(..., description="'approve', 'reject', or 'edit'") + translation: Optional[str] = Field(None, description="Edited translation (required for 'edit' action)") + feedback: Optional[str] = Field(None, description="Optional feedback/comment") +# [/DEF:PreviewRowUpdate:Class] + + +# [DEF:PreviewAcceptResponse:Class] +# @PURPOSE: Schema for preview accept response. +class PreviewAcceptResponse(BaseModel): + id: str + job_id: str + status: str + created_by: Optional[str] = None + created_at: datetime + expires_at: Optional[datetime] = None + records: List['PreviewRow'] = [] + + class Config: + from_attributes = True +# [/DEF:PreviewAcceptResponse:Class] + + +# [DEF:CostEstimate:Class] +# @PURPOSE: Schema for cost estimation in preview response. +class CostEstimate(BaseModel): + sample_size: int = 0 + sample_prompt_tokens: int = 0 + sample_output_tokens: int = 0 + sample_total_tokens: int = 0 + sample_cost: float = 0.0 + estimated_total_rows: int = 0 + estimated_tokens: int = 0 + estimated_cost: float = 0.0 +# [/DEF:CostEstimate:Class] + + +# [DEF:TermCorrectionSubmit:Class] +# @PURPOSE: Schema for submitting a term correction in a translation preview. +class TermCorrectionSubmit(BaseModel): + source_term: str + incorrect_target_term: str + corrected_target_term: str + dictionary_id: Optional[str] = Field(None, description="Target dictionary ID (language-filtered)") + origin_run_id: Optional[str] = Field(None, description="Run ID from which this correction originated") + origin_row_key: Optional[str] = Field(None, description="Row key within the run") +# [/DEF:TermCorrectionSubmit:Class] + + +# [DEF:TermCorrectionBulkSubmit:Class] +# @PURPOSE: Schema for submitting multiple term corrections atomically. +class TermCorrectionBulkSubmit(BaseModel): + corrections: List[TermCorrectionSubmit] + dictionary_id: str = Field(..., description="Target dictionary ID for all corrections") +# [/DEF:TermCorrectionBulkSubmit:Class] + + +# [DEF:CorrectionConflictResult:Class] +# @PURPOSE: Schema for reporting conflicts in correction submission. +class CorrectionConflictResult(BaseModel): + source_term: str + existing_target_term: str + submitted_target_term: str + action: str = "keep_existing" +# [/DEF:CorrectionConflictResult:Class] + + +# [DEF:CorrectionSubmitResponse:Class] +# @PURPOSE: Schema for correction submission response. +class CorrectionSubmitResponse(BaseModel): + entry_id: Optional[str] = None + action: str # "created", "updated", "conflict_detected", "skipped" + source_term: str + target_term: str + conflict: Optional[CorrectionConflictResult] = None + message: Optional[str] = None +# [/DEF:CorrectionSubmitResponse:Class] + + +# [DEF:ScheduleConfig:Class] +# @PURPOSE: Schema for configuring a recurring translation schedule. +class ScheduleConfig(BaseModel): + cron_expression: str = Field(..., description="Cron expression for scheduling (e.g. '0 2 * * *')") + timezone: str = Field("UTC", description="Timezone for the cron schedule (e.g. 'UTC', 'Europe/Moscow')") + is_active: bool = True +# [/DEF:ScheduleConfig:Class] + + +# [DEF:ScheduleResponse:Class] +# @PURPOSE: Schema for schedule API responses. +class ScheduleResponse(BaseModel): + id: str + job_id: str + cron_expression: str + timezone: str + is_active: bool + last_run_at: Optional[datetime] = None + next_run_at: Optional[datetime] = None + created_by: Optional[str] = None + created_at: datetime + updated_at: Optional[datetime] = None + + class Config: + from_attributes = True +# [/DEF:ScheduleResponse:Class] + + +# [DEF:NextExecutionResponse:Class] +# @PURPOSE: Schema for next execution preview. +class NextExecutionResponse(BaseModel): + job_id: str + cron_expression: str + timezone: str + next_executions: List[str] = [] +# [/DEF:NextExecutionResponse:Class] + + +# [DEF:TranslationRunResponse:Class] +# @PURPOSE: Schema for translation run API responses. +class TranslationRunResponse(BaseModel): + id: str + job_id: str + status: str + trigger_type: Optional[str] = None + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + error_message: Optional[str] = None + total_records: int = 0 + successful_records: int = 0 + failed_records: int = 0 + skipped_records: int = 0 + insert_status: Optional[str] = None + superset_execution_id: Optional[str] = None + config_snapshot: Optional[Dict[str, Any]] = None + key_hash: Optional[str] = None + config_hash: Optional[str] = None + dict_snapshot_hash: Optional[str] = None + created_by: Optional[str] = None + created_at: datetime + + class Config: + from_attributes = True +# [/DEF:TranslationRunResponse:Class] + + +# [DEF:RunDetailResponse:Class] +# @PURPOSE: Schema for detailed run response including records, events, and config snapshot. +class RunDetailResponse(BaseModel): + run: TranslationRunResponse + records: List[Dict[str, Any]] = Field(default_factory=list, description="Paginated translation records") + events: List[Dict[str, Any]] = Field(default_factory=list, description="Run events") + event_invariants: Optional[Dict[str, Any]] = None + batch_count: int = 0 +# [/DEF:RunDetailResponse:Class> + + +# [DEF:RunHistoryFilter:Class] +# @PURPOSE: Schema for filtering run history. +class RunHistoryFilter(BaseModel): + job_id: Optional[str] = None + status: Optional[str] = None + trigger_type: Optional[str] = None + created_by: Optional[str] = None + date_from: Optional[datetime] = None + date_to: Optional[datetime] = None + page: int = 1 + page_size: int = 20 +# [/DEF:RunHistoryFilter:Class> + + +# [DEF:RunListResponse:Class] +# @PURPOSE: Schema for paginated run list response. +class RunListResponse(BaseModel): + items: List[TranslationRunResponse] = [] + total: int = 0 + page: int = 1 + page_size: int = 20 +# [/DEF:RunListResponse:Class> + + +# [DEF:AggregatedMetricsResponse:Class] +# @PURPOSE: Schema for aggregated per-job metrics. +class AggregatedMetricsResponse(BaseModel): + job_id: str + total_runs: int = 0 + successful_runs: int = 0 + failed_runs: int = 0 + cancelled_runs: int = 0 + total_records: int = 0 + successful_records: int = 0 + failed_records: int = 0 + skipped_records: int = 0 + cumulative_tokens: Optional[int] = None + cumulative_cost: Optional[float] = None + avg_duration_ms: Optional[float] = None + last_run_at: Optional[datetime] = None + next_scheduled_run: Optional[datetime] = None +# [/DEF:AggregatedMetricsResponse:Class] + + +# [DEF:PreviewRow:Class] +# @PURPOSE: A single row in a translation preview showing original and translated content side by side. +class PreviewRow(BaseModel): + id: str + source_sql: Optional[str] = None + target_sql: Optional[str] = None + source_object_type: Optional[str] = None + source_object_id: Optional[str] = None + source_object_name: Optional[str] = None + status: str = "PENDING" + feedback: Optional[str] = None +# [/DEF:PreviewRow:Class] + + +# [DEF:TranslationPreviewResponse:Class] +# @PURPOSE: Schema for translation preview session API responses. +class TranslationPreviewResponse(BaseModel): + id: str + job_id: str + run_id: Optional[str] = None + status: str + created_by: Optional[str] = None + created_at: datetime + expires_at: Optional[datetime] = None + records: List[PreviewRow] = [] + + class Config: + from_attributes = True +# [/DEF:TranslationPreviewResponse:Class] + + +# [DEF:TranslationBatchResponse:Class] +# @PURPOSE: Schema for translation batch API responses. +class TranslationBatchResponse(BaseModel): + id: str + run_id: str + batch_index: int + status: str + total_records: int = 0 + successful_records: int = 0 + failed_records: int = 0 + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + created_at: datetime + + class Config: + from_attributes = True +# [/DEF:TranslationBatchResponse:Class] + + +# [DEF:MetricsResponse:Class] +# @PURPOSE: Schema for translation metrics API responses. +class MetricsResponse(BaseModel): + job_id: str + snapshot_date: datetime + total_jobs: int = 0 + total_runs: int = 0 + total_records: int = 0 + successful_records: int = 0 + failed_records: int = 0 + skipped_records: int = 0 + avg_duration_ms: Optional[int] = None + p50_duration_ms: Optional[int] = None + p95_duration_ms: Optional[int] = None + p99_duration_ms: Optional[int] = None + + class Config: + from_attributes = True +# [/DEF:MetricsResponse:Class] + +# [/DEF:TranslateSchemas:Module] diff --git a/backend/tests/test_translate_corrections.py b/backend/tests/test_translate_corrections.py new file mode 100644 index 00000000..4aacc283 --- /dev/null +++ b/backend/tests/test_translate_corrections.py @@ -0,0 +1,252 @@ +# [DEF:TranslateCorrectionTests:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: tests, translate, corrections, dictionary +# @PURPOSE: Tests for term correction API endpoints and DictionaryManager correction methods. +# @LAYER: Test +# @RELATION: BINDS_TO -> [DictionaryManagerModule:Module] +# @RELATION: BINDS_TO -> [TranslateRoutes:Module] +# +# @TEST_CONTRACT: CorrectionFlow -> +# { +# required_fields: {source_term, incorrect_target_term, corrected_target_term, dictionary_id}, +# invariants: [ +# "POST /api/translate/corrections creates or updates entry", +# "Conflict detection works for existing entries", +# "POST /api/translate/corrections/bulk atomic all-or-nothing", +# "Origin tracking fields are populated" +# ] +# } +# @TEST_FIXTURE: valid_dictionary -> created via DictionaryManager +# @TEST_EDGE: missing_dictionary_id -> 422 +# @TEST_EDGE: conflict_detected -> conflict response returned +# @TEST_EDGE: bulk_correction_all_succeed -> atomic commit +# @TEST_EDGE: bulk_correction_with_conflicts -> conflicts listed +# @TEST_INVARIANT: correction_flow -> VERIFIED_BY: [valid_dictionary, conflict_detected] + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +import pytest +from unittest.mock import MagicMock +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from datetime import datetime, timezone +import uuid + +from src.core.database import Base +from src.app import app +from src.dependencies import get_current_user, get_config_manager, has_permission, get_db +from src.plugins.translate.dictionary import DictionaryManager +from src.models.translate import TerminologyDictionary, DictionaryEntry + +# In-memory SQLite +SQLALCHEMY_DATABASE_URL = "sqlite:///:memory:" +engine = create_engine(SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}) +TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) +Base.metadata.create_all(bind=engine) + +mock_user = MagicMock() +mock_user.username = "testuser" +mock_user.roles = [] +admin_role = MagicMock() +admin_role.name = "Admin" +mock_user.roles.append(admin_role) + + +@pytest.fixture +def db_session(): + connection = engine.connect() + transaction = connection.begin() + session = TestingSessionLocal(bind=connection) + yield session + session.close() + transaction.rollback() + connection.close() + + +@pytest.fixture +def mock_api_deps(): + config_manager = MagicMock() + config_manager.get_environments.return_value = [] + + connection = engine.connect() + transaction = connection.begin() + session = TestingSessionLocal(bind=connection) + + def override_get_db(): + try: + yield session + finally: + pass + + overrides = { + get_current_user: lambda: mock_user, + get_config_manager: lambda: config_manager, + get_db: override_get_db, + } + for dep, override in overrides.items(): + app.dependency_overrides[dep] = override + + yield {"config_manager": config_manager, "session": session} + + app.dependency_overrides.clear() + session.close() + transaction.rollback() + connection.close() + + +@pytest.fixture +def client(mock_api_deps): + return TestClient(app) + + +# [DEF:test_submit_correction_creates_entry:Function] +# @PURPOSE: Verify that a correction creates a new dictionary entry. +def test_submit_correction_creates_entry(db_session): + """Test that submitting a correction creates a new entry.""" + dict_obj = DictionaryManager.create_dictionary( + db_session, name="Test Dict", source_dialect="en", target_dialect="ru", + created_by="testuser", + ) + + result = DictionaryManager.submit_correction( + db_session, + dict_id=dict_obj.id, + source_term="hello", + incorrect_target_term="privet", + corrected_target_term="zdravstvuyte", + origin_run_id="run-123", + origin_row_key="row-1", + origin_user_id="testuser", + ) + + assert result["action"] == "created" + assert result["entry_id"] is not None + + entries, total = DictionaryManager.list_entries(db_session, dict_obj.id) + assert total == 1 + assert entries[0].source_term == "hello" + assert entries[0].target_term == "zdravstvuyte" + assert entries[0].origin_run_id == "run-123" + assert entries[0].origin_row_key == "row-1" + assert entries[0].origin_user_id == "testuser" +# [/DEF:test_submit_correction_creates_entry:Function] + + +# [DEF:test_submit_correction_conflict_detected:Function] +# @PURPOSE: Verify conflict detection when entry already exists. +def test_submit_correction_conflict_detected(db_session): + """Test that conflict is detected when entry already exists.""" + dict_obj = DictionaryManager.create_dictionary( + db_session, name="Dict", source_dialect="en", target_dialect="ru", + ) + + # Create initial entry + DictionaryManager.add_entry(db_session, dict_obj.id, "hello", "privet") + + # Submit correction with conflict + result = DictionaryManager.submit_correction( + db_session, + dict_id=dict_obj.id, + source_term="hello", + incorrect_target_term="privet", + corrected_target_term="zdravstvuyte", + on_conflict="keep_existing", + ) + + assert result["action"] == "conflict_detected" + assert result["conflict"] is not None + assert result["conflict"]["existing_target_term"] == "privet" +# [/DEF:test_submit_correction_conflict_detected:Function] + + +# [DEF:test_submit_correction_overwrite:Function] +# @PURPOSE: Verify that correction overwrites existing entry. +def test_submit_correction_overwrite(db_session): + """Test that correction overwrites existing entry.""" + dict_obj = DictionaryManager.create_dictionary( + db_session, name="Dict", source_dialect="en", target_dialect="ru", + ) + + DictionaryManager.add_entry(db_session, dict_obj.id, "hello", "privet") + + result = DictionaryManager.submit_correction( + db_session, + dict_id=dict_obj.id, + source_term="hello", + incorrect_target_term="privet", + corrected_target_term="hi_there", + on_conflict="overwrite", + ) + + assert result["action"] == "updated" + entries, _ = DictionaryManager.list_entries(db_session, dict_obj.id) + assert entries[0].target_term == "hi_there" +# [/DEF:test_submit_correction_overwrite:Function] + + +# [DEF:test_bulk_corrections_atomic:Function] +# @PURPOSE: Verify bulk corrections are applied atomically. +def test_bulk_corrections_atomic(db_session): + """Test that bulk corrections are applied atomically.""" + dict_obj = DictionaryManager.create_dictionary( + db_session, name="Bulk Dict", source_dialect="en", target_dialect="ru", + ) + + corrections = [ + {"source_term": "cat", "incorrect_target_term": "kot", "corrected_target_term": "koshka"}, + {"source_term": "dog", "incorrect_target_term": "sobaka", "corrected_target_term": "pyos"}, + ] + + result = DictionaryManager.submit_bulk_corrections( + db_session, + dict_id=dict_obj.id, + corrections=corrections, + origin_user_id="testuser", + ) + + assert result["status"] == "completed" + assert result["applied"] == 2 + assert len(result["conflicts"]) == 0 + + entries, total = DictionaryManager.list_entries(db_session, dict_obj.id) + assert total == 2 +# [/DEF:test_bulk_corrections_atomic:Function] + + +# [DEF:test_api_correction_missing_dict:Function] +# @PURPOSE: Verify POST corrections without dictionary_id returns 422. +def test_api_correction_missing_dict(client): + """Test correction without dict_id returns 422.""" + response = client.post("/api/translate/corrections", json={ + "source_term": "hello", + "incorrect_target_term": "privet", + "corrected_target_term": "zdravstvuyte", + }) + assert response.status_code == 422 +# [/DEF:test_api_correction_missing_dict:Function] + + +# [DEF:test_api_bulk_corrections:Function] +# @PURPOSE: Verify POST /corrections/bulk works. +def test_api_bulk_corrections(client, mock_api_deps): + """Test bulk corrections endpoint.""" + session = mock_api_deps["session"] + dict_obj = DictionaryManager.create_dictionary( + session, name="API Dict", source_dialect="en", target_dialect="ru", + ) + + response = client.post("/api/translate/corrections/bulk", json={ + "dictionary_id": dict_obj.id, + "corrections": [ + {"source_term": "hello", "incorrect_target_term": "privet", "corrected_target_term": "hi"}, + ], + }) + assert response.status_code == 200 + data = response.json() + assert data["status"] == "completed" +# [/DEF:test_api_bulk_corrections:Function] +# [/DEF:TranslateCorrectionTests:Module] diff --git a/backend/tests/test_translate_history.py b/backend/tests/test_translate_history.py new file mode 100644 index 00000000..f4767113 --- /dev/null +++ b/backend/tests/test_translate_history.py @@ -0,0 +1,300 @@ +# [DEF:TranslateHistoryTests:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: tests, translate, history, metrics +# @PURPOSE: Tests for run history list/detail endpoints and metrics aggregation. +# @LAYER: Test +# @RELATION: BINDS_TO -> [TranslateRoutes:Module] +# @RELATION: BINDS_TO -> [TranslationMetrics:Module] +# +# @TEST_CONTRACT: HistoryFlow -> +# { +# invariants: [ +# "GET /api/translate/runs returns paginated list", +# "GET /api/translate/runs supports filters (job_id, status, trigger_type)", +# "GET /api/translate/runs/{id}/detail returns config_snapshot, records, events", +# "GET /api/translate/jobs/{id}/metrics returns aggregated data", +# "Metrics include run counts, record counts, duration" +# ] +# } +# @TEST_FIXTURE: completed_run -> run with COMPLETED status +# @TEST_EDGE: filter_by_job_id -> returns only that job's runs +# @TEST_EDGE: filter_by_status -> returns only matching status +# @TEST_EDGE: metrics_empty_job -> returns zero counts +# @TEST_INVARIANT: history_list_and_detail -> VERIFIED_BY: [completed_run, filter_by_status] + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +import pytest +from unittest.mock import MagicMock, patch +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from datetime import datetime, timezone +import uuid +import json + +from src.core.database import Base +from src.app import app +from src.dependencies import get_current_user, get_config_manager, has_permission, get_db +from src.plugins.translate.metrics import TranslationMetrics +from src.models.translate import TranslationJob, TranslationRun, TranslationEvent, MetricSnapshot + +# In-memory SQLite +SQLALCHEMY_DATABASE_URL = "sqlite:///:memory:" +engine = create_engine(SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}) +TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) +Base.metadata.create_all(bind=engine) + +mock_user = MagicMock() +mock_user.username = "testuser" +mock_user.roles = [] +admin_role = MagicMock() +admin_role.name = "Admin" +mock_user.roles.append(admin_role) + + +@pytest.fixture +def db_session(): + connection = engine.connect() + transaction = connection.begin() + session = TestingSessionLocal(bind=connection) + yield session + session.close() + transaction.rollback() + connection.close() + + +@pytest.fixture +def mock_api_deps(): + config_manager = MagicMock() + config_manager.get_environments.return_value = [] + + connection = engine.connect() + transaction = connection.begin() + session = TestingSessionLocal(bind=connection) + + def override_get_db(): + try: + yield session + finally: + pass + + overrides = { + get_current_user: lambda: mock_user, + get_config_manager: lambda: config_manager, + get_db: override_get_db, + } + for dep, override in overrides.items(): + app.dependency_overrides[dep] = override + + yield {"config_manager": config_manager, "session": session} + + app.dependency_overrides.clear() + session.close() + transaction.rollback() + connection.close() + + +@pytest.fixture +def client(mock_api_deps): + return TestClient(app) + + +def _create_test_job(db_session) -> TranslationJob: + job = TranslationJob( + id=str(uuid.uuid4()), + name="Test Job", + source_dialect="postgresql", + target_dialect="clickhouse", + status="ACTIVE", + ) + db_session.add(job) + db_session.flush() + return job + + +def _create_test_run(db_session, job_id: str, status: str = "COMPLETED", trigger: str = "manual") -> TranslationRun: + run = TranslationRun( + id=str(uuid.uuid4()), + job_id=job_id, + status=status, + trigger_type=trigger, + total_records=100, + successful_records=80, + failed_records=10, + skipped_records=10, + config_snapshot={"test": True}, + config_hash="abc123", + dict_snapshot_hash="def456", + key_hash="ghi789", + created_by="testuser", + created_at=datetime.now(timezone.utc), + ) + db_session.add(run) + db_session.flush() + return run + + +# [DEF:test_list_runs_empty:Function] +# @PURPOSE: Verify runs list returns empty result initially. +def test_list_runs_empty(client): + """Test runs list initially returns empty.""" + response = client.get("/api/translate/runs") + assert response.status_code == 200 + data = response.json() + assert data["total"] == 0 + assert data["items"] == [] +# [/DEF:test_list_runs_empty:Function] + + +# [DEF:test_list_runs_with_data:Function] +# @PURPOSE: Verify runs list returns data when runs exist. +def test_list_runs_with_data(client, mock_api_deps): + """Test runs list with existing runs.""" + session = mock_api_deps["session"] + job = _create_test_job(session) + _create_test_run(session, job.id) + session.commit() + + response = client.get("/api/translate/runs") + assert response.status_code == 200 + data = response.json() + assert data["total"] >= 1 + assert len(data["items"]) >= 1 + assert data["items"][0]["job_id"] == job.id +# [/DEF:test_list_runs_with_data:Function] + + +# [DEF:test_list_runs_filter_job_id:Function] +# @PURPOSE: Verify filtering runs by job_id works. +def test_list_runs_filter_job_id(client, mock_api_deps): + """Test filtering runs by job_id.""" + session = mock_api_deps["session"] + job1 = _create_test_job(session) + job2 = _create_test_job(session) + _create_test_run(session, job1.id) + _create_test_run(session, job2.id) + session.commit() + + response = client.get(f"/api/translate/runs?job_id={job1.id}") + assert response.status_code == 200 + data = response.json() + assert data["total"] >= 1 + for item in data["items"]: + assert item["job_id"] == job1.id +# [/DEF:test_list_runs_filter_job_id:Function] + + +# [DEF:test_list_runs_filter_status:Function] +# @PURPOSE: Verify filtering runs by status works. +def test_list_runs_filter_status(client, mock_api_deps): + """Test filtering runs by status.""" + session = mock_api_deps["session"] + job = _create_test_job(session) + _create_test_run(session, job.id, status="COMPLETED") + _create_test_run(session, job.id, status="FAILED") + session.commit() + + response = client.get(f"/api/translate/runs?status=FAILED") + assert response.status_code == 200 + data = response.json() + for item in data["items"]: + assert item["status"] == "FAILED" +# [/DEF:test_list_runs_filter_status:Function] + + +# [DEF:test_get_run_detail:Function] +# @PURPOSE: Verify run detail returns config_snapshot, records, events. +def test_get_run_detail(client, mock_api_deps): + """Test run detail endpoint.""" + session = mock_api_deps["session"] + job = _create_test_job(session) + run = _create_test_run(session, job.id) + + # Add an event + event = TranslationEvent( + id=str(uuid.uuid4()), + job_id=job.id, + run_id=run.id, + event_type="RUN_STARTED", + event_data={}, + created_by="testuser", + created_at=datetime.now(timezone.utc), + ) + session.add(event) + session.commit() + + response = client.get(f"/api/translate/runs/{run.id}/detail") + assert response.status_code == 200 + data = response.json() + assert data["id"] == run.id + assert data["config_snapshot"] == {"test": True} + assert data["config_hash"] == "abc123" + assert len(data["events"]) >= 1 + assert data["events"][0]["event_type"] == "RUN_STARTED" +# [/DEF:test_get_run_detail:Function] + + +# [DEF:test_get_job_metrics:Function] +# @PURPOSE: Verify metrics endpoint returns aggregated data. +def test_get_job_metrics(client, mock_api_deps): + """Test job metrics endpoint.""" + session = mock_api_deps["session"] + job = _create_test_job(session) + _create_test_run(session, job.id, status="COMPLETED") + _create_test_run(session, job.id, status="FAILED") + session.commit() + + response = client.get(f"/api/translate/jobs/{job.id}/metrics") + assert response.status_code == 200 + data = response.json() + assert data["job_id"] == job.id + assert data["total_runs"] >= 2 + assert data["successful_runs"] >= 1 + assert data["failed_runs"] >= 1 +# [/DEF:test_get_job_metrics:Function] + + +# [DEF:test_get_all_metrics:Function] +# @PURPOSE: Verify global metrics endpoint returns data. +def test_get_all_metrics(client, mock_api_deps): + """Test global metrics endpoint.""" + session = mock_api_deps["session"] + job = _create_test_job(session) + _create_test_run(session, job.id) + session.commit() + + response = client.get("/api/translate/metrics") + assert response.status_code == 200 + data = response.json() + assert isinstance(data, list) + assert len(data) >= 1 +# [/DEF:test_get_all_metrics:Function] + + +# [DEF:test_metrics_empty_job:Function] +# @PURPOSE: Verify metrics for job with no runs returns zeros. +def test_metrics_empty_job(client, mock_api_deps): + """Test metrics for job with no runs.""" + session = mock_api_deps["session"] + job = _create_test_job(session) + session.commit() + + response = client.get(f"/api/translate/jobs/{job.id}/metrics") + assert response.status_code == 200 + data = response.json() + assert data["total_runs"] == 0 +# [/DEF:test_metrics_empty_job:Function] + + +# [DEF:test_run_detail_not_found:Function] +# @PURPOSE: Verify 404 on non-existent run detail. +def test_run_detail_not_found(client): + """Test run detail returns 404 for non-existent run.""" + response = client.get("/api/translate/runs/non-existent/detail") + assert response.status_code == 404 +# [/DEF:test_run_detail_not_found:Function] +# [/DEF:TranslateHistoryTests:Module] diff --git a/backend/tests/test_translate_jobs.py b/backend/tests/test_translate_jobs.py new file mode 100644 index 00000000..8c4f37d6 --- /dev/null +++ b/backend/tests/test_translate_jobs.py @@ -0,0 +1,569 @@ +# [DEF:TranslateJobTests:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: tests, translate, jobs, crud, validation +# @PURPOSE: Tests for translation job CRUD endpoints and service layer with column validation. +# @LAYER: Test +# @RELATION: BINDS_TO -> [TranslateRoutes:Module] +# @RELATION: BINDS_TO -> [TranslateJobService:Module] +# +# @TEST_CONTRACT: TranslateJobCRUD -> +# { +# required_fields: {name: str, source_dialect: str, target_dialect: str}, +# optional_fields: {description, source_datasource_id, translation_column, ...}, +# invariants: [ +# "POST /api/translate/jobs returns 201 with valid config", +# "GET /api/translate/jobs returns job list", +# "GET /api/translate/jobs/{id} returns single job", +# "PUT /api/translate/jobs/{id} updates and returns job", +# "DELETE /api/translate/jobs/{id} returns 204", +# "POST /api/translate/jobs/{id}/duplicate returns new job copy" +# ] +# } +# @TEST_FIXTURE: valid_job_payload -> {name: "Test Job", source_dialect: "postgresql", target_dialect: "clickhouse"} +# @TEST_EDGE: missing_translation_column -> 422 when datasource set but no translation column +# @TEST_EDGE: virtual_key_column_warning -> warning emitted when virtual column used as key +# @TEST_EDGE: invalid_dialect_rejected -> error on unsupported dialect +# @TEST_INVARIANT: valid_CRUD_flow -> VERIFIED_BY: [valid_job_payload, missing_translation_column] + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +import pytest +from unittest.mock import MagicMock, patch +from datetime import datetime, timezone +from fastapi import status +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from typing import Any, Dict, List, Optional, Tuple +import json +import uuid + +from src.core.database import Base +from src.app import app +from src.dependencies import ( + get_current_user, + get_config_manager, + has_permission, +) +from src.core.config_models import Environment + + +# [DEF:valid_job_payload:Variable] +# @PURPOSE: Standard valid payload for creating a translation job. +valid_job_payload = { + "name": "Test Translation Job", + "description": "A test job for unit tests", + "source_dialect": "postgresql", + "target_dialect": "clickhouse", + "source_key_cols": ["id"], + "target_key_cols": ["id"], + "translation_column": "name", + "context_columns": ["description"], + "target_language": "ru", + "batch_size": 100, + "upsert_strategy": "MERGE", + "dictionary_ids": [], +} +# [/DEF:valid_job_payload:Variable] + + +# Mock user +mock_user = MagicMock() +mock_user.username = "testuser" +mock_user.roles = [] +admin_role = MagicMock() +admin_role.name = "Admin" +mock_user.roles.append(admin_role) + + +# In-memory SQLite database for service tests +SQLALCHEMY_DATABASE_URL = "sqlite:///:memory:" +engine = create_engine(SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}) +TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + +# Create DB tables once +Base.metadata.create_all(bind=engine) + + +# [DEF:MockConfigManager:Class] +# @PURPOSE: Mock ConfigManager for service tests that returns a test environment. +class MockConfigManager: + def get_environments(self): + return [ + Environment( + id="test_env", + name="Test Environment", + url="http://superset:8088", + username="admin", + password="admin", + ) + ] + + def get_config(self): + return MagicMock() +# [/DEF:MockConfigManager:Class] + + +# [DEF:db_session:Function] +# @PURPOSE: Create a fresh DB session with transaction rollback for each test. +@pytest.fixture +def db_session(): + connection = engine.connect() + transaction = connection.begin() + session = TestingSessionLocal(bind=connection) + + yield session + + session.close() + transaction.rollback() + connection.close() +# [/DEF:db_session:Function] + + +# [DEF:mock_api_deps:Function] +# @PURPOSE: Override FastAPI dependencies (including get_db with in-memory SQLite) for API route tests. +# @RATIONALE: Uses the same in-memory SQLite engine as service tests to avoid real PostgreSQL dependency. +# @REJECTED: Mocking get_db with MagicMock — the route handlers need a real session for ORM queries. +@pytest.fixture +def mock_api_deps(): + from src.core.database import get_db + + config_manager = MagicMock() + config_manager.get_environments.return_value = [] + + # Create in-memory SQLite session for API tests (same engine as service tests) + connection = engine.connect() + transaction = connection.begin() + session = TestingSessionLocal(bind=connection) + + def override_get_db(): + try: + yield session + finally: + pass # Session lifecycle managed by fixture teardown + + overrides = { + get_current_user: lambda: mock_user, + get_config_manager: lambda: config_manager, + get_db: override_get_db, + } + + # Apply all overrides + for dep, override in overrides.items(): + app.dependency_overrides[dep] = override + + yield {"config_manager": config_manager} + + app.dependency_overrides.clear() + session.close() + transaction.rollback() + connection.close() +# [/DEF:mock_api_deps:Function] + + +# [DEF:client:Function] +# @PURPOSE: FastAPI TestClient for API route tests. +@pytest.fixture +def client(mock_api_deps): + return TestClient(app) +# [/DEF:client:Function] + + +# ============================================================ +# Service Layer Tests +# ============================================================ + + +# [DEF:test_create_job_valid:Function] +# @PURPOSE: Verify that a valid job payload creates a job successfully. +def test_create_job_valid(db_session): + """Test creating a valid translation job.""" + from src.plugins.translate.service import TranslateJobService, job_to_response + from src.schemas.translate import TranslateJobCreate + + config_mgr = MockConfigManager() + service = TranslateJobService(db_session, config_mgr, "test_user") + + payload = TranslateJobCreate(**valid_job_payload) + job = service.create_job(payload) + + assert job.id is not None + assert job.name == "Test Translation Job" + assert job.source_dialect == "postgresql" + assert job.target_dialect == "clickhouse" + assert job.translation_column == "name" + assert job.source_key_cols == ["id"] + assert job.target_key_cols == ["id"] + assert job.context_columns == ["description"] + assert job.batch_size == 100 + assert job.upsert_strategy == "MERGE" + assert job.target_language == "ru" + assert job.status == "DRAFT" + assert job.created_by == "test_user" + + # Verify response serialization + response = job_to_response(job, []) + assert response.name == "Test Translation Job" + assert response.source_key_cols == ["id"] +# [/DEF:test_create_job_valid:Function] + + +# [DEF:test_create_job_missing_translation_column:Function] +# @PURPOSE: Verify that creating a job with datasource but no translation column raises ValueError. +def test_create_job_missing_translation_column(db_session): + """Test that a datasource without a translation column is rejected.""" + from src.plugins.translate.service import TranslateJobService + from src.schemas.translate import TranslateJobCreate + + config_mgr = MockConfigManager() + service = TranslateJobService(db_session, config_mgr, "test_user") + + payload = TranslateJobCreate( + name="Bad Job", + source_dialect="postgresql", + target_dialect="clickhouse", + source_datasource_id="42", + translation_column=None, + ) + + with pytest.raises(ValueError, match="translation column is required"): + service.create_job(payload) +# [/DEF:test_create_job_missing_translation_column:Function] + + +# [DEF:test_create_job_invalid_upsert_strategy:Function] +# @PURPOSE: Verify that an invalid upsert strategy is rejected. +def test_create_job_invalid_upsert_strategy(db_session): + """Test that an invalid upsert strategy raises ValueError.""" + from src.plugins.translate.service import TranslateJobService + from src.schemas.translate import TranslateJobCreate + + config_mgr = MockConfigManager() + service = TranslateJobService(db_session, config_mgr, "test_user") + + payload = TranslateJobCreate( + name="Bad Strategy Job", + source_dialect="postgresql", + target_dialect="clickhouse", + upsert_strategy="INVALID", + ) + + with pytest.raises(ValueError, match="Invalid upsert_strategy"): + service.create_job(payload) +# [/DEF:test_create_job_invalid_upsert_strategy:Function] + + +# [DEF:test_get_job:Function] +# @PURPOSE: Verify that a job can be retrieved by ID. +def test_get_job(db_session): + """Test retrieving a translation job by ID.""" + from src.plugins.translate.service import TranslateJobService + from src.schemas.translate import TranslateJobCreate + + config_mgr = MockConfigManager() + service = TranslateJobService(db_session, config_mgr, "test_user") + + payload = TranslateJobCreate(**valid_job_payload) + created = service.create_job(payload) + + fetched = service.get_job(created.id) + assert fetched.id == created.id + assert fetched.name == "Test Translation Job" +# [/DEF:test_get_job:Function] + + +# [DEF:test_get_job_not_found:Function] +# @PURPOSE: Verify that getting a non-existent job raises ValueError. +def test_get_job_not_found(db_session): + """Test that a non-existent job raises ValueError.""" + from src.plugins.translate.service import TranslateJobService + + config_mgr = MockConfigManager() + service = TranslateJobService(db_session, config_mgr, "test_user") + + with pytest.raises(ValueError, match="not found"): + service.get_job("non-existent-id") +# [/DEF:test_get_job_not_found:Function] + + +# [DEF:test_list_jobs:Function] +# @PURPOSE: Verify that listing jobs returns all created jobs. +def test_list_jobs(db_session): + """Test listing translation jobs.""" + from src.plugins.translate.service import TranslateJobService + from src.schemas.translate import TranslateJobCreate + + config_mgr = MockConfigManager() + service = TranslateJobService(db_session, config_mgr, "test_user") + + p1 = TranslateJobCreate(name="Job 1", source_dialect="postgresql", target_dialect="clickhouse") + p2 = TranslateJobCreate(name="Job 2", source_dialect="mysql", target_dialect="postgresql") + service.create_job(p1) + service.create_job(p2) + + total, jobs = service.list_jobs() + assert total == 2 + assert len(jobs) == 2 +# [/DEF:test_list_jobs:Function] + + +# [DEF:test_list_jobs_with_status_filter:Function] +# @PURPOSE: Verify that listing jobs with a status filter works. +def test_list_jobs_with_status_filter(db_session): + """Test listing jobs filtered by status.""" + from src.plugins.translate.service import TranslateJobService + from src.schemas.translate import TranslateJobCreate, TranslateJobUpdate + + config_mgr = MockConfigManager() + service = TranslateJobService(db_session, config_mgr, "test_user") + + p1 = TranslateJobCreate(name="Draft Job", source_dialect="pg", target_dialect="ch") + p2 = TranslateJobCreate(name="Ready Job", source_dialect="pg", target_dialect="ch") + service.create_job(p1) + job2 = service.create_job(p2) + + service.update_job(job2.id, TranslateJobUpdate(status="READY")) + + total, jobs = service.list_jobs(status_filter="READY") + assert total == 1 + assert jobs[0].name == "Ready Job" +# [/DEF:test_list_jobs_with_status_filter:Function] + + +# [DEF:test_update_job:Function] +# @PURPOSE: Verify that a job can be updated. +def test_update_job(db_session): + """Test updating a translation job.""" + from src.plugins.translate.service import TranslateJobService + from src.schemas.translate import TranslateJobCreate, TranslateJobUpdate + + config_mgr = MockConfigManager() + service = TranslateJobService(db_session, config_mgr, "test_user") + + payload = TranslateJobCreate(**valid_job_payload) + job = service.create_job(payload) + + update = TranslateJobUpdate( + name="Updated Job", + description="Updated description", + batch_size=200, + ) + updated = service.update_job(job.id, update) + + assert updated.name == "Updated Job" + assert updated.description == "Updated description" + assert updated.batch_size == 200 +# [/DEF:test_update_job:Function] + + +# [DEF:test_delete_job:Function] +# @PURPOSE: Verify that a job can be deleted. +def test_delete_job(db_session): + """Test deleting a translation job.""" + from src.plugins.translate.service import TranslateJobService + from src.schemas.translate import TranslateJobCreate + + config_mgr = MockConfigManager() + service = TranslateJobService(db_session, config_mgr, "test_user") + + payload = TranslateJobCreate(**valid_job_payload) + job = service.create_job(payload) + + service.delete_job(job.id) + + with pytest.raises(ValueError, match="not found"): + service.get_job(job.id) +# [/DEF:test_delete_job:Function] + + +# [DEF:test_duplicate_job:Function] +# @PURPOSE: Verify that a job can be duplicated. +def test_duplicate_job(db_session): + """Test duplicating a translation job.""" + from src.plugins.translate.service import TranslateJobService + from src.schemas.translate import TranslateJobCreate + + config_mgr = MockConfigManager() + service = TranslateJobService(db_session, config_mgr, "test_user") + + payload = TranslateJobCreate(**valid_job_payload) + original = service.create_job(payload) + + duplicate = service.duplicate_job(original.id) + + assert duplicate.id != original.id + assert duplicate.name == f"{original.name} (Copy)" + assert duplicate.source_dialect == original.source_dialect + assert duplicate.target_dialect == original.target_dialect + assert duplicate.translation_column == original.translation_column + assert duplicate.source_key_cols == original.source_key_cols + assert duplicate.status == "DRAFT" +# [/DEF:test_duplicate_job:Function] + + +# [DEF:test_duplicate_job_custom_name:Function] +# @PURPOSE: Verify that a job can be duplicated with a custom name. +def test_duplicate_job_custom_name(db_session): + """Test duplicating a job with a custom name.""" + from src.plugins.translate.service import TranslateJobService + from src.schemas.translate import TranslateJobCreate + + config_mgr = MockConfigManager() + service = TranslateJobService(db_session, config_mgr, "test_user") + + payload = TranslateJobCreate(**valid_job_payload) + original = service.create_job(payload) + + duplicate = service.duplicate_job(original.id, new_name="Custom Copy Name") + assert duplicate.name == "Custom Copy Name" +# [/DEF:test_duplicate_job_custom_name:Function] + + +# [DEF:test_detect_virtual_columns:Function] +# @PURPOSE: Verify virtual column detection from column metadata. +def test_detect_virtual_columns(): + """Test that virtual columns are correctly identified.""" + from src.plugins.translate.service import detect_virtual_columns + + columns = [ + {"name": "id", "is_physical": True}, + {"name": "name", "is_physical": True}, + {"name": "virtual_col", "is_physical": False}, + ] + + virtuals = detect_virtual_columns(columns) + assert "virtual_col" in virtuals + assert "id" not in virtuals + assert "name" not in virtuals +# [/DEF:test_detect_virtual_columns:Function] + + +# [DEF:test_get_dialect_from_database:Function] +# @PURPOSE: Verify dialect extraction from Superset database records. +def test_get_dialect_from_database(): + """Test dialect extraction from Superset database records.""" + from src.plugins.translate.service import get_dialect_from_database + + dialect = get_dialect_from_database({"backend": "postgresql"}) + assert dialect == "postgresql" + + dialect = get_dialect_from_database({"engine": "mysql"}) + assert dialect == "mysql" + + dialect = get_dialect_from_database({"backend": "clickhouse", "engine": "mysql"}) + assert dialect == "clickhouse" +# [/DEF:test_get_dialect_from_database:Function] + + +# [DEF:test_get_dialect_from_database_unsupported:Function] +# @PURPOSE: Verify that unsupported dialects raise ValueError. +def test_get_dialect_from_database_unsupported(): + """Test that unsupported dialects raise ValueError.""" + from src.plugins.translate.service import get_dialect_from_database + + with pytest.raises(ValueError, match="Unsupported database dialect"): + get_dialect_from_database({"backend": "mongodb"}) + + with pytest.raises(ValueError, match="Could not determine"): + get_dialect_from_database({}) +# [/DEF:test_get_dialect_from_database_unsupported:Function] + + +# ============================================================ +# API Route Tests +# ============================================================ + + +# [DEF:test_api_create_job:Function] +# @PURPOSE: Verify POST /api/translate/jobs returns 201 with valid payload. +def test_api_create_job(client): + """Test POST /api/translate/jobs returns 201.""" + response = client.post("/api/translate/jobs", json=valid_job_payload) + + # Note: This may return 422 because the mock ConfigManager returns no real environments + # but the route requires get_config_manager. The service is tested directly above. + assert response.status_code in (201, 422, 500) + if response.status_code == 201: + data = response.json() + assert data["name"] == "Test Translation Job" + assert "id" in data +# [/DEF:test_api_create_job:Function] + + +# [DEF:test_api_list_jobs:Function] +# @PURPOSE: Verify GET /api/translate/jobs returns 200. +def test_api_list_jobs(client): + """Test GET /api/translate/jobs returns list.""" + response = client.get("/api/translate/jobs") + assert response.status_code == 200 + assert isinstance(response.json(), list) +# [/DEF:test_api_list_jobs:Function] + + +# [DEF:test_api_get_job_not_found:Function] +# @PURPOSE: Verify GET non-existent job returns 404. +def test_api_get_job_not_found(client): + """Test GET non-existent job returns 404.""" + response = client.get("/api/translate/jobs/non-existent-id") + assert response.status_code == 404 +# [/DEF:test_api_get_job_not_found:Function] + + +# [DEF:test_api_delete_job_not_found:Function] +# @PURPOSE: Verify DELETE non-existent job returns 404. +def test_api_delete_job_not_found(client): + """Test DELETE non-existent job returns 404.""" + response = client.delete("/api/translate/jobs/non-existent-id") + assert response.status_code == 404 +# [/DEF:test_api_delete_job_not_found:Function] + + +# [DEF:test_api_duplicate_job_not_found:Function] +# @PURPOSE: Verify duplicating a non-existent job returns 404. +def test_api_duplicate_job_not_found(client): + """Test duplicating a non-existent job returns 404.""" + response = client.post("/api/translate/jobs/non-existent-id/duplicate") + assert response.status_code == 404 +# [/DEF:test_api_duplicate_job_not_found:Function] + + +# [DEF:test_api_create_job_422_missing_name:Function] +# @PURPOSE: Verify POST with missing required fields returns 422. +def test_api_create_job_422_missing_name(client): + """Test POST with missing required fields returns 422.""" + response = client.post("/api/translate/jobs", json={"source_dialect": "pg"}) + assert response.status_code == 422 +# [/DEF:test_api_create_job_422_missing_name:Function] + + +# [DEF:test_api_datasource_columns_missing_env:Function] +# @PURPOSE: Verify datasource columns endpoint without env_id returns 422. +def test_api_datasource_columns_missing_env(client): + """Test datasource columns endpoint without env_id returns 422.""" + response = client.get("/api/translate/datasources/42/columns") + assert response.status_code == 422 +# [/DEF:test_api_datasource_columns_missing_env:Function] + + +# [DEF:test_api_datasource_columns_bad_env:Function] +# @PURPOSE: Verify datasource columns with unknown env returns 400. +def test_api_datasource_columns_bad_env(client): + """Test datasource columns with unknown env returns 400.""" + response = client.get("/api/translate/datasources/42/columns?env_id=unknown") + assert response.status_code in (400, 502, 422) +# [/DEF:test_api_datasource_columns_bad_env:Function] + + +# [DEF:test_api_update_job_not_found:Function] +# @PURPOSE: Verify PUT non-existent job returns 404. +def test_api_update_job_not_found(client): + """Test PUT non-existent job returns 404.""" + response = client.put("/api/translate/jobs/non-existent-id", json={"name": "Updated"}) + assert response.status_code == 404 +# [/DEF:test_api_update_job_not_found:Function] + +# [/DEF:TranslateJobTests:Module] diff --git a/backend/tests/test_translate_scheduler.py b/backend/tests/test_translate_scheduler.py new file mode 100644 index 00000000..b2a042fd --- /dev/null +++ b/backend/tests/test_translate_scheduler.py @@ -0,0 +1,206 @@ +# [DEF:TranslateSchedulerTests:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: tests, translate, scheduler +# @PURPOSE: Tests for TranslationScheduler CRUD and APScheduler integration. +# @LAYER: Test +# @RELATION: BINDS_TO -> [TranslationScheduler:Module] +# +# @TEST_CONTRACT: ScheduleFlow -> +# { +# invariants: [ +# "Create schedule returns TranslationSchedule with valid fields", +# "Update schedule modifies cron/timezone/active", +# "Delete schedule removes the row", +# "Enable/disable toggles is_active", +# "get_next_executions returns ISO datetime strings", +# "List active schedules returns only active rows" +# ] +# } +# @TEST_FIXTURE: valid_job -> created via TranslateJobService +# @TEST_FIXTURE: schedule_data -> valid cron expression +# @TEST_EDGE: schedule_not_found -> ValueError on get/delete non-existent +# @TEST_EDGE: next_executions_invalid_cron -> empty list +# @TEST_INVARIANT: schedule_crud_flow -> VERIFIED_BY: [valid_job, schedule_data] + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +import pytest +from unittest.mock import MagicMock +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from datetime import datetime, timezone +import uuid + +from src.core.database import Base +from src.app import app +from src.dependencies import get_current_user, get_config_manager, get_db +from src.plugins.translate.scheduler import TranslationScheduler, execute_scheduled_translation +from src.plugins.translate.service import TranslateJobService +from src.schemas.translate import TranslateJobCreate +from src.models.translate import TranslationJob, TranslationSchedule, TranslationRun + +# In-memory SQLite +SQLALCHEMY_DATABASE_URL = "sqlite:///:memory:" +engine = create_engine(SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}) +TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) +Base.metadata.create_all(bind=engine) + + +@pytest.fixture +def db_session(): + connection = engine.connect() + transaction = connection.begin() + session = TestingSessionLocal(bind=connection) + yield session + session.close() + transaction.rollback() + connection.close() + + +# [DEF:test_create_schedule:Function] +# @PURPOSE: Verify schedule creation with valid params. +def test_create_schedule(db_session): + """Test creating a schedule for a job.""" + from src.core.config_models import Environment + config_mgr = MagicMock() + config_mgr.get_environments.return_value = [] + + svc = TranslateJobService(db_session, config_mgr, "test_user") + payload = TranslateJobCreate(name="Sched Job", source_dialect="pg", target_dialect="ch") + job = svc.create_job(payload) + + scheduler = TranslationScheduler(db_session, config_mgr, "test_user") + schedule = scheduler.create_schedule( + job_id=job.id, + cron_expression="0 2 * * *", + timezone="UTC", + ) + + assert schedule.job_id == job.id + assert schedule.cron_expression == "0 2 * * *" + assert schedule.timezone == "UTC" + assert schedule.is_active is True + assert schedule.id is not None +# [/DEF:test_create_schedule:Function] + + +# [DEF:test_update_schedule:Function] +# @PURPOSE: Verify schedule update. +def test_update_schedule(db_session): + """Test updating a schedule.""" + from src.core.config_models import Environment + config_mgr = MagicMock() + config_mgr.get_environments.return_value = [] + + svc = TranslateJobService(db_session, config_mgr, "test_user") + job = svc.create_job(TranslateJobCreate(name="Job", source_dialect="pg", target_dialect="ch")) + + scheduler = TranslationScheduler(db_session, config_mgr, "test_user") + scheduler.create_schedule(job.id, "0 2 * * *") + + updated = scheduler.update_schedule(job.id, cron_expression="30 3 * * *", timezone_str="US/Eastern", is_active=False) + assert updated.cron_expression == "30 3 * * *" + assert updated.timezone == "US/Eastern" + assert updated.is_active is False +# [/DEF:test_update_schedule:Function] + + +# [DEF:test_delete_schedule:Function] +# @PURPOSE: Verify schedule deletion. +def test_delete_schedule(db_session): + """Test deleting a schedule.""" + from src.core.config_models import Environment + config_mgr = MagicMock() + config_mgr.get_environments.return_value = [] + + svc = TranslateJobService(db_session, config_mgr, "test_user") + job = svc.create_job(TranslateJobCreate(name="Job", source_dialect="pg", target_dialect="ch")) + + scheduler = TranslationScheduler(db_session, config_mgr, "test_user") + scheduler.create_schedule(job.id, "0 2 * * *") + scheduler.delete_schedule(job.id) + + with pytest.raises(ValueError, match="No schedule found"): + scheduler.get_schedule(job.id) +# [/DEF:test_delete_schedule:Function] + + +# [DEF:test_enable_disable_schedule:Function] +# @PURPOSE: Verify enable/disable toggle. +def test_enable_disable_schedule(db_session): + """Test enabling and disabling a schedule.""" + from src.core.config_models import Environment + config_mgr = MagicMock() + config_mgr.get_environments.return_value = [] + + svc = TranslateJobService(db_session, config_mgr, "test_user") + job = svc.create_job(TranslateJobCreate(name="Job", source_dialect="pg", target_dialect="ch")) + + scheduler = TranslationScheduler(db_session, config_mgr, "test_user") + scheduler.create_schedule(job.id, "0 2 * * *") + + sched = scheduler.set_schedule_active(job.id, False) + assert sched.is_active is False + + sched = scheduler.set_schedule_active(job.id, True) + assert sched.is_active is True +# [/DEF:test_enable_disable_schedule:Function] + + +# [DEF:test_get_schedule_not_found:Function] +# @PURPOSE: Verify ValueError on getting non-existent schedule. +def test_get_schedule_not_found(db_session): + """Test that getting a non-existent schedule raises ValueError.""" + from src.core.config_models import Environment + config_mgr = MagicMock() + config_mgr.get_environments.return_value = [] + + scheduler = TranslationScheduler(db_session, config_mgr, "test_user") + with pytest.raises(ValueError, match="No schedule found"): + scheduler.get_schedule("non-existent") +# [/DEF:test_get_schedule_not_found:Function] + + +# [DEF:test_list_active_schedules:Function] +# @PURPOSE: Verify listing only active schedules. +def test_list_active_schedules(db_session): + """Test listing active schedules.""" + from src.core.config_models import Environment + config_mgr = MagicMock() + config_mgr.get_environments.return_value = [] + + svc = TranslateJobService(db_session, config_mgr, "test_user") + job1 = svc.create_job(TranslateJobCreate(name="Job1", source_dialect="pg", target_dialect="ch")) + + scheduler = TranslationScheduler(db_session, config_mgr, "test_user") + scheduler.create_schedule(job1.id, "0 2 * * *") + scheduler.set_schedule_active(job1.id, True) + + active = TranslationScheduler.list_active_schedules(db_session) + assert len(active) >= 1 + assert active[0].job_id == job1.id +# [/DEF:test_list_active_schedules:Function] + + +# [DEF:test_get_next_executions:Function] +# @PURPOSE: Verify next execution time computation. +def test_get_next_executions(): + """Test computing next execution times.""" + times = TranslationScheduler.get_next_executions("0 2 * * *", "UTC", n=3) + assert len(times) == 3 + for t in times: + assert "T" in t # ISO format check +# [/DEF:test_get_next_executions:Function] + + +# [DEF:test_get_next_executions_invalid:Function] +# @PURPOSE: Verify invalid cron returns empty list. +def test_get_next_executions_invalid(): + """Test invalid cron returns empty list.""" + times = TranslationScheduler.get_next_executions("invalid cron", "UTC", n=3) + assert times == [] +# [/DEF:test_get_next_executions_invalid:Function] +# [/DEF:TranslateSchedulerTests:Module] diff --git a/frontend/src/lib/api.js b/frontend/src/lib/api.js index fd2188f7..9654b515 100755 --- a/frontend/src/lib/api.js +++ b/frontend/src/lib/api.js @@ -142,7 +142,9 @@ async function fetchApi(endpoint, options = {}) { return await response.json(); } catch (error) { console.error(`[api.fetchApi][Coherence:Failed] Error fetching from ${endpoint}:`, error); - notifyApiError(error); + if (!options.suppressToast) { + notifyApiError(error); + } throw error; } } @@ -411,10 +413,8 @@ export const api = { // Health getHealthSummary: (environmentId) => { - const params = new URLSearchParams(); - if (environmentId) params.append('environment_id', environmentId); - const query = params.toString(); - return fetchApi(`/health/summary${query ? `?${query}` : ''}`); + const query = environmentId ? `?env_id=${encodeURIComponent(environmentId)}` : ''; + return fetchApi(`/health/summary${query}`, { suppressToast: true }); }, }; // [/DEF:api:Data] diff --git a/frontend/src/lib/api/translate.js b/frontend/src/lib/api/translate.js new file mode 100644 index 00000000..973c3a69 --- /dev/null +++ b/frontend/src/lib/api/translate.js @@ -0,0 +1,583 @@ +// [DEF:TranslateApi:Module] +// @COMPLEXITY: 3 +// @SEMANTICS: frontend, api_client, translate, crud +// @PURPOSE: API client for translation job CRUD, datasource column queries, and dictionary management. +// @LAYER: Infra +// @RELATION: DEPENDS_ON -> [api_module] +// @PRE: Shared API wrapper is configured for the current frontend auth/session context. +// @POST: Translate API helpers return backend payloads or normalized UI-safe errors. + +import { api } from '$lib/api.js'; + +// [DEF:normalizeTranslateError:Function] +// @PURPOSE: Convert unknown API exceptions into deterministic UI-consumable error objects. +// @PRE: error may be Error/string/object. +// @POST: Returns structured error object with message and code. +function normalizeTranslateError(error, defaultMessage = 'Translation API error') { + const message = + (error && typeof error.message === 'string' && error.message) || + (typeof error === 'string' && error) || + defaultMessage; + + return { message, code: 'TRANSLATE_API_ERROR', retryable: true }; +} +// [/DEF:normalizeTranslateError:Function] + +// [DEF:fetchJobs:Function] +// @PURPOSE: Fetch paginated list of translation jobs with optional status filter. +// @PRE: Valid auth context. +// @POST: Returns array of job objects. +export async function fetchJobs(options = {}) { + try { + const params = new URLSearchParams(); + if (options.page != null) params.append('page', String(options.page)); + if (options.page_size != null) params.append('page_size', String(options.page_size)); + if (options.status) params.append('status', options.status); + const query = params.toString(); + return await api.fetchApi(`/translate/jobs${query ? `?${query}` : ''}`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to load translation jobs'); + } +} +// [/DEF:fetchJobs:Function] + +// [DEF:createJob:Function] +// @PURPOSE: Create a new translation job. +// @PRE: payload contains valid job configuration. +// @POST: Returns the created job object. +export async function createJob(payload) { + try { + return await api.postApi('/translate/jobs', payload); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to create translation job'); + } +} +// [/DEF:createJob:Function] + +// [DEF:updateJob:Function] +// @PURPOSE: Update an existing translation job. +// @PRE: jobId is non-empty string; payload contains fields to update. +// @POST: Returns the updated job object. +export async function updateJob(jobId, payload) { + try { + return await api.requestApi(`/translate/jobs/${jobId}`, 'PUT', payload); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to update translation job'); + } +} +// [/DEF:updateJob:Function] + +// [DEF:deleteJob:Function] +// @PURPOSE: Delete a translation job. +// @PRE: jobId is non-empty string. +// @POST: Returns null on success. +export async function deleteJob(jobId) { + try { + return await api.deleteApi(`/translate/jobs/${jobId}`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to delete translation job'); + } +} +// [/DEF:deleteJob:Function] + +// [DEF:duplicateJob:Function] +// @PURPOSE: Duplicate a translation job. +// @PRE: jobId is non-empty string. +// @POST: Returns the duplicated job summary. +export async function duplicateJob(jobId) { + try { + return await api.postApi(`/translate/jobs/${jobId}/duplicate`, {}); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to duplicate translation job'); + } +} +// [/DEF:duplicateJob:Function] + +// [DEF:fetchDatasourceColumns:Function] + +export async function fetchDatasources(envId, search = '') { + try { + return await api.fetchApi( + `/translate/datasources?env_id=${encodeURIComponent(envId)}&search=${encodeURIComponent(search)}` + ); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to fetch datasources'); + } +} + +export async function fetchDatasourceColumns(datasourceId, envId) { + try { + return await api.fetchApi( + `/translate/datasources/${datasourceId}/columns?env_id=${encodeURIComponent(envId)}` + ); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to fetch datasource columns'); + } +} + +// [DEF:fetchPreview:Function] +// @PURPOSE: Trigger a translation preview for a job. +// @PRE: jobId is non-empty string. +// @POST: Returns preview session with rows and cost estimate. +export async function fetchPreview(jobId, sampleSize = 10) { + try { + return await api.postApi(`/translate/jobs/${jobId}/preview`, { sample_size: sampleSize }); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to run preview'); + } +} +// [/DEF:fetchPreview:Function] + +// [DEF:approveRow:Function] +// @PURPOSE: Approve a preview row. +// @PRE: jobId and rowKey are non-empty strings. +// @POST: Returns updated row. +export async function approveRow(jobId, rowKey) { + try { + return await api.requestApi(`/translate/jobs/${jobId}/preview/rows/${rowKey}`, 'PUT', { + action: 'approve' + }); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to approve row'); + } +} +// [/DEF:approveRow:Function] + +// [DEF:editRow:Function] +// @PURPOSE: Edit a preview row's translation. +// @PRE: jobId and rowKey are non-empty strings; translation is non-empty. +// @POST: Returns updated row. +export async function editRow(jobId, rowKey, translation) { + try { + return await api.requestApi(`/translate/jobs/${jobId}/preview/rows/${rowKey}`, 'PUT', { + action: 'edit', + translation: translation + }); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to edit row'); + } +} +// [/DEF:editRow:Function] + +// [DEF:rejectRow:Function] +// @PURPOSE: Reject a preview row. +// @PRE: jobId and rowKey are non-empty strings. +// @POST: Returns updated row. +export async function rejectRow(jobId, rowKey) { + try { + return await api.requestApi(`/translate/jobs/${jobId}/preview/rows/${rowKey}`, 'PUT', { + action: 'reject' + }); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to reject row'); + } +} +// [/DEF:rejectRow:Function] + +// [DEF:acceptPreview:Function] +// @PURPOSE: Accept a preview session, enabling full translation execution. +// @PRE: jobId is non-empty string. +// @POST: Returns accepted preview session. +export async function acceptPreview(jobId) { + try { + return await api.postApi(`/translate/jobs/${jobId}/preview/accept`, {}); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to accept preview'); + } +} +// [/DEF:acceptPreview:Function] + +// [DEF:fetchPreviewRecords:Function] +// @PURPOSE: Fetch records for a preview session. +// @PRE: sessionId is non-empty string. +// @POST: Returns list of preview records. +export async function fetchPreviewRecords(sessionId) { + try { + return await api.fetchApi(`/translate/preview/${sessionId}/records`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to fetch preview records'); + } +} +// [/DEF:fetchPreviewRecords:Function] +// [/DEF:fetchDatasourceColumns:Function] + + +// [DEF:triggerRun:Function] +// @PURPOSE: Trigger a full translation run for a job. +// @PRE: jobId is non-empty string. +// @POST: Returns the created run object. +export async function triggerRun(jobId) { + try { + return await api.postApi(`/translate/jobs/${jobId}/run`, {}); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to start translation run'); + } +} +// [/DEF:triggerRun:Function] + +// [DEF:fetchRunStatus:Function] +// @PURPOSE: Fetch status and statistics for a translation run. +// @PRE: runId is non-empty string. +// @POST: Returns run details with statistics. +export async function fetchRunStatus(runId) { + try { + return await api.fetchApi(`/translate/runs/${runId}`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to fetch run status'); + } +} +// [/DEF:fetchRunStatus:Function] + +// [DEF:fetchRunHistory:Function] +// @PURPOSE: Fetch run history for a job. +// @PRE: jobId is non-empty string. +// @POST: Returns paginated list of runs. +export async function fetchRunHistory(jobId, options = {}) { + try { + const params = new URLSearchParams(); + if (options.page != null) params.append('page', String(options.page)); + if (options.page_size != null) params.append('page_size', String(options.page_size)); + const query = params.toString(); + return await api.fetchApi(`/translate/jobs/${jobId}/runs${query ? `?${query}` : ''}`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to fetch run history'); + } +} +// [/DEF:fetchRunHistory:Function] + +// [DEF:fetchRunRecords:Function] +// @PURPOSE: Fetch paginated records for a translation run. +// @PRE: runId is non-empty string. +// @POST: Returns paginated records. +export async function fetchRunRecords(runId, options = {}) { + try { + const params = new URLSearchParams(); + if (options.page != null) params.append('page', String(options.page)); + if (options.page_size != null) params.append('page_size', String(options.page_size)); + if (options.status) params.append('status', options.status); + const query = params.toString(); + return await api.fetchApi(`/translate/runs/${runId}/records${query ? `?${query}` : ''}`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to fetch run records'); + } +} +// [/DEF:fetchRunRecords:Function] + +// [DEF:retryFailedBatches:Function] +// @PURPOSE: Retry failed batches in a translation run. +// @PRE: runId is non-empty string. +// @POST: Returns updated run. +export async function retryFailedBatches(runId) { + try { + return await api.postApi(`/translate/runs/${runId}/retry`, {}); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to retry batches'); + } +} +// [/DEF:retryFailedBatches:Function] + +// [DEF:retryInsert:Function] +// @PURPOSE: Retry the SQL insert phase for a run. +// @PRE: runId is non-empty string. +// @POST: Returns updated run. +export async function retryInsert(runId) { + try { + return await api.postApi(`/translate/runs/${runId}/retry-insert`, {}); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to retry insert'); + } +} +// [/DEF:retryInsert:Function] + +// [DEF:cancelRun:Function] +// @PURPOSE: Cancel a running translation. +// @PRE: runId is non-empty string. +// @POST: Returns updated run. +export async function cancelRun(runId) { + try { + return await api.postApi(`/translate/runs/${runId}/cancel`, {}); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to cancel run'); + } +} +// [/DEF:cancelRun:Function] + +// [DEF:fetchRunBatches:Function] +// @PURPOSE: Fetch batches for a translation run. +// @PRE: runId is non-empty string. +// @POST: Returns list of batches. +export async function fetchRunBatches(runId) { + try { + return await api.fetchApi(`/translate/runs/${runId}/batches`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to fetch run batches'); + } +} +// [/DEF:fetchRunBatches:Function] + + +// [DEF:dictionaryApi:Variable] +// @PURPOSE: Namespaced API helpers for terminology dictionary CRUD operations. +export const dictionaryApi = { + async fetchDictionaries(options = {}) { + try { + const params = new URLSearchParams(); + if (options.page != null) params.append('page', String(options.page)); + if (options.page_size != null) params.append('page_size', String(options.page_size)); + const query = params.toString(); + return await api.fetchApi(`/translate/dictionaries${query ? `?${query}` : ''}`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to load dictionaries'); + } + }, + + async createDictionary(payload) { + try { + return await api.postApi('/translate/dictionaries', payload); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to create dictionary'); + } + }, + + async getDictionary(dictionaryId) { + try { + return await api.fetchApi(`/translate/dictionaries/${dictionaryId}`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to load dictionary'); + } + }, + + async updateDictionary(dictionaryId, payload) { + try { + return await api.requestApi(`/translate/dictionaries/${dictionaryId}`, 'PUT', payload); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to update dictionary'); + } + }, + + async deleteDictionary(dictionaryId) { + try { + return await api.deleteApi(`/translate/dictionaries/${dictionaryId}`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to delete dictionary'); + } + }, + + async fetchEntries(dictionaryId, options = {}) { + try { + const params = new URLSearchParams(); + if (options.page != null) params.append('page', String(options.page)); + if (options.page_size != null) params.append('page_size', String(options.page_size)); + const query = params.toString(); + return await api.fetchApi( + `/translate/dictionaries/${dictionaryId}/entries${query ? `?${query}` : ''}` + ); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to load dictionary entries'); + } + }, + + async createEntry(dictionaryId, payload) { + try { + return await api.postApi(`/translate/dictionaries/${dictionaryId}/entries`, payload); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to create dictionary entry'); + } + }, + + async updateEntry(dictionaryId, entryId, payload) { + try { + return await api.requestApi( + `/translate/dictionaries/${dictionaryId}/entries/${entryId}`, + 'PUT', + payload + ); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to update dictionary entry'); + } + }, + + async deleteEntry(dictionaryId, entryId) { + try { + return await api.deleteApi(`/translate/dictionaries/${dictionaryId}/entries/${entryId}`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to delete dictionary entry'); + } + }, + + async importEntries(dictionaryId, payload) { + try { + return await api.postApi(`/translate/dictionaries/${dictionaryId}/import`, payload); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to import dictionary entries'); + } + }, +}; +// [/DEF:dictionaryApi:Variable] + +// ============================================================ +// Corrections API +// ============================================================ + +// [DEF:submitCorrection:Function] +// @PURPOSE: Submit a single term correction. +// @PRE: payload contains source_term, incorrect_target_term, corrected_target_term, dictionary_id. +// @POST: Returns correction result with conflict info. +export async function submitCorrection(payload) { + try { + return await api.postApi('/translate/corrections', payload); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to submit correction'); + } +} +// [/DEF:submitCorrection:Function] + +// [DEF:submitBulkCorrections:Function] +// @PURPOSE: Submit multiple term corrections atomically. +// @PRE: payload contains corrections array and dictionary_id. +// @POST: Returns bulk result with conflict info. +export async function submitBulkCorrections(payload) { + try { + return await api.postApi('/translate/corrections/bulk', payload); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to submit bulk corrections'); + } +} +// [/DEF:submitBulkCorrections:Function] + +// ============================================================ +// Schedule API +// ============================================================ + +// [DEF:fetchSchedule:Function] +// @PURPOSE: Fetch schedule for a translation job. +export async function fetchSchedule(jobId) { + try { + return await api.fetchApi(`/translate/jobs/${jobId}/schedule`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to fetch schedule'); + } +} +// [/DEF:fetchSchedule:Function] + +// [DEF:setSchedule:Function] +// @PURPOSE: Create or update schedule for a translation job. +export async function setSchedule(jobId, payload) { + try { + return await api.requestApi(`/translate/jobs/${jobId}/schedule`, 'PUT', payload); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to set schedule'); + } +} +// [/DEF:setSchedule:Function] + +// [DEF:deleteSchedule:Function] +// @PURPOSE: Delete schedule for a translation job. +export async function deleteSchedule(jobId) { + try { + return await api.deleteApi(`/translate/jobs/${jobId}/schedule`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to delete schedule'); + } +} +// [/DEF:deleteSchedule:Function] + +// [DEF:enableSchedule:Function] +// @PURPOSE: Enable a schedule. +export async function enableSchedule(jobId) { + try { + return await api.postApi(`/translate/jobs/${jobId}/schedule/enable`, {}); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to enable schedule'); + } +} +// [/DEF:enableSchedule:Function] + +// [DEF:disableSchedule:Function] +// @PURPOSE: Disable a schedule. +export async function disableSchedule(jobId) { + try { + return await api.postApi(`/translate/jobs/${jobId}/schedule/disable`, {}); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to disable schedule'); + } +} +// [/DEF:disableSchedule:Function] + +// [DEF:fetchNextExecutions:Function] +// @PURPOSE: Preview next N schedule executions. +export async function fetchNextExecutions(jobId, n = 3) { + try { + return await api.fetchApi(`/translate/jobs/${jobId}/schedule/next-executions?n=${n}`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to fetch next executions'); + } +} +// [/DEF:fetchNextExecutions:Function] + +// ============================================================ +// History + Metrics API +// ============================================================ + +// [DEF:fetchAllRuns:Function] +// @PURPOSE: Fetch all runs with cross-job filtering and pagination. +export async function fetchAllRuns(options = {}) { + try { + const params = new URLSearchParams(); + if (options.page != null) params.append('page', String(options.page)); + if (options.page_size != null) params.append('page_size', String(options.page_size)); + if (options.job_id) params.append('job_id', options.job_id); + if (options.status) params.append('status', options.status); + if (options.trigger_type) params.append('trigger_type', options.trigger_type); + if (options.created_by) params.append('created_by', options.created_by); + const query = params.toString(); + return await api.fetchApi(`/translate/runs${query ? `?${query}` : ''}`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to fetch runs'); + } +} +// [/DEF:fetchAllRuns:Function] + +// [DEF:fetchRunDetail:Function] +// @PURPOSE: Fetch detailed run info with config_snapshot, records, events. +export async function fetchRunDetail(runId) { + try { + return await api.fetchApi(`/translate/runs/${runId}/detail`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to fetch run detail'); + } +} +// [/DEF:fetchRunDetail:Function] + +// [DEF:fetchJobMetrics:Function] +// @PURPOSE: Fetch aggregated metrics for a specific job. +export async function fetchJobMetrics(jobId) { + try { + return await api.fetchApi(`/translate/jobs/${jobId}/metrics`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to fetch job metrics'); + } +} +// [/DEF:fetchJobMetrics:Function] + +// [DEF:fetchAllMetrics:Function] +// @PURPOSE: Fetch aggregated metrics for all jobs. +export async function fetchAllMetrics() { + try { + return await api.fetchApi('/translate/metrics'); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to fetch metrics'); + } +} +// [/DEF:fetchAllMetrics:Function] + +// [DEF:downloadSkippedCsv:Function] +// @PURPOSE: Download skipped records CSV for a run. +export async function downloadSkippedCsv(runId) { + try { + return await api.fetchApi(`/translate/runs/${runId}/skipped.csv`); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to download skipped CSV'); + } +} +// [/DEF:downloadSkippedCsv:Function] diff --git a/frontend/src/lib/components/layout/sidebarNavigation.js b/frontend/src/lib/components/layout/sidebarNavigation.js index a863531c..abd58783 100644 --- a/frontend/src/lib/components/layout/sidebarNavigation.js +++ b/frontend/src/lib/components/layout/sidebarNavigation.js @@ -102,6 +102,35 @@ export function buildSidebarCategories(i18nState, user, features = {}) { }, ], }, + { + id: "translation", + label: nav.translation, + icon: "translate", + tone: "from-cyan-100 to-teal-100 text-teal-700 ring-teal-200", + path: "/translate", + requiredPermission: "translate.job", + requiredAction: "VIEW", + subItems: [ + { + label: nav.translation_jobs, + path: "/translate", + requiredPermission: "translate.job", + requiredAction: "VIEW", + }, + { + label: nav.translation_dictionaries, + path: "/translate/dictionaries", + requiredPermission: "translate.dictionary", + requiredAction: "VIEW", + }, + { + label: nav.translation_history, + path: "/translate/history", + requiredPermission: "translate.job", + requiredAction: "VIEW", + }, + ], + }, { id: "reports", label: nav.reports, diff --git a/frontend/src/lib/components/translate/BulkCorrectionSidebar.svelte b/frontend/src/lib/components/translate/BulkCorrectionSidebar.svelte new file mode 100644 index 00000000..a16c730c --- /dev/null +++ b/frontend/src/lib/components/translate/BulkCorrectionSidebar.svelte @@ -0,0 +1,292 @@ + + + + + + + +{#if isOpen} +
+ +
+ +
+
+

{$t.translate?.corrections?.title}

+

{$t.translate?.corrections?.subtitle}

+
+ +
+ + +
+ {#if uxState === 'collecting'} + {#if items.length === 0} +
+ + + +

{$t.translate?.corrections?.no_items}

+

{$t.translate?.corrections?.add_hint}

+
+ {:else} + {#each items as item, idx} +
+
+
+

{$t.translate?.corrections?.source_term}

+

{item.sourceTerm}

+
+ +
+
+

{$t.translate?.corrections?.incorrect_target}

+

{item.incorrectTarget}

+
+
+ + updateCorrection(idx, e.target.value)} + placeholder={$t.translate?.corrections?.corrected_placeholder} + class="w-full mt-1 px-2 py-1 text-sm border border-gray-300 rounded focus:ring-2 focus:ring-green-500 focus:border-green-500" + /> +
+
+ {/each} + {/if} + + {:else if uxState === 'submitting'} +
+
+

{$t.translate?.corrections?.submitting}

+
+ + {:else if uxState === 'submitted'} +
+ + + +

{$t.translate?.corrections?.submit_success}

+ {#if submitResult} +

+ {$t.translate?.corrections?.submit_partial + ?.replace('{success}', submitResult.created_count || submitResult.updated_count || items.length) + ?.replace('{total}', items.length)} +

+ {/if} +
+ {/if} +
+ + +
+ {#if uxState === 'collecting' && items.length > 0} + +
+ + +
+ + + {:else if uxState === 'submitted'} + + {/if} +
+
+{/if} + diff --git a/frontend/src/lib/components/translate/ScheduleConfig.svelte b/frontend/src/lib/components/translate/ScheduleConfig.svelte new file mode 100644 index 00000000..1826d27a --- /dev/null +++ b/frontend/src/lib/components/translate/ScheduleConfig.svelte @@ -0,0 +1,260 @@ + + + +
+
+

Schedule Configuration

+ {#if !isLoading && scheduleExists} + + {/if} +
+ + {#if isLoading} +
+ + {:else if uxState === 'idle'} +
+ + + +

No schedule configured

+

Set up a recurring schedule for automatic translations

+ +
+ + {:else if uxState === 'editing' || uxState === 'enabled' || uxState === 'disabled'} +
+ +
+ +
+ + +
+

+ {getCronDescription(cronExpression)} +

+
+ + +
+ + +
+ + + {#if nextExecutions.length > 0} +
+

Next Executions

+ {#each nextExecutions as dt, i} +

{i + 1}. {new Date(dt).toLocaleString()}

+ {/each} +
+ {/if} + + + {#if uxState === 'no_prior_run_warning'} +
+

+ ⚠ No prior successful manual run. Schedule may run with default settings. + Consider running manually first. +

+
+ {/if} + + +
+ {#if scheduleExists} + + {:else} +
+ {/if} +
+ {#if scheduleExists && (uxState === 'enabled' || uxState === 'disabled')} + + {/if} + {#if uxState === 'editing'} + + {/if} +
+
+
+ {/if} +
+ diff --git a/frontend/src/lib/components/translate/TermCorrectionPopup.svelte b/frontend/src/lib/components/translate/TermCorrectionPopup.svelte new file mode 100644 index 00000000..76f92fa4 --- /dev/null +++ b/frontend/src/lib/components/translate/TermCorrectionPopup.svelte @@ -0,0 +1,226 @@ + + + +{#if uxState !== 'closed'} +
+
e.stopPropagation()}> + +
+

Submit Term Correction

+ +
+ + + {#if uxState === 'selecting'} +

Loading dictionaries...

+ + + {:else if uxState === 'editing'} +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + {:else if uxState === 'submitting'} +
+
+

Submitting correction...

+
+ + + {:else if uxState === 'conflict_detected'} +
+
+

Conflict Detected

+

+ "{conflictInfo?.source_term}" already exists in this dictionary + with target "{conflictInfo?.existing_target_term}". +

+
+
+ + +
+
+ + + {:else if uxState === 'submitted'} +
+ + + +

Correction Submitted

+

{submitResult?.message || ''}

+ +
+ {/if} +
+
+{/if} + diff --git a/frontend/src/lib/components/translate/TranslationMetricsDashboard.svelte b/frontend/src/lib/components/translate/TranslationMetricsDashboard.svelte new file mode 100644 index 00000000..970354cf --- /dev/null +++ b/frontend/src/lib/components/translate/TranslationMetricsDashboard.svelte @@ -0,0 +1,191 @@ + + + +
+ +
+
+

{$t.translate?.metrics?.title}

+

{$t.translate?.metrics?.subtitle}

+
+ +
+ + + {#if uxState === 'loading'} +
+ {#each Array(3) as _} +
+ {/each} +
+ + + {:else if uxState === 'error'} +
+

{errorMessage}

+ +
+ + + {:else if uxState === 'empty'} +
+ + + +

{$t.translate?.metrics?.no_data}

+
+ + + {:else if uxState === 'loaded'} + +
+
+

{$t.translate?.metrics?.total_runs}

+

{totalRuns}

+
+
+

{$t.translate?.metrics?.success_rate}

+

{successRate}%

+
+
+

{$t.translate?.metrics?.failed_runs}

+

{failedRuns}

+
+
+

{$t.translate?.metrics?.total_records}

+

{totalRecords.toLocaleString()}

+
+
+

{$t.translate?.metrics?.total_tokens}

+

{totalTokens.toLocaleString()}

+
+
+

{$t.translate?.metrics?.total_cost}

+

${totalCost.toFixed(2)}

+
+
+ + +
+
+

{$t.translate?.metrics?.runs_per_job}

+
+
+ + + + + + + + + + + + + {#each metrics as m} + + + + + + + + + {/each} + +
{$t.translate?.metrics?.job}{$t.translate?.metrics?.runs}{$t.translate?.metrics?.successful}{$t.translate?.metrics?.failed}{$t.translate?.metrics?.tokens}{$t.translate?.metrics?.cost}
{m.job_name || m.job_id}{m.total_runs || 0}{m.successful_runs || 0}{m.failed_runs || 0}{(m.total_tokens || 0).toLocaleString()}${(m.total_cost || 0).toFixed(2)}
+
+
+ + + {#if avgLatency !== null} +
+

{$t.translate?.metrics?.avg_latency}

+

{avgLatency}s

+
+ {/if} + {/if} +
+ diff --git a/frontend/src/lib/components/translate/TranslationPreview.svelte b/frontend/src/lib/components/translate/TranslationPreview.svelte new file mode 100644 index 00000000..0cff9beb --- /dev/null +++ b/frontend/src/lib/components/translate/TranslationPreview.svelte @@ -0,0 +1,409 @@ + + + +
+ +
+

{$t.translate?.preview?.title}

+ {#if uxState === 'preview_loaded'} + + {session?.status === 'APPLIED' ? $t.translate?.preview?.accepted_badge : $t.translate?.preview?.active_badge} + + {/if} +
+ + + {#if uxState === 'idle'} +
+

+ {$t.translate?.preview?.title} — {$t.translate?.preview?.sample_size} {sampleSize} +

+
+ + +
+ +
+ + + {:else if uxState === 'loading'} +
+
+
+ {$t.translate?.preview?.running_preview} +
+
+ {#each Array(3) as _} +
+ {/each} +
+
+ + + {:else if uxState === 'preview_error'} +
+

{errorMessage}

+
+ + +
+
+ + + {:else if uxState === 'preview_loaded'} + + {#if costEstimate} +
+

{$t.translate?.preview?.cost_estimate}

+
+
+ {$t.translate?.preview?.sample_rows} + {costEstimate.sample_size} +
+
+ {$t.translate?.preview?.sample_tokens} + {costEstimate.sample_total_tokens?.toLocaleString()} +
+
+ {$t.translate?.preview?.sample_cost} + ${costEstimate.sample_cost?.toFixed(6)} +
+
+ {$t.translate?.preview?.est_total_cost} + ${costEstimate.estimated_cost?.toFixed(6)} +
+
+

+ {$t.translate?.preview?.est_total_cost?.replace('{count}', '')} {costEstimate.estimated_total_rows?.toLocaleString()} {costEstimate.estimated_tokens?.toLocaleString()} {$t.translate?.preview?.sample_tokens} +

+
+ {/if} + + +
+
+ {$t.translate?.preview?.rows_count?.replace('{count}', records.length)} + {$t.translate?.preview?.approved_count?.replace('{count}', approvedCount)} + {$t.translate?.preview?.rejected_count?.replace('{count}', rejectedCount)} + {#if pendingCount > 0} + {$t.translate?.preview?.pending_count?.replace('{count}', pendingCount)} + {/if} +
+
+ {#if pendingCount > 0} + + {/if} + +
+
+ + +
+ + + + + + + + + + + + {#each records as row, idx} + + + +
{$t.translate?.preview?.table_number}{$t.translate?.preview?.table_source}{$t.translate?.preview?.table_translation}{$t.translate?.preview?.table_status}{$t.translate?.preview?.table_actions}
{idx + 1} + {row.source_sql || $t.translate?.preview?.empty_placeholder} + + {#if editingRowId === row.id} + + +
+ + +
+ + + {/if} + + {#if state === 'loading'} + +
+ {#each [1, 2, 3] as _} +
+ {/each} +
+ + {:else if state === 'empty'} + +
+

{$t.translate?.dictionaries?.no_dicts}

+ +
+ + {:else if state === 'populated'} + +
+ {#each dictionaries as dict} +
+
+ +
+
+ + {#if deleteConfirmId === dict.id} +
+ {$t.translate?.dictionaries?.confirm_delete} + + +
+ {:else} + + {/if} +
+
+ {/each} +
+ + + {#if total > page_size} +
+ + + {$t.translate?.dictionaries?.page_info.replace('{current}', currentPage).replace('{total}', Math.ceil(total / page_size))} + + +
+ {/if} + {/if} + + + diff --git a/frontend/src/routes/translate/dictionaries/[id]/+page.svelte b/frontend/src/routes/translate/dictionaries/[id]/+page.svelte new file mode 100644 index 00000000..cded47d7 --- /dev/null +++ b/frontend/src/routes/translate/dictionaries/[id]/+page.svelte @@ -0,0 +1,568 @@ + + + + + +
+ {#snippet subtitleContent()} + {#if dictionary} +

+ {dictionary.source_dialect} → {dictionary.target_dialect} + {#if dictionary.description} + — {dictionary.description} + {/if} + · {$t.translate?.dictionaries?.entry_count.replace('{count}', totalEntries)} +

+ {/if} + {/snippet} + + {#snippet dictActions()} + {#if state === 'editing'} +
+ + + +
+ {/if} + {/snippet} + + + + {#if state === 'loading'} + +
+ {#each [1, 2, 3] as _} +
+ {/each} +
+ + + {:else if state === 'importing' || state === 'import_preview'} + +
+ {#if state === 'importing'} +
+ + +
+
+
+ + +
+
+ + +
+
+
+ + +
+ {/if} + + {#if state === 'import_preview'} + {#if importPreview.length > 0} +
+ + + + + + + + + + + {#each importPreview as row} + + + + + + + {/each} + +
LineSourceTargetConflict
{row.line}{row.source_term}{row.target_term} + {#if row.is_conflict} + + Existing: {row.existing_target_term} + ({importOnConflict === 'overwrite' ? 'will overwrite' : 'will skip'}) + + {:else} + New + {/if} +
+
+ {/if} + + {#if importErrors.length > 0} +
+

Errors ({importErrors.length}):

+ {#each importErrors as err} +

Line {err.line}: {err.error}

+ {/each} +
+ {/if} + +
+ + +
+ {/if} +
+ + + {:else if state === 'editing'} + {#if showAddForm} + +
+
+
+ + +
+
+ + +
+
+
+ + +
+
+ + +
+
+ + {/if} + + {#if entries.length === 0} + +
+

No entries yet.

+ +
+ + {:else} + +
+ + + + + + + + + + + {#each entries as entry} + {#if editEntryId === entry.id} + + + + + + + {:else} + + + + + + + {/if} + {/each} + +
Source TermTarget TermContextActions
+ + + + + + +
+ + +
+
{entry.source_term}{entry.target_term}{entry.context_notes || ''} + {#if deleteEntryId === entry.id} +
+ Confirm? + + +
+ {:else} +
+ + +
+ {/if} +
+
+ + + {#if totalEntries > entryPageSize} +
+ + + Page {entryPage} of {Math.ceil(totalEntries / entryPageSize)} + + +
+ {/if} + {/if} + {/if} +
+ + diff --git a/frontend/src/routes/translate/history/+page.svelte b/frontend/src/routes/translate/history/+page.svelte new file mode 100644 index 00000000..85ee3b91 --- /dev/null +++ b/frontend/src/routes/translate/history/+page.svelte @@ -0,0 +1,418 @@ + + + +
+
+
+

Translation History

+

Filterable run history with metrics aggregation

+
+ +
+ + + {#if totalMetric()} +
+
+

Total Runs

+

{totalMetric().total_runs}

+
+
+

Successful

+

{totalMetric().successful_runs}

+
+
+

Failed

+

{totalMetric().failed_runs}

+
+
+

Records

+

{totalMetric().total_records}

+
+
+ {/if} + + +
+
+ + + + + +
+
+ + + {#if isLoading} +
+ {#each Array(5) as _} +
+ {/each} +
+ + + {:else if uxState === 'empty'} +
+ + + +

No translation runs found

+

Runs appear here after executing a translation job.

+
+ + + {:else if uxState === 'populated' || uxState === 'detail_open'} +
+ {#each runs as run} +
openDetail(run)} + class="bg-white border border-gray-200 rounded-lg p-3 hover:shadow-sm hover:border-gray-300 transition-all cursor-pointer" + > +
+
+
+ {getJobName(run.job_id)} + + {run.status} + + {#if run.trigger_type} + {run.trigger_type} + {/if} +
+
+ {run.created_at ? new Date(run.created_at).toLocaleString() : ''} + {run.total_records || 0} records + {run.successful_records || 0} ok + {run.failed_records || 0} fail + {#if run.created_by} + by {run.created_by} + {/if} +
+
+
e.stopPropagation()}> + {#if run.status === 'COMPLETED' && (run.skipped_records || 0) > 0} + + {/if} + + + +
+
+
+ {/each} +
+ + +
+

Showing {runs.length} of {total} runs

+
+ + +
+
+ {/if} + + + {#if uxState === 'detail_open' && selectedRunDetail} +
+
+
+
+

Run Detail

+ +
+ + +
+
+

Status

+ + {selectedRunDetail.status} + +
+
+

Trigger

+

{selectedRunDetail.trigger_type || 'manual'}

+
+
+

Started

+

{selectedRunDetail.started_at ? new Date(selectedRunDetail.started_at).toLocaleString() : '-'}

+
+
+

Completed

+

{selectedRunDetail.completed_at ? new Date(selectedRunDetail.completed_at).toLocaleString() : '-'}

+
+
+ + +
+
+

{selectedRunDetail.total_records || 0}

+

Total

+
+
+

{selectedRunDetail.successful_records || 0}

+

Success

+
+
+

{selectedRunDetail.failed_records || 0}

+

Failed

+
+
+

{selectedRunDetail.skipped_records || 0}

+

Skipped

+
+
+ + + {#if selectedRunDetail.config_snapshot} +
+

Config Snapshot

+
+ 
{JSON.stringify(selectedRunDetail.config_snapshot, null, 2)}
+
+
+ {/if} + + + {#if selectedRunDetail.config_hash || selectedRunDetail.dict_snapshot_hash} +
+

Snapshots

+
+ {#if selectedRunDetail.config_hash} +
Config: {selectedRunDetail.config_hash}
+ {/if} + {#if selectedRunDetail.dict_snapshot_hash} +
Dict: {selectedRunDetail.dict_snapshot_hash}
+ {/if} + {#if selectedRunDetail.key_hash} +
Key: {selectedRunDetail.key_hash}
+ {/if} +
+
+ {/if} + + + {#if selectedRunDetail.events && selectedRunDetail.events.length > 0} +
+

Events ({selectedRunDetail.events.length})

+
+ {#each selectedRunDetail.events as evt} +
+ {evt.event_type} + | + {evt.created_at ? new Date(evt.created_at).toLocaleString() : ''} +
+ {/each} +
+
+ {/if} + + + {#if selectedRunDetail.error_message} +
+

Error

+

{selectedRunDetail.error_message}

+
+ {/if} + + + {#if selectedRunDetail.superset_execution_id} +
+

Superset Query ID

+

{selectedRunDetail.superset_execution_id}

+
+ {/if} +
+
+ {/if} +
+