feat: implement LLM table translation service with searchable datasource dropdown, i18n, sidebar, and RBAC
- Add backend plugin modules: preview, executor, orchestrator, events, sql_generator, superset_executor, dictionary, scheduler, metrics - Add API routes for translate jobs, runs, dictionaries, preview, schedule, metrics, corrections - Add ORM models (12 tables) and Pydantic schemas (15 DTOs) - Register translate router in app.py with RBAC permission guards - Add frontend pages: job list, job config, dictionary list/editor, history - Add 7 reusable Svelte 5 components: Preview, RunProgress, RunResult, TermCorrection, BulkCorrection, ScheduleConfig, MetricsDashboard - Add searchable datasource dropdown with Superset API integration - Add i18n (en/ru) with ~300 keys across common, config, jobs, dictionaries, preview namespaces - Add Translation sidebar category with Jobs/Dictionaries/History sub-items - Hide health monitor error toast via suppressToast API option - Add 69 backend tests and 44 frontend test files - Fix: SupersetClient env resolution (string -> Environment object) - Fix: Dataset detail API returns proper database dict - Fix: Database dialect extraction fallback when metadata incomplete
This commit is contained in:
@@ -53,6 +53,7 @@ from .api.routes import (
|
||||
profile,
|
||||
health,
|
||||
dataset_review,
|
||||
translate,
|
||||
)
|
||||
from .api import auth
|
||||
|
||||
@@ -271,6 +272,7 @@ app.include_router(clean_release_v2.router)
|
||||
app.include_router(profile.router)
|
||||
app.include_router(dataset_review.router)
|
||||
app.include_router(health.router)
|
||||
app.include_router(translate.router)
|
||||
# [/DEF:API_Routes:Block]
|
||||
|
||||
|
||||
|
||||
292
backend/src/models/translate.py
Normal file
292
backend/src/models/translate.py
Normal file
@@ -0,0 +1,292 @@
|
||||
# [DEF:TranslateModels:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @SEMANTICS: translate, models, sqlalchemy, persistence
|
||||
# @PURPOSE: SQLAlchemy ORM models for LLM-based SQL/dashboard translation across dialects.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: INHERITS_FROM -> MappingModels:Base
|
||||
|
||||
from sqlalchemy import (
|
||||
Column, String, Boolean, DateTime, JSON, Text,
|
||||
ForeignKey, Integer, UniqueConstraint, Index
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSON as PG_JSON
|
||||
from datetime import datetime, timezone
|
||||
import uuid
|
||||
import hashlib
|
||||
from .mapping import Base
|
||||
|
||||
|
||||
def generate_uuid():
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
# [DEF:TranslationJob:Class]
|
||||
# @PURPOSE: A translation job representing a multi-dialect conversion task with column mappings, LLM config, and dictionary attachments.
|
||||
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
|
||||
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity.
|
||||
class TranslationJob(Base):
|
||||
__tablename__ = "translation_jobs"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
name = Column(String, nullable=False)
|
||||
description = Column(Text, nullable=True)
|
||||
source_dialect = Column(String, nullable=False)
|
||||
target_dialect = Column(String, nullable=False)
|
||||
database_dialect = Column(String, nullable=True, comment="Detected dialect from Superset connection at save time")
|
||||
status = Column(String, nullable=False, default="DRAFT") # DRAFT, READY, RUNNING, COMPLETED, FAILED, CANCELLED
|
||||
|
||||
# Datasource & target table configuration
|
||||
source_datasource_id = Column(String, nullable=True, comment="Superset datasource ID")
|
||||
source_table = Column(String, nullable=True, comment="Source table name resolved from datasource")
|
||||
target_schema = Column(String, nullable=True, comment="Target table schema")
|
||||
target_table = Column(String, nullable=True, comment="Target table name")
|
||||
|
||||
# Column mapping
|
||||
source_key_cols = Column(JSON, nullable=True, comment="Source key column names for composite key")
|
||||
target_key_cols = Column(JSON, nullable=True, comment="Target key column names for composite key")
|
||||
translation_column = Column(String, nullable=True, comment="The column whose values will be translated")
|
||||
context_columns = Column(JSON, nullable=True, comment="Context column names included in LLM prompt")
|
||||
|
||||
# LLM & processing settings
|
||||
target_language = Column(String, nullable=True, comment="Target language code (e.g. en, ru)")
|
||||
provider_id = Column(String, nullable=True, comment="LLM provider ID")
|
||||
batch_size = Column(Integer, nullable=False, default=50, comment="Records per batch")
|
||||
upsert_strategy = Column(String, nullable=False, default="MERGE", comment="MERGE, INSERT, UPDATE")
|
||||
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
|
||||
# [/DEF:TranslationJob:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationRun:Class]
|
||||
# @PURPOSE: Represents a single execution of a translation job, capturing status, timing, and Superset execution metadata.
|
||||
class TranslationRun(Base):
|
||||
__tablename__ = "translation_runs"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
||||
status = Column(String, nullable=False, default="PENDING") # PENDING, RUNNING, COMPLETED, FAILED, CANCELLED
|
||||
trigger_type = Column(String, nullable=True, comment="manual, scheduled, retry, baseline_expired")
|
||||
started_at = Column(DateTime, nullable=True)
|
||||
completed_at = Column(DateTime, nullable=True)
|
||||
error_message = Column(Text, nullable=True)
|
||||
total_records = Column(Integer, default=0)
|
||||
successful_records = Column(Integer, default=0)
|
||||
failed_records = Column(Integer, default=0)
|
||||
skipped_records = Column(Integer, default=0)
|
||||
insert_status = Column(String, nullable=True, comment="Status of the Superset insert/update operation")
|
||||
superset_execution_id = Column(String, nullable=True, comment="Superset execution/task ID")
|
||||
superset_execution_log = Column(JSON, nullable=True, comment="Superset execution log output")
|
||||
config_snapshot = Column(JSON, nullable=True, comment="Snapshot of job config at run creation time")
|
||||
key_hash = Column(String, nullable=True, comment="Hash of source key fields for dedup")
|
||||
config_hash = Column(String, nullable=True, comment="Hash of translation configuration state")
|
||||
dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at run time")
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
# [/DEF:TranslationRun:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationBatch:Class]
|
||||
# @PURPOSE: Groups translation records within a run into manageable batches with timing and record counts.
|
||||
class TranslationBatch(Base):
|
||||
__tablename__ = "translation_batches"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=False, index=True)
|
||||
batch_index = Column(Integer, nullable=False)
|
||||
status = Column(String, nullable=False, default="PENDING")
|
||||
total_records = Column(Integer, default=0)
|
||||
successful_records = Column(Integer, default=0)
|
||||
failed_records = Column(Integer, default=0)
|
||||
started_at = Column(DateTime, nullable=True)
|
||||
completed_at = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
# [/DEF:TranslationBatch:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationRecord:Class]
|
||||
# @PURPOSE: Individual translation result for a single SQL statement or dashboard element, tracking source, target, and error state.
|
||||
class TranslationRecord(Base):
|
||||
__tablename__ = "translation_records"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
batch_id = Column(String, ForeignKey("translation_batches.id"), nullable=False, index=True)
|
||||
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=False, index=True)
|
||||
source_sql = Column(Text, nullable=True)
|
||||
target_sql = Column(Text, nullable=True)
|
||||
source_object_type = Column(String, nullable=True) # query, dashboard, chart, dataset
|
||||
source_object_id = Column(String, nullable=True)
|
||||
source_object_name = Column(String, nullable=True)
|
||||
status = Column(String, nullable=False, default="PENDING") # PENDING, SUCCESS, FAILED, SKIPPED
|
||||
error_message = Column(Text, nullable=True)
|
||||
token_count_input = Column(Integer, nullable=True)
|
||||
token_count_output = Column(Integer, nullable=True)
|
||||
translation_duration_ms = Column(Integer, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_translation_records_run_status", "run_id", "status"),
|
||||
)
|
||||
# [/DEF:TranslationRecord:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationEvent:Class]
|
||||
# @PURPOSE: Audit/event log for translation operations, with optional run_id for context.
|
||||
class TranslationEvent(Base):
|
||||
__tablename__ = "translation_events"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
||||
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=True, index=True)
|
||||
event_type = Column(String, nullable=False) # JOB_CREATED, JOB_STARTED, JOB_COMPLETED, JOB_FAILED, etc.
|
||||
event_data = Column(JSON, nullable=True)
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
# [/DEF:TranslationEvent:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationPreviewSession:Class]
|
||||
# @PURPOSE: A preview session allowing users to review proposed translations before applying them.
|
||||
class TranslationPreviewSession(Base):
|
||||
__tablename__ = "translation_preview_sessions"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
||||
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=True)
|
||||
status = Column(String, nullable=False, default="ACTIVE") # ACTIVE, APPLIED, DISCARDED, EXPIRED
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
expires_at = Column(DateTime, nullable=True)
|
||||
# [/DEF:TranslationPreviewSession:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationPreviewRecord:Class]
|
||||
# @PURPOSE: Individual preview entry within a preview session, showing original and translated content side by side.
|
||||
class TranslationPreviewRecord(Base):
|
||||
__tablename__ = "translation_preview_records"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
session_id = Column(String, ForeignKey("translation_preview_sessions.id"), nullable=False, index=True)
|
||||
source_sql = Column(Text, nullable=True)
|
||||
target_sql = Column(Text, nullable=True)
|
||||
source_object_type = Column(String, nullable=True)
|
||||
source_object_id = Column(String, nullable=True)
|
||||
source_object_name = Column(String, nullable=True)
|
||||
status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED
|
||||
feedback = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
# [/DEF:TranslationPreviewRecord:Class]
|
||||
|
||||
|
||||
# [DEF:TerminologyDictionary:Class]
|
||||
# @PURPOSE: A named collection of terminology mappings used during translation to ensure consistent term translation.
|
||||
class TerminologyDictionary(Base):
|
||||
__tablename__ = "terminology_dictionaries"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
name = Column(String, nullable=False)
|
||||
description = Column(Text, nullable=True)
|
||||
source_dialect = Column(String, nullable=False)
|
||||
target_dialect = Column(String, nullable=False)
|
||||
is_active = Column(Boolean, default=True)
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
|
||||
# [/DEF:TerminologyDictionary:Class]
|
||||
|
||||
|
||||
# [DEF:DictionaryEntry:Class]
|
||||
# @PURPOSE: A single term mapping entry with case-insensitive unique constraint on source_term_normalized and origin tracking.
|
||||
class DictionaryEntry(Base):
|
||||
__tablename__ = "dictionary_entries"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
dictionary_id = Column(String, ForeignKey("terminology_dictionaries.id"), nullable=False, index=True)
|
||||
source_term = Column(String, nullable=False)
|
||||
source_term_normalized = Column(String, nullable=False)
|
||||
target_term = Column(String, nullable=False)
|
||||
context_notes = Column(Text, nullable=True)
|
||||
origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated")
|
||||
origin_row_key = Column(String, nullable=True, comment="Row key within the run that triggered this correction")
|
||||
origin_user_id = Column(String, nullable=True, comment="User who submitted the correction")
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"dictionary_id", "source_term_normalized",
|
||||
name="uq_dictionary_entry_term"
|
||||
),
|
||||
)
|
||||
# [/DEF:DictionaryEntry:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationSchedule:Class]
|
||||
# @PURPOSE: Defines a cron-based schedule for recurring translation jobs.
|
||||
class TranslationSchedule(Base):
|
||||
__tablename__ = "translation_schedules"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
||||
cron_expression = Column(String, nullable=False)
|
||||
timezone = Column(String, nullable=False, default="UTC")
|
||||
is_active = Column(Boolean, default=True)
|
||||
last_run_at = Column(DateTime, nullable=True)
|
||||
next_run_at = Column(DateTime, nullable=True)
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
|
||||
# [/DEF:TranslationSchedule:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationJobDictionary:Class]
|
||||
# @PURPOSE: Many-to-many association between translation jobs and terminology dictionaries.
|
||||
class TranslationJobDictionary(Base):
|
||||
__tablename__ = "translation_job_dictionaries"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
||||
dictionary_id = Column(String, ForeignKey("terminology_dictionaries.id"), nullable=False, index=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"job_id", "dictionary_id",
|
||||
name="uq_job_dictionary"
|
||||
),
|
||||
)
|
||||
# [/DEF:TranslationJobDictionary:Class]
|
||||
|
||||
|
||||
# [DEF:MetricSnapshot:Class]
|
||||
# @PURPOSE: Captures translation performance metrics at a point in time with config/content/dictionary hash keys for aggregation.
|
||||
class MetricSnapshot(Base):
|
||||
__tablename__ = "translation_metric_snapshots"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
||||
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=True, index=True)
|
||||
key_hash = Column(String, nullable=False, comment="Hash of dimension key fields for aggregation")
|
||||
config_hash = Column(String, nullable=True, comment="Hash of translation configuration state")
|
||||
dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at capture time")
|
||||
covers_events_before = Column(DateTime, nullable=True, comment="Indicates snapshot covers events before this timestamp")
|
||||
total_jobs = Column(Integer, default=0)
|
||||
total_runs = Column(Integer, default=0)
|
||||
total_records = Column(Integer, default=0)
|
||||
successful_records = Column(Integer, default=0)
|
||||
failed_records = Column(Integer, default=0)
|
||||
skipped_records = Column(Integer, default=0)
|
||||
avg_duration_ms = Column(Integer, nullable=True)
|
||||
p50_duration_ms = Column(Integer, nullable=True)
|
||||
p95_duration_ms = Column(Integer, nullable=True)
|
||||
p99_duration_ms = Column(Integer, nullable=True)
|
||||
snapshot_date = Column(DateTime, nullable=False, default=lambda: datetime.now(timezone.utc))
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_metric_snapshots_job_date", "job_id", "snapshot_date"),
|
||||
)
|
||||
# [/DEF:MetricSnapshot:Class]
|
||||
|
||||
# [/DEF:TranslateModels:Module]
|
||||
3
backend/src/plugins/translate/__init__.py
Normal file
3
backend/src/plugins/translate/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
# [DEF:TranslatePluginPackage:Package]
|
||||
# @PURPOSE: Translation plugin package for LLM-based cross-Superset SQL/dashboard translation.
|
||||
# [/DEF:TranslatePluginPackage:Package]
|
||||
3
backend/src/plugins/translate/__tests__/__init__.py
Normal file
3
backend/src/plugins/translate/__tests__/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
# [DEF:TranslatePluginTestsPackage:Package]
|
||||
# @PURPOSE: Tests for the translate plugin package.
|
||||
# [/DEF:TranslatePluginTestsPackage:Package]
|
||||
644
backend/src/plugins/translate/__tests__/test_dictionary.py
Normal file
644
backend/src/plugins/translate/__tests__/test_dictionary.py
Normal file
@@ -0,0 +1,644 @@
|
||||
# [DEF:DictionaryTests:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @SEMANTICS: tests, dictionary, crud, import, filter
|
||||
# @PURPOSE: Validate DictionaryManager CRUD, import, deletion guards, and batch filtering.
|
||||
# @RELATION: BINDS_TO -> [DictionaryManager:Class]
|
||||
#
|
||||
# @TEST_CONTRACT: [DictionaryManager] -> {
|
||||
# invariants: [
|
||||
# "Create/Read/Update/Delete dictionaries works",
|
||||
# "Entry CRUD enforces unique source_term_normalized per dictionary",
|
||||
# "Import CSV/TSV handles overwrite/keep_existing/cancel conflict modes",
|
||||
# "Delete blocked when dictionary attached to active/scheduled jobs",
|
||||
# "filter_for_batch returns matched entries with word-boundary awareness"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_EDGE: duplicate_entry -> 409-style ValueError on repeated source_term
|
||||
# @TEST_EDGE: delete_active_job -> ValueError with active/scheduled message
|
||||
# @TEST_EDGE: import_invalid_format -> ValueError for missing columns
|
||||
# @TEST_INVARIANT: unique_normalized -> verifies: [duplicate_entry]
|
||||
|
||||
import pytest
|
||||
import csv
|
||||
import io
|
||||
from datetime import datetime, timezone
|
||||
from sqlalchemy import create_engine, event
|
||||
from sqlalchemy.orm import sessionmaker, Session
|
||||
|
||||
from src.models.translate import (
|
||||
Base,
|
||||
TerminologyDictionary,
|
||||
DictionaryEntry,
|
||||
TranslationJob,
|
||||
TranslationJobDictionary,
|
||||
TranslationRun,
|
||||
)
|
||||
from src.plugins.translate.dictionary import DictionaryManager
|
||||
from src.plugins.translate._utils import _normalize_term, _detect_delimiter
|
||||
|
||||
|
||||
# [DEF:_FakeJob:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Helper to create inline TranslationJob records.
|
||||
class _FakeJob:
|
||||
pass
|
||||
# [/DEF:_FakeJob:Class]
|
||||
|
||||
|
||||
# [DEF:db_session:Fixture]
|
||||
# @PURPOSE: Provide an in-memory SQLite session for each test, with tables created and torn down.
|
||||
@pytest.fixture
|
||||
def db_session():
|
||||
engine = create_engine("sqlite:///:memory:", echo=False)
|
||||
|
||||
# Enable WAL and foreign keys for SQLite
|
||||
@event.listens_for(engine, "connect")
|
||||
def _set_sqlite_pragma(dbapi_connection, connection_record):
|
||||
cursor = dbapi_connection.cursor()
|
||||
cursor.execute("PRAGMA foreign_keys=ON")
|
||||
cursor.close()
|
||||
|
||||
Base.metadata.create_all(bind=engine)
|
||||
session = sessionmaker(bind=engine)()
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
session.close()
|
||||
Base.metadata.drop_all(bind=engine)
|
||||
# [/DEF:db_session:Fixture]
|
||||
|
||||
|
||||
# [DEF:test_create_dictionary:Function]
|
||||
# @PURPOSE: Verify dictionary creation and read-back.
|
||||
def test_create_dictionary(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session,
|
||||
name="Finance Terms",
|
||||
source_dialect="postgresql",
|
||||
target_dialect="clickhouse",
|
||||
created_by="test_user",
|
||||
description="Finance-related term mappings",
|
||||
)
|
||||
assert d.id is not None
|
||||
assert d.name == "Finance Terms"
|
||||
assert d.source_dialect == "postgresql"
|
||||
assert d.target_dialect == "clickhouse"
|
||||
assert d.created_by == "test_user"
|
||||
assert d.is_active is True
|
||||
|
||||
# Read back
|
||||
fetched = DictionaryManager.get_dictionary(db_session, d.id)
|
||||
assert fetched.id == d.id
|
||||
assert fetched.name == "Finance Terms"
|
||||
# [/DEF:test_create_dictionary:Function]
|
||||
|
||||
|
||||
# [DEF:test_update_dictionary:Function]
|
||||
# @PURPOSE: Verify dictionary metadata update.
|
||||
def test_update_dictionary(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Old Name",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
updated = DictionaryManager.update_dictionary(
|
||||
db_session, d.id,
|
||||
name="New Name",
|
||||
description="Updated desc",
|
||||
is_active=False,
|
||||
)
|
||||
assert updated.name == "New Name"
|
||||
assert updated.description == "Updated desc"
|
||||
assert updated.is_active is False
|
||||
# [/DEF:test_update_dictionary:Function]
|
||||
|
||||
|
||||
# [DEF:test_delete_dictionary:Function]
|
||||
# @PURPOSE: Verify dictionary deletion.
|
||||
def test_delete_dictionary(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="To Delete",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
# Add an entry
|
||||
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
assert entry.id is not None
|
||||
|
||||
DictionaryManager.delete_dictionary(db_session, d.id)
|
||||
with pytest.raises(ValueError, match="Dictionary not found"):
|
||||
DictionaryManager.get_dictionary(db_session, d.id)
|
||||
# [/DEF:test_delete_dictionary:Function]
|
||||
|
||||
|
||||
# [DEF:test_list_dictionaries:Function]
|
||||
# @PURPOSE: Verify paginated dictionary listing.
|
||||
def test_list_dictionaries(db_session: Session):
|
||||
for i in range(5):
|
||||
DictionaryManager.create_dictionary(
|
||||
db_session, name=f"Dict {i}",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
|
||||
dicts, total = DictionaryManager.list_dictionaries(db_session, page=1, page_size=2)
|
||||
assert total == 5
|
||||
assert len(dicts) == 2
|
||||
# [/DEF:test_list_dictionaries:Function]
|
||||
|
||||
|
||||
# [DEF:test_add_entry_duplicate:Function]
|
||||
# @PURPOSE: Verify duplicate entry raises ValueError and unique constraint is enforced.
|
||||
def test_add_entry_duplicate(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "Hello", "Hola")
|
||||
|
||||
# Same normalized term should raise
|
||||
with pytest.raises(ValueError, match="already exists"):
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "Bonjour")
|
||||
|
||||
# Different case, same normalized should also raise
|
||||
with pytest.raises(ValueError, match="already exists"):
|
||||
DictionaryManager.add_entry(db_session, d.id, "HELLO", "Ciao")
|
||||
# [/DEF:test_add_entry_duplicate:Function]
|
||||
|
||||
|
||||
# [DEF:test_add_entry_duplicate_per_dictionary:Function]
|
||||
# @PURPOSE: Verify duplicate is per-dictionary (same term in different dictionaries is OK).
|
||||
def test_add_entry_duplicate_per_dictionary(db_session: Session):
|
||||
d1 = DictionaryManager.create_dictionary(
|
||||
db_session, name="Dict1",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
d2 = DictionaryManager.create_dictionary(
|
||||
db_session, name="Dict2",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d1.id, "hello", "hola")
|
||||
# Same term in different dictionary should work
|
||||
entry = DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour")
|
||||
assert entry.id is not None
|
||||
# [/DEF:test_add_entry_duplicate_per_dictionary:Function]
|
||||
|
||||
|
||||
# [DEF:test_edit_entry:Function]
|
||||
# @PURPOSE: Verify entry edit updates fields and enforces uniqueness.
|
||||
def test_edit_entry(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
|
||||
updated = DictionaryManager.edit_entry(db_session, entry.id, target_term="HOLA!")
|
||||
assert updated.target_term == "HOLA!"
|
||||
|
||||
# Edit source term to a value that's already taken should fail
|
||||
DictionaryManager.add_entry(db_session, d.id, "world", "mundo")
|
||||
with pytest.raises(ValueError, match="already exists"):
|
||||
DictionaryManager.edit_entry(db_session, entry.id, source_term="WORLD")
|
||||
# [/DEF:test_edit_entry:Function]
|
||||
|
||||
|
||||
# [DEF:test_delete_entry:Function]
|
||||
# @PURPOSE: Verify entry deletion.
|
||||
def test_delete_entry(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
DictionaryManager.delete_entry(db_session, entry.id)
|
||||
|
||||
entries, total = DictionaryManager.list_entries(db_session, d.id)
|
||||
assert total == 0
|
||||
# [/DEF:test_delete_entry:Function]
|
||||
|
||||
|
||||
# [DEF:test_import_csv_overwrite:Function]
|
||||
# @PURPOSE: Verify CSV import with overwrite conflict mode.
|
||||
def test_import_csv_overwrite(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
# Pre-add an entry
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
|
||||
csv_content = "source_term,target_term,context_notes\nhello,HELLO,\nworld,mundo,"
|
||||
result = DictionaryManager.import_entries(
|
||||
db_session, d.id, csv_content,
|
||||
delimiter=",", on_conflict="overwrite",
|
||||
)
|
||||
assert result["total"] == 2
|
||||
assert result["created"] == 1 # world
|
||||
assert result["updated"] == 1 # hello (overwritten)
|
||||
assert result["skipped"] == 0
|
||||
|
||||
# Verify update
|
||||
entries, _ = DictionaryManager.list_entries(db_session, d.id)
|
||||
hello_entry = [e for e in entries if e.source_term_normalized == "hello"][0]
|
||||
assert hello_entry.target_term == "HELLO"
|
||||
# [/DEF:test_import_csv_overwrite:Function]
|
||||
|
||||
|
||||
# [DEF:test_import_csv_keep_existing:Function]
|
||||
# @PURPOSE: Verify CSV import with keep_existing conflict mode.
|
||||
def test_import_csv_keep_existing(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
|
||||
csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo"
|
||||
result = DictionaryManager.import_entries(
|
||||
db_session, d.id, csv_content,
|
||||
delimiter=",", on_conflict="keep_existing",
|
||||
)
|
||||
assert result["created"] == 1 # world
|
||||
assert result["updated"] == 0
|
||||
assert result["skipped"] == 1 # hello kept
|
||||
|
||||
# Verify hello was NOT overwritten
|
||||
entries, _ = DictionaryManager.list_entries(db_session, d.id)
|
||||
hello_entry = [e for e in entries if e.source_term_normalized == "hello"][0]
|
||||
assert hello_entry.target_term == "hola"
|
||||
# [/DEF:test_import_csv_keep_existing:Function]
|
||||
|
||||
|
||||
# [DEF:test_import_csv_cancel_on_conflict:Function]
|
||||
# @PURPOSE: Verify CSV import with cancel conflict mode raises errors.
|
||||
def test_import_csv_cancel_on_conflict(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
|
||||
csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo"
|
||||
result = DictionaryManager.import_entries(
|
||||
db_session, d.id, csv_content,
|
||||
delimiter=",", on_conflict="cancel",
|
||||
)
|
||||
assert result["total"] == 2
|
||||
assert result["created"] == 1 # world
|
||||
assert result["updated"] == 0
|
||||
assert len(result["errors"]) == 1 # hello conflict error
|
||||
assert "Conflict" in result["errors"][0]["error"]
|
||||
# [/DEF:test_import_csv_cancel_on_conflict:Function]
|
||||
|
||||
|
||||
# [DEF:test_import_tsv:Function]
|
||||
# @PURPOSE: Verify TSV import.
|
||||
def test_import_tsv(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
tsv_content = "source_term\ttarget_term\nhello\thola\nworld\tmundo"
|
||||
result = DictionaryManager.import_entries(
|
||||
db_session, d.id, tsv_content,
|
||||
delimiter="\t", on_conflict="overwrite",
|
||||
)
|
||||
assert result["created"] == 2
|
||||
assert result["total"] == 2
|
||||
# [/DEF:test_import_tsv:Function]
|
||||
|
||||
|
||||
# [DEF:test_import_invalid_format:Function]
|
||||
# @PURPOSE: Verify import raises ValueError for missing required columns.
|
||||
def test_import_invalid_format(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
bad_content = "name,value\nhello,hola"
|
||||
with pytest.raises(ValueError, match="source_term"):
|
||||
DictionaryManager.import_entries(
|
||||
db_session, d.id, bad_content,
|
||||
delimiter=",", on_conflict="overwrite",
|
||||
)
|
||||
# [/DEF:test_import_invalid_format:Function]
|
||||
|
||||
|
||||
# [DEF:test_import_empty_rows:Function]
|
||||
# @PURPOSE: Verify import handles rows with missing fields gracefully.
|
||||
def test_import_empty_rows(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
csv_content = "source_term,target_term\nhello,hola\n,world\nfoo,"
|
||||
result = DictionaryManager.import_entries(
|
||||
db_session, d.id, csv_content,
|
||||
delimiter=",", on_conflict="overwrite",
|
||||
)
|
||||
assert result["created"] == 1 # hello
|
||||
assert len(result["errors"]) == 2 # empty source or target
|
||||
# [/DEF:test_import_empty_rows:Function]
|
||||
|
||||
|
||||
# [DEF:test_import_preview:Function]
|
||||
# @PURPOSE: Verify import preview returns conflicts without mutating.
|
||||
def test_import_preview(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
|
||||
csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo"
|
||||
result = DictionaryManager.import_entries(
|
||||
db_session, d.id, csv_content,
|
||||
delimiter=",", on_conflict="overwrite",
|
||||
preview_only=True,
|
||||
)
|
||||
assert len(result["preview"]) == 2
|
||||
# First row should be conflict
|
||||
assert result["preview"][0]["is_conflict"] is True
|
||||
assert result["preview"][0]["existing_target_term"] == "hola"
|
||||
# Second row should be new
|
||||
assert result["preview"][1]["is_conflict"] is False
|
||||
|
||||
# Verify no mutation happened
|
||||
entries, total = DictionaryManager.list_entries(db_session, d.id)
|
||||
assert total == 1 # still only the original entry
|
||||
# [/DEF:test_import_preview:Function]
|
||||
|
||||
|
||||
# [DEF:test_delete_dictionary_blocked_by_active_job:Function]
|
||||
# @PURPOSE: Verify deletion is blocked when dictionary is attached to active/scheduled jobs.
|
||||
def test_delete_dictionary_blocked_by_active_job(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
# Create an active job
|
||||
job = TranslationJob(
|
||||
name="Active Job",
|
||||
source_dialect="a",
|
||||
target_dialect="b",
|
||||
status="ACTIVE",
|
||||
created_by="test_user",
|
||||
)
|
||||
db_session.add(job)
|
||||
db_session.flush()
|
||||
|
||||
# Attach dictionary to job
|
||||
link = TranslationJobDictionary(
|
||||
job_id=job.id,
|
||||
dictionary_id=d.id,
|
||||
)
|
||||
db_session.add(link)
|
||||
db_session.commit()
|
||||
|
||||
with pytest.raises(ValueError, match="active/scheduled"):
|
||||
DictionaryManager.delete_dictionary(db_session, d.id)
|
||||
# [/DEF:test_delete_dictionary_blocked_by_active_job:Function]
|
||||
|
||||
|
||||
# [DEF:test_delete_dictionary_allowed_with_completed_job:Function]
|
||||
# @PURPOSE: Verify deletion is allowed when only completed/failed jobs reference the dictionary.
|
||||
def test_delete_dictionary_allowed_with_completed_job(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
# Create a completed job
|
||||
job = TranslationJob(
|
||||
name="Completed Job",
|
||||
source_dialect="a",
|
||||
target_dialect="b",
|
||||
status="COMPLETED",
|
||||
created_by="test_user",
|
||||
)
|
||||
db_session.add(job)
|
||||
db_session.flush()
|
||||
|
||||
link = TranslationJobDictionary(
|
||||
job_id=job.id,
|
||||
dictionary_id=d.id,
|
||||
)
|
||||
db_session.add(link)
|
||||
db_session.commit()
|
||||
|
||||
# Should not raise
|
||||
DictionaryManager.delete_dictionary(db_session, d.id)
|
||||
with pytest.raises(ValueError, match="Dictionary not found"):
|
||||
DictionaryManager.get_dictionary(db_session, d.id)
|
||||
# [/DEF:test_delete_dictionary_allowed_with_completed_job:Function]
|
||||
|
||||
|
||||
# [DEF:test_filter_for_batch_no_dictionaries:Function]
|
||||
# @PURPOSE: Verify filter_for_batch returns empty when job has no dictionaries.
|
||||
def test_filter_for_batch_no_dictionaries(db_session: Session):
|
||||
job = TranslationJob(
|
||||
name="No Dict Job",
|
||||
source_dialect="a", target_dialect="b",
|
||||
status="DRAFT",
|
||||
)
|
||||
db_session.add(job)
|
||||
db_session.commit()
|
||||
|
||||
result = DictionaryManager.filter_for_batch(db_session, ["hello world"], job.id)
|
||||
assert result == []
|
||||
# [/DEF:test_filter_for_batch_no_dictionaries:Function]
|
||||
|
||||
|
||||
# [DEF:test_filter_for_batch_matches:Function]
|
||||
# @PURPOSE: Verify filter_for_batch returns correct matched entries with word-boundary awareness.
|
||||
def test_filter_for_batch_matches(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test Dict",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
DictionaryManager.add_entry(db_session, d.id, "world", "mundo")
|
||||
DictionaryManager.add_entry(db_session, d.id, "foo", "bar")
|
||||
|
||||
job = TranslationJob(
|
||||
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
|
||||
)
|
||||
db_session.add(job)
|
||||
db_session.flush()
|
||||
|
||||
link = TranslationJobDictionary(job_id=job.id, dictionary_id=d.id)
|
||||
db_session.add(link)
|
||||
db_session.commit()
|
||||
|
||||
result = DictionaryManager.filter_for_batch(
|
||||
db_session,
|
||||
["hello beautiful world", "nothing here", "foo bar"],
|
||||
job.id,
|
||||
)
|
||||
|
||||
assert len(result) >= 3 # hello, world, foo
|
||||
|
||||
# Check that 'hello' matched in text 0
|
||||
hello_matches = [m for m in result if m["source_term"] == "hello"]
|
||||
assert len(hello_matches) == 1
|
||||
assert hello_matches[0]["text_index"] == 0
|
||||
|
||||
# Check that 'world' matched in text 0
|
||||
world_matches = [m for m in result if m["source_term"] == "world"]
|
||||
assert len(world_matches) == 1
|
||||
assert world_matches[0]["text_index"] == 0
|
||||
|
||||
# Check that 'foo' matched in text 2
|
||||
foo_matches = [m for m in result if m["source_term"] == "foo"]
|
||||
assert len(foo_matches) == 1
|
||||
assert foo_matches[0]["text_index"] == 2
|
||||
# [/DEF:test_filter_for_batch_matches:Function]
|
||||
|
||||
|
||||
# [DEF:test_filter_for_batch_case_insensitive:Function]
|
||||
# @PURPOSE: Verify filter_for_batch matching is case-insensitive.
|
||||
def test_filter_for_batch_case_insensitive(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test Dict",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "Hello World", "Hola Mundo")
|
||||
|
||||
job = TranslationJob(
|
||||
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
|
||||
)
|
||||
db_session.add(job)
|
||||
db_session.flush()
|
||||
|
||||
link = TranslationJobDictionary(job_id=job.id, dictionary_id=d.id)
|
||||
db_session.add(link)
|
||||
db_session.commit()
|
||||
|
||||
# Different case should still match
|
||||
result = DictionaryManager.filter_for_batch(
|
||||
db_session,
|
||||
["hello world is great"],
|
||||
job.id,
|
||||
)
|
||||
assert len(result) == 1
|
||||
assert result[0]["source_term"] == "Hello World"
|
||||
assert result[0]["target_term"] == "Hola Mundo"
|
||||
# [/DEF:test_filter_for_batch_case_insensitive:Function]
|
||||
|
||||
|
||||
# [DEF:test_filter_for_batch_word_boundary:Function]
|
||||
# @PURPOSE: Verify filter_for_batch respects word boundaries (no substring matching within words).
|
||||
def test_filter_for_batch_word_boundary(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test Dict",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "cat", "gato")
|
||||
|
||||
job = TranslationJob(
|
||||
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
|
||||
)
|
||||
db_session.add(job)
|
||||
db_session.flush()
|
||||
|
||||
link = TranslationJobDictionary(job_id=job.id, dictionary_id=d.id)
|
||||
db_session.add(link)
|
||||
db_session.commit()
|
||||
|
||||
# "cat" in "catalog" should NOT match (word boundary)
|
||||
result = DictionaryManager.filter_for_batch(
|
||||
db_session,
|
||||
["catalog"],
|
||||
job.id,
|
||||
)
|
||||
assert len(result) == 0
|
||||
|
||||
# "cat" as standalone word should match
|
||||
result = DictionaryManager.filter_for_batch(
|
||||
db_session,
|
||||
["the cat sat"],
|
||||
job.id,
|
||||
)
|
||||
assert len(result) == 1
|
||||
# [/DEF:test_filter_for_batch_word_boundary:Function]
|
||||
|
||||
|
||||
# [DEF:test_filter_for_batch_multi_dictionary_priority:Function]
|
||||
# @PURPOSE: Verify filter_for_batch respects dictionary link order priority.
|
||||
def test_filter_for_batch_multi_dictionary_priority(db_session: Session):
|
||||
d1 = DictionaryManager.create_dictionary(
|
||||
db_session, name="Priority1", source_dialect="a", target_dialect="b",
|
||||
)
|
||||
d2 = DictionaryManager.create_dictionary(
|
||||
db_session, name="Priority2", source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d1.id, "hello", "hola")
|
||||
DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour")
|
||||
|
||||
job = TranslationJob(
|
||||
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
|
||||
)
|
||||
db_session.add(job)
|
||||
db_session.flush()
|
||||
|
||||
# Add d1 first (higher priority), then d2
|
||||
link1 = TranslationJobDictionary(job_id=job.id, dictionary_id=d1.id)
|
||||
link2 = TranslationJobDictionary(job_id=job.id, dictionary_id=d2.id)
|
||||
db_session.add(link1)
|
||||
db_session.add(link2)
|
||||
db_session.commit()
|
||||
|
||||
result = DictionaryManager.filter_for_batch(
|
||||
db_session,
|
||||
["hello"],
|
||||
job.id,
|
||||
)
|
||||
assert len(result) == 2
|
||||
# d1 match should come first (higher priority by link order)
|
||||
assert result[0]["dictionary_id"] == d1.id
|
||||
assert result[0]["target_term"] == "hola"
|
||||
assert result[1]["dictionary_id"] == d2.id
|
||||
assert result[1]["target_term"] == "bonjour"
|
||||
# [/DEF:test_filter_for_batch_multi_dictionary_priority:Function]
|
||||
|
||||
|
||||
# [DEF:test_normalize_term:Function]
|
||||
# @PURPOSE: Verify _normalize_term produces lowercase NFC-normalized strings.
|
||||
def test_normalize_term():
|
||||
assert _normalize_term("Hello") == "hello"
|
||||
assert _normalize_term("HELLO") == "hello"
|
||||
assert _normalize_term(" hello ") == "hello"
|
||||
# NFC normalization test
|
||||
composed = "\u00C9" # É precomposed
|
||||
decomposed = "\u0045\u0301" # E + combining acute
|
||||
assert _normalize_term(composed) == _normalize_term(decomposed)
|
||||
# [/DEF:test_normalize_term:Function]
|
||||
|
||||
|
||||
# [DEF:test_detect_delimiter:Function]
|
||||
# @PURPOSE: Verify _detect_delimiter correctly identifies CSV vs TSV.
|
||||
def test_detect_delimiter():
|
||||
csv_content = "source_term,target_term,context_notes\nhello,hola,"
|
||||
assert _detect_delimiter(csv_content) == ","
|
||||
|
||||
tsv_content = "source_term\ttarget_term\nhello\thola"
|
||||
assert _detect_delimiter(tsv_content) == "\t"
|
||||
|
||||
empty_content = ""
|
||||
assert _detect_delimiter(empty_content) == "," # default fallback
|
||||
# [/DEF:test_detect_delimiter:Function]
|
||||
|
||||
|
||||
# [DEF:test_clear_entries:Function]
|
||||
# @PURPOSE: Verify clearing all entries for a dictionary.
|
||||
def test_clear_entries(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test", source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
DictionaryManager.add_entry(db_session, d.id, "world", "mundo")
|
||||
|
||||
deleted = DictionaryManager.clear_entries(db_session, d.id)
|
||||
assert deleted == 2
|
||||
|
||||
entries, total = DictionaryManager.list_entries(db_session, d.id)
|
||||
assert total == 0
|
||||
# [/DEF:test_clear_entries:Function]
|
||||
# [/DEF:DictionaryTests:Module]
|
||||
376
backend/src/plugins/translate/__tests__/test_orchestrator.py
Normal file
376
backend/src/plugins/translate/__tests__/test_orchestrator.py
Normal file
@@ -0,0 +1,376 @@
|
||||
# [DEF:OrchestratorTests:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @SEMANTICS: test, translate, orchestrator, events
|
||||
# @PURPOSE: Tests for TranslationOrchestrator: run lifecycle, partial failure, batch retry, event invariants, NULL handling.
|
||||
# @LAYER: Test
|
||||
# @RELATION: BINDS_TO -> [TranslationOrchestrator:Module]
|
||||
# @RELATION: BINDS_TO -> [TranslationEventLog:Module]
|
||||
# @TEST_CONTRACT: TranslationOrchestrator -> start_run, execute_run, cancel_run, retry_failed_batches
|
||||
# @TEST_FIXTURE: mock_db -> MagicMock SQLAlchemy session
|
||||
# @TEST_FIXTURE: mock_config_manager -> MagicMock ConfigManager
|
||||
# @TEST_EDGE: missing_preview -> raises ValueError
|
||||
# @TEST_EDGE: invalid_run_status -> raises ValueError
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch, PropertyMock
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from src.plugins.translate.orchestrator import TranslationOrchestrator
|
||||
from src.plugins.translate.events import TranslationEventLog
|
||||
from src.models.translate import (
|
||||
TranslationJob,
|
||||
TranslationRun,
|
||||
TranslationBatch,
|
||||
TranslationRecord,
|
||||
TranslationPreviewSession,
|
||||
)
|
||||
|
||||
|
||||
# [DEF:mock_job:Function]
|
||||
# @PURPOSE: Create a mock TranslationJob with standard config.
|
||||
@pytest.fixture
|
||||
def mock_job() -> MagicMock:
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.id = "job-123"
|
||||
job.name = "Test Job"
|
||||
job.status = "ACTIVE"
|
||||
job.source_dialect = "postgresql"
|
||||
job.target_dialect = "postgresql"
|
||||
job.database_dialect = "postgresql"
|
||||
job.source_datasource_id = "42"
|
||||
job.translation_column = "name"
|
||||
job.context_columns = ["category"]
|
||||
job.target_schema = "public"
|
||||
job.target_table = "translated_data"
|
||||
job.target_key_cols = ["id"]
|
||||
job.source_key_cols = ["id"]
|
||||
job.target_language = "en"
|
||||
job.provider_id = "provider-1"
|
||||
job.batch_size = 50
|
||||
job.upsert_strategy = "MERGE"
|
||||
return job
|
||||
|
||||
|
||||
# [DEF:mock_preview_session:Function]
|
||||
# @PURPOSE: Create a mock accepted preview session.
|
||||
@pytest.fixture
|
||||
def mock_preview_session() -> MagicMock:
|
||||
session = MagicMock(spec=TranslationPreviewSession)
|
||||
session.id = "session-1"
|
||||
session.job_id = "job-123"
|
||||
session.status = "APPLIED"
|
||||
session.created_at = datetime.now(timezone.utc)
|
||||
return session
|
||||
|
||||
|
||||
# [DEF:TestTranslationEventLog:Class]
|
||||
# @PURPOSE: Tests for TranslationEventLog terminal event invariant.
|
||||
class TestTranslationEventLog:
|
||||
|
||||
# [DEF:test_log_event_creates_record:Function]
|
||||
# @PURPOSE: log_event creates a TranslationEvent row.
|
||||
def test_log_event_creates_record(self) -> None:
|
||||
db = MagicMock()
|
||||
log = TranslationEventLog(db)
|
||||
event = log.log_event(
|
||||
job_id="job-1",
|
||||
event_type="RUN_STARTED",
|
||||
payload={"key": "value"},
|
||||
run_id="run-1",
|
||||
)
|
||||
assert event is not None
|
||||
assert event.event_type == "RUN_STARTED"
|
||||
db.add.assert_called_once()
|
||||
db.flush.assert_called_once()
|
||||
|
||||
# [DEF:test_invalid_event_type_raises:Function]
|
||||
# @PURPOSE: Invalid event type raises ValueError.
|
||||
def test_invalid_event_type_raises(self) -> None:
|
||||
db = MagicMock()
|
||||
log = TranslationEventLog(db)
|
||||
with pytest.raises(ValueError, match="Invalid event_type"):
|
||||
log.log_event(
|
||||
job_id="job-1",
|
||||
event_type="UNKNOWN_EVENT",
|
||||
)
|
||||
|
||||
# [DEF:test_terminal_event_invariant:Function]
|
||||
# @PURPOSE: Only one terminal event allowed per run.
|
||||
def test_terminal_event_invariant(self) -> None:
|
||||
db = MagicMock()
|
||||
# Mock existing terminal event
|
||||
db.query.return_value.filter.return_value.first.return_value = ("event-1",)
|
||||
|
||||
log = TranslationEventLog(db)
|
||||
with pytest.raises(ValueError, match="already has a terminal event"):
|
||||
log.log_event(
|
||||
job_id="job-1",
|
||||
run_id="run-1",
|
||||
event_type="RUN_COMPLETED",
|
||||
)
|
||||
|
||||
# [DEF:test_run_started_must_precede_other_events:Function]
|
||||
# @PURPOSE: Other run events require RUN_STARTED to exist first.
|
||||
def test_run_started_must_precede_other_events(self) -> None:
|
||||
db = MagicMock()
|
||||
# Mock no RUN_STARTED event
|
||||
db.query.return_value.filter.return_value.first.side_effect = [None, None]
|
||||
|
||||
log = TranslationEventLog(db)
|
||||
with pytest.raises(ValueError, match="RUN_STARTED event must precede"):
|
||||
log.log_event(
|
||||
job_id="job-1",
|
||||
run_id="run-1",
|
||||
event_type="BATCH_STARTED",
|
||||
)
|
||||
|
||||
# [DEF:test_prune_expired_creates_snapshot:Function]
|
||||
# @PURPOSE: prune_expired creates MetricSnapshot before deleting.
|
||||
def test_prune_expired_creates_snapshot(self) -> None:
|
||||
db = MagicMock()
|
||||
# Mock that there are expired events
|
||||
db.query.return_value.filter.return_value.count.return_value = 10
|
||||
# Use side_effect to simulate data being consumed by deletion.
|
||||
# The while loop re-queries; first call returns items, second returns empty.
|
||||
db.query.return_value.filter.return_value.limit.return_value.all.side_effect = [
|
||||
[("e1",), ("e2",)],
|
||||
[],
|
||||
]
|
||||
|
||||
log = TranslationEventLog(db)
|
||||
result = log.prune_expired(retention_days=90)
|
||||
assert result["pruned"] >= 0
|
||||
# [/DEF:TestTranslationEventLog:Class]
|
||||
|
||||
|
||||
# [DEF:TestTranslationOrchestrator:Class]
|
||||
# @PURPOSE: Tests for TranslationOrchestrator run lifecycle.
|
||||
class TestTranslationOrchestrator:
|
||||
|
||||
# [DEF:test_start_run_success:Function]
|
||||
# @PURPOSE: start_run creates a PENDING run with event.
|
||||
def test_start_run_success(
|
||||
self,
|
||||
mock_service,
|
||||
mock_job: MagicMock,
|
||||
mock_preview_session: MagicMock,
|
||||
) -> None:
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
# Mock job query
|
||||
db.query.return_value.filter.return_value.first.return_value = mock_job
|
||||
|
||||
# Mock accepted preview session
|
||||
preview_query = (
|
||||
db.query.return_value.filter.return_value
|
||||
)
|
||||
preview_query.order_by.return_value.first.return_value = mock_preview_session
|
||||
|
||||
orch = TranslationOrchestrator(db, config_manager, "test-user")
|
||||
run = orch.start_run(job_id="job-123")
|
||||
|
||||
assert run.status == "PENDING"
|
||||
assert run.job_id == "job-123"
|
||||
|
||||
# [DEF:test_start_run_missing_preview_raises:Function]
|
||||
# @PURPOSE: Manual run without accepted preview raises ValueError.
|
||||
def test_start_run_missing_preview_raises(
|
||||
self,
|
||||
mock_job: MagicMock,
|
||||
) -> None:
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
# Mock job query
|
||||
db.query.return_value.filter.return_value.first.return_value = mock_job
|
||||
|
||||
# No accepted preview session
|
||||
db.query.return_value.filter.return_value.order_by.return_value.first.return_value = None
|
||||
|
||||
orch = TranslationOrchestrator(db, config_manager, "test-user")
|
||||
with pytest.raises(ValueError, match="no accepted preview"):
|
||||
orch.start_run(job_id="job-123")
|
||||
|
||||
# [DEF:test_start_run_draft_job_raises:Function]
|
||||
# @PURPOSE: Draft job cannot be run.
|
||||
def test_start_run_draft_job_raises(self) -> None:
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
draft_job = MagicMock(spec=TranslationJob)
|
||||
draft_job.status = "DRAFT"
|
||||
|
||||
db.query.return_value.filter.return_value.first.return_value = draft_job
|
||||
|
||||
orch = TranslationOrchestrator(db, config_manager, "test-user")
|
||||
with pytest.raises(ValueError, match="DRAFT"):
|
||||
orch.start_run(job_id="job-123")
|
||||
|
||||
# [DEF:test_execute_run_invalid_status:Function]
|
||||
# @PURPOSE: Cannot execute a run that is not in PENDING status.
|
||||
def test_execute_run_invalid_status(self, mock_job: MagicMock) -> None:
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
run = MagicMock(spec=TranslationRun)
|
||||
run.status = "COMPLETED"
|
||||
run.job_id = "job-123"
|
||||
|
||||
db.query.return_value.filter.return_value.first.return_value = mock_job
|
||||
|
||||
orch = TranslationOrchestrator(db, config_manager, "test-user")
|
||||
with pytest.raises(ValueError, match="PENDING"):
|
||||
orch.execute_run(run)
|
||||
|
||||
# [DEF:test_cancel_run:Function]
|
||||
# @PURPOSE: Cancel a run changes status to CANCELLED.
|
||||
def test_cancel_run(self) -> None:
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
run = MagicMock(spec=TranslationRun)
|
||||
run.id = "run-1"
|
||||
run.job_id = "job-123"
|
||||
run.status = "RUNNING"
|
||||
|
||||
db.query.return_value.filter.return_value.first.return_value = run
|
||||
|
||||
orch = TranslationOrchestrator(db, config_manager, "test-user")
|
||||
result = orch.cancel_run("run-1")
|
||||
|
||||
assert result.status == "CANCELLED"
|
||||
|
||||
# [DEF:test_cancel_run_invalid_status:Function]
|
||||
# @PURPOSE: Cannot cancel a completed run.
|
||||
def test_cancel_run_invalid_status(self) -> None:
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
run = MagicMock(spec=TranslationRun)
|
||||
run.status = "COMPLETED"
|
||||
|
||||
db.query.return_value.filter.return_value.first.return_value = run
|
||||
|
||||
orch = TranslationOrchestrator(db, config_manager, "test-user")
|
||||
with pytest.raises(ValueError, match="cancelled"):
|
||||
orch.cancel_run("run-1")
|
||||
|
||||
# [DEF:test_retry_failed_batches_no_failures:Function]
|
||||
# @PURPOSE: Raises if no failed batches found.
|
||||
def test_retry_failed_batches_no_failures(self) -> None:
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
run = MagicMock(spec=TranslationRun)
|
||||
run.id = "run-1"
|
||||
run.job_id = "job-123"
|
||||
run.status = "COMPLETED"
|
||||
|
||||
# Mock run query
|
||||
db.query.return_value.filter.return_value.first.return_value = run
|
||||
|
||||
# No failed batches
|
||||
db.query.return_value.filter.return_value.all.return_value = []
|
||||
|
||||
orch = TranslationOrchestrator(db, config_manager, "test-user")
|
||||
with pytest.raises(ValueError, match="No failed batches"):
|
||||
orch.retry_failed_batches("run-1")
|
||||
|
||||
# [DEF:test_get_run_status:Function]
|
||||
# @PURPOSE: get_run_status returns structured status.
|
||||
def test_get_run_status(self) -> None:
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
run = MagicMock(spec=TranslationRun)
|
||||
run.id = "run-1"
|
||||
run.job_id = "job-123"
|
||||
run.status = "COMPLETED"
|
||||
run.started_at = datetime.now(timezone.utc)
|
||||
run.completed_at = datetime.now(timezone.utc)
|
||||
run.error_message = None
|
||||
run.total_records = 100
|
||||
run.successful_records = 95
|
||||
run.failed_records = 3
|
||||
run.skipped_records = 2
|
||||
run.insert_status = "success"
|
||||
run.superset_execution_id = "query-1"
|
||||
run.created_by = "test-user"
|
||||
run.created_at = datetime.now(timezone.utc)
|
||||
|
||||
db.query.return_value.filter.return_value.first.return_value = run
|
||||
db.query.return_value.filter.return_value.count.return_value = 5
|
||||
|
||||
orch = TranslationOrchestrator(db, config_manager, "test-user")
|
||||
status = orch.get_run_status("run-1")
|
||||
|
||||
assert status["id"] == "run-1"
|
||||
assert status["status"] == "COMPLETED"
|
||||
assert status["total_records"] == 100
|
||||
assert status["successful_records"] == 95
|
||||
assert status["insert_status"] == "success"
|
||||
|
||||
# [DEF:test_get_run_records:Function]
|
||||
# @PURPOSE: get_run_records returns paginated records.
|
||||
def test_get_run_records(self) -> None:
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
record = MagicMock(spec=TranslationRecord)
|
||||
record.id = "rec-1"
|
||||
record.batch_id = "batch-1"
|
||||
record.run_id = "run-1"
|
||||
record.source_sql = "SELECT 1"
|
||||
record.target_sql = "SELECT 2"
|
||||
record.source_object_type = "table_row"
|
||||
record.source_object_id = "0"
|
||||
record.source_object_name = ""
|
||||
record.status = "SUCCESS"
|
||||
record.error_message = None
|
||||
record.created_at = datetime.now(timezone.utc)
|
||||
|
||||
db.query.return_value.filter.return_value.count.return_value = 1
|
||||
db.query.return_value.filter.return_value.order_by.return_value.offset.return_value.limit.return_value.all.return_value = [record]
|
||||
|
||||
orch = TranslationOrchestrator(db, config_manager, "test-user")
|
||||
result = orch.get_run_records("run-1")
|
||||
|
||||
assert result["total"] == 1
|
||||
assert len(result["items"]) == 1
|
||||
assert result["items"][0]["status"] == "SUCCESS"
|
||||
|
||||
# [DEF:test_get_run_history:Function]
|
||||
# @PURPOSE: get_run_history returns list of runs.
|
||||
def test_get_run_history(self) -> None:
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
run = MagicMock(spec=TranslationRun)
|
||||
run.id = "run-1"
|
||||
run.job_id = "job-123"
|
||||
run.status = "COMPLETED"
|
||||
run.started_at = datetime.now(timezone.utc)
|
||||
run.completed_at = datetime.now(timezone.utc)
|
||||
run.error_message = None
|
||||
run.total_records = 100
|
||||
run.successful_records = 95
|
||||
run.failed_records = 3
|
||||
run.skipped_records = 2
|
||||
run.insert_status = "success"
|
||||
run.superset_execution_id = "query-1"
|
||||
run.created_by = "test-user"
|
||||
run.created_at = datetime.now(timezone.utc)
|
||||
|
||||
db.query.return_value.filter.return_value.count.return_value = 1
|
||||
db.query.return_value.filter.return_value.order_by.return_value.offset.return_value.limit.return_value.all.return_value = [run]
|
||||
|
||||
orch = TranslationOrchestrator(db, config_manager, "test-user")
|
||||
total, runs = orch.get_run_history("job-123")
|
||||
|
||||
assert total == 1
|
||||
assert len(runs) == 1
|
||||
assert runs[0]["id"] == "run-1"
|
||||
# [/DEF:TestTranslationOrchestrator:Class]
|
||||
# [/DEF:OrchestratorTests:Module]
|
||||
500
backend/src/plugins/translate/__tests__/test_preview.py
Normal file
500
backend/src/plugins/translate/__tests__/test_preview.py
Normal file
@@ -0,0 +1,500 @@
|
||||
# [DEF:TranslationPreviewTests:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @SEMANTICS: test, translate, preview, session
|
||||
# @PURPOSE: Test TranslationPreview service: preview, approve/edit/reject state transitions, cost estimation, acceptance gate.
|
||||
# @LAYER: Test
|
||||
# @RELATION: BINDS_TO -> [TranslationPreview:Module]
|
||||
|
||||
from unittest.mock import MagicMock, patch, PropertyMock
|
||||
from datetime import datetime, timezone, timedelta
|
||||
import json
|
||||
import pytest
|
||||
|
||||
from src.models.translate import (
|
||||
TranslationJob,
|
||||
TranslationPreviewSession,
|
||||
TranslationPreviewRecord,
|
||||
TranslationJobDictionary,
|
||||
)
|
||||
from src.plugins.translate.preview import TranslationPreview, TokenEstimator
|
||||
|
||||
|
||||
# [DEF:_make_mock_job:Function]
|
||||
# @PURPOSE: Create a mock TranslationJob with test config.
|
||||
def _make_mock_job(**overrides):
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.id = overrides.get("id", "job-123")
|
||||
job.name = overrides.get("name", "Test Job")
|
||||
job.source_dialect = overrides.get("source_dialect", "postgresql")
|
||||
job.target_dialect = overrides.get("target_dialect", "clickhouse")
|
||||
job.source_datasource_id = overrides.get("source_datasource_id", "42")
|
||||
job.source_table = overrides.get("source_table", "source_table")
|
||||
job.translation_column = overrides.get("translation_column", "title")
|
||||
job.context_columns = overrides.get("context_columns", ["category", "description"])
|
||||
job.target_language = overrides.get("target_language", "ru")
|
||||
job.provider_id = overrides.get("provider_id", "provider-1")
|
||||
job.batch_size = overrides.get("batch_size", 50)
|
||||
job.upsert_strategy = overrides.get("upsert_strategy", "MERGE")
|
||||
job.status = overrides.get("status", "DRAFT")
|
||||
return job
|
||||
# [/DEF:_make_mock_job:Function]
|
||||
|
||||
|
||||
# [DEF:_make_mock_provider:Function]
|
||||
# @PURPOSE: Create a mock LLMProvider.
|
||||
def _make_mock_provider(**overrides):
|
||||
provider = MagicMock()
|
||||
provider.id = overrides.get("id", "provider-1")
|
||||
provider.provider_type = overrides.get("provider_type", "openai")
|
||||
provider.base_url = overrides.get("base_url", "https://api.openai.com/v1")
|
||||
provider.default_model = overrides.get("default_model", "gpt-4o-mini")
|
||||
provider.api_key = "encrypted-key-123"
|
||||
return provider
|
||||
# [/DEF:_make_mock_provider:Function]
|
||||
|
||||
|
||||
# [DEF:TestTranslationPreview:Class]
|
||||
# @PURPOSE: Test suite for TranslationPreview service.
|
||||
class TestTranslationPreview:
|
||||
|
||||
# [DEF:test_preview_valid_job:Function]
|
||||
# @PURPOSE: Verify preview creates session and records successfully.
|
||||
def test_preview_valid_job(self):
|
||||
"""Preview with valid job should create session and return records."""
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
# Mock job query
|
||||
job = _make_mock_job()
|
||||
db.query.return_value.filter.return_value.first.return_value = job
|
||||
|
||||
# Mock environments
|
||||
env = MagicMock()
|
||||
env.id = "postgresql"
|
||||
env.url = "https://superset.example.com"
|
||||
env.username = "admin"
|
||||
env.password = "admin"
|
||||
env.verify_ssl = True
|
||||
env.timeout = 30
|
||||
config_manager.get_environments.return_value = [env]
|
||||
|
||||
# Mock SupersetClient
|
||||
with patch("src.plugins.translate.preview.SupersetClient") as MockClient:
|
||||
mock_client = MagicMock()
|
||||
MockClient.return_value = mock_client
|
||||
|
||||
# Mock dataset_detail
|
||||
mock_client.get_dataset_detail.return_value = {
|
||||
"table_name": "test_table",
|
||||
"columns": [
|
||||
{"column_name": "title", "type": "VARCHAR"},
|
||||
{"column_name": "category", "type": "VARCHAR"},
|
||||
{"column_name": "description", "type": "TEXT"},
|
||||
],
|
||||
}
|
||||
|
||||
# Mock build_dataset_preview_query_context
|
||||
mock_client.build_dataset_preview_query_context.return_value = {
|
||||
"datasource": {"id": 42, "type": "table"},
|
||||
"queries": [{
|
||||
"columns": [],
|
||||
"metrics": ["count"],
|
||||
"row_limit": 10,
|
||||
}],
|
||||
"form_data": {},
|
||||
"result_format": "json",
|
||||
"result_type": "query",
|
||||
"force": True,
|
||||
}
|
||||
|
||||
# Mock network request to Superset
|
||||
mock_client.network.request.return_value = {
|
||||
"result": [{
|
||||
"status": "success",
|
||||
"data": [
|
||||
{"title": "Hello World", "category": "greeting", "description": "A friendly message"},
|
||||
{"title": "Goodbye", "category": "farewell", "description": "A parting message"},
|
||||
],
|
||||
}]
|
||||
}
|
||||
|
||||
# Mock LLM provider
|
||||
with patch("src.plugins.translate.preview.LLMProviderService") as MockProviderService:
|
||||
mock_provider_svc = MagicMock()
|
||||
MockProviderService.return_value = mock_provider_svc
|
||||
mock_provider = _make_mock_provider()
|
||||
mock_provider_svc.get_provider.return_value = mock_provider
|
||||
mock_provider_svc.get_decrypted_api_key.return_value = "sk-test-key"
|
||||
|
||||
# Mock _call_openai_compatible to return structured JSON
|
||||
with patch.object(TranslationPreview, "_call_openai_compatible") as mock_llm:
|
||||
mock_llm.return_value = json.dumps({
|
||||
"rows": [
|
||||
{"row_id": "0", "translation": "Привет мир"},
|
||||
{"row_id": "1", "translation": "Прощай"},
|
||||
]
|
||||
})
|
||||
|
||||
# Mock DictionaryManager.filter_for_batch
|
||||
with patch("src.plugins.translate.preview.DictionaryManager.filter_for_batch") as mock_filter:
|
||||
mock_filter.return_value = []
|
||||
|
||||
preview_service = TranslationPreview(db, config_manager, "test-user")
|
||||
result = preview_service.preview_rows(job_id="job-123", sample_size=10)
|
||||
|
||||
# Verify result structure
|
||||
assert result["job_id"] == "job-123"
|
||||
assert result["status"] == "ACTIVE"
|
||||
assert len(result["records"]) == 2
|
||||
assert result["records"][0]["target_sql"] == "Привет мир"
|
||||
assert result["records"][1]["target_sql"] == "Прощай"
|
||||
|
||||
# Verify cost estimate
|
||||
assert result["cost_estimate"]["sample_size"] == 2
|
||||
assert result["cost_estimate"]["sample_total_tokens"] > 0
|
||||
assert result["cost_estimate"]["sample_cost"] >= 0
|
||||
|
||||
# Verify config and dict hashes
|
||||
assert result["config_hash"] is not None
|
||||
assert result["dict_snapshot_hash"] is not None
|
||||
|
||||
# Verify session was added to DB
|
||||
assert db.add.called
|
||||
assert db.commit.called
|
||||
# [/DEF:test_preview_valid_job:Function]
|
||||
|
||||
# [DEF:test_preview_with_dictionary:Function]
|
||||
# @PURPOSE: Verify glossary entries from dictionary are included in LLM prompt.
|
||||
def test_preview_with_dictionary(self):
|
||||
"""Preview should include dictionary glossary in the prompt sent to LLM."""
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
job = _make_mock_job()
|
||||
db.query.return_value.filter.return_value.first.return_value = job
|
||||
|
||||
env = MagicMock()
|
||||
env.id = "postgresql"
|
||||
config_manager.get_environments.return_value = [env]
|
||||
|
||||
with patch("src.plugins.translate.preview.SupersetClient") as MockClient:
|
||||
mock_client = MagicMock()
|
||||
MockClient.return_value = mock_client
|
||||
mock_client.get_dataset_detail.return_value = {"table_name": "test", "columns": []}
|
||||
mock_client.build_dataset_preview_query_context.return_value = {
|
||||
"datasource": {"id": 42, "type": "table"},
|
||||
"queries": [{"columns": [], "metrics": ["count"], "row_limit": 10}],
|
||||
"form_data": {},
|
||||
"result_format": "json",
|
||||
"result_type": "query",
|
||||
"force": True,
|
||||
}
|
||||
mock_client.network.request.return_value = {
|
||||
"result": [{"status": "success", "data": [{"title": "Hello World", "category": "greeting"}]}]
|
||||
}
|
||||
|
||||
with patch("src.plugins.translate.preview.LLMProviderService") as MockProviderService:
|
||||
mock_provider_svc = MagicMock()
|
||||
MockProviderService.return_value = mock_provider_svc
|
||||
mock_provider_svc.get_provider.return_value = _make_mock_provider()
|
||||
mock_provider_svc.get_decrypted_api_key.return_value = "sk-test-key"
|
||||
|
||||
with patch.object(TranslationPreview, "_call_openai_compatible") as mock_llm:
|
||||
mock_llm.return_value = json.dumps({"rows": [{"row_id": "0", "translation": "Привет мир"}]})
|
||||
|
||||
# Mock dictionary filter to return glossary entries
|
||||
with patch("src.plugins.translate.preview.DictionaryManager.filter_for_batch") as mock_filter:
|
||||
mock_filter.return_value = [
|
||||
{
|
||||
"source_term": "Hello",
|
||||
"target_term": "Здравствуйте",
|
||||
"dictionary_id": "dict-1",
|
||||
"dictionary_name": "Test Dict",
|
||||
"context_notes": "Formal greeting",
|
||||
}
|
||||
]
|
||||
|
||||
preview_service = TranslationPreview(db, config_manager, "test-user")
|
||||
result = preview_service.preview_rows(job_id="job-123", sample_size=5)
|
||||
|
||||
# Verify LLM was called
|
||||
mock_llm.assert_called_once()
|
||||
assert len(result["records"]) == 1
|
||||
# [/DEF:test_preview_with_dictionary:Function]
|
||||
|
||||
# [DEF:test_preview_row_approve:Function]
|
||||
# @PURPOSE: Verify approving a preview row changes its status.
|
||||
def test_preview_row_approve(self):
|
||||
"""Approve action should set record status to APPROVED."""
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
# Mock active session
|
||||
session = MagicMock(spec=TranslationPreviewSession)
|
||||
session.id = "session-1"
|
||||
session.job_id = "job-123"
|
||||
session.status = "ACTIVE"
|
||||
session.created_at = datetime.now(timezone.utc)
|
||||
|
||||
# Mock record
|
||||
record = MagicMock(spec=TranslationPreviewRecord)
|
||||
record.id = "record-1"
|
||||
record.session_id = "session-1"
|
||||
record.status = "PENDING"
|
||||
record.target_sql = "test translation"
|
||||
record.feedback = None
|
||||
|
||||
# Setup: db.query().filter().order_by().first() -> session (session lookup)
|
||||
db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session
|
||||
# Setup: db.query().filter().first() -> record (record lookup)
|
||||
db.query.return_value.filter.return_value.first.return_value = record
|
||||
|
||||
preview_service = TranslationPreview(db, config_manager, "test-user")
|
||||
result = preview_service.update_preview_row(
|
||||
job_id="job-123",
|
||||
row_id="record-1",
|
||||
action="approve",
|
||||
)
|
||||
|
||||
assert record.status == "APPROVED"
|
||||
assert result["status"] == "APPROVED"
|
||||
assert db.commit.called
|
||||
# [/DEF:test_preview_row_approve:Function]
|
||||
|
||||
# [DEF:test_preview_row_reject:Function]
|
||||
# @PURPOSE: Verify rejecting a preview row changes its status.
|
||||
def test_preview_row_reject(self):
|
||||
"""Reject action should set record status to REJECTED."""
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
session = MagicMock(spec=TranslationPreviewSession)
|
||||
session.id = "session-1"
|
||||
session.job_id = "job-123"
|
||||
session.status = "ACTIVE"
|
||||
|
||||
record = MagicMock(spec=TranslationPreviewRecord)
|
||||
record.id = "record-1"
|
||||
record.session_id = "session-1"
|
||||
record.status = "PENDING"
|
||||
record.target_sql = "test"
|
||||
record.feedback = None
|
||||
|
||||
db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session
|
||||
db.query.return_value.filter.return_value.first.return_value = record
|
||||
|
||||
preview_service = TranslationPreview(db, config_manager, "test-user")
|
||||
result = preview_service.update_preview_row(
|
||||
job_id="job-123",
|
||||
row_id="record-1",
|
||||
action="reject",
|
||||
)
|
||||
|
||||
assert record.status == "REJECTED"
|
||||
assert result["status"] == "REJECTED"
|
||||
# [/DEF:test_preview_row_reject:Function]
|
||||
|
||||
# [DEF:test_preview_row_edit:Function]
|
||||
# @PURPOSE: Verify editing a preview row updates its translation and status.
|
||||
def test_preview_row_edit(self):
|
||||
"""Edit action should update translation and set status to APPROVED."""
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
session = MagicMock(spec=TranslationPreviewSession)
|
||||
session.id = "session-1"
|
||||
session.job_id = "job-123"
|
||||
session.status = "ACTIVE"
|
||||
|
||||
record = MagicMock(spec=TranslationPreviewRecord)
|
||||
record.id = "record-1"
|
||||
record.session_id = "session-1"
|
||||
record.status = "PENDING"
|
||||
record.target_sql = "old translation"
|
||||
record.feedback = None
|
||||
|
||||
db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session
|
||||
db.query.return_value.filter.return_value.first.return_value = record
|
||||
|
||||
preview_service = TranslationPreview(db, config_manager, "test-user")
|
||||
result = preview_service.update_preview_row(
|
||||
job_id="job-123",
|
||||
row_id="record-1",
|
||||
action="edit",
|
||||
translation="edited translation",
|
||||
)
|
||||
|
||||
assert record.target_sql == "edited translation"
|
||||
assert record.status == "APPROVED"
|
||||
assert result["status"] == "APPROVED"
|
||||
assert result["target_sql"] == "edited translation"
|
||||
# [/DEF:test_preview_row_edit:Function]
|
||||
|
||||
# [DEF:test_preview_row_invalid_action:Function]
|
||||
# @PURPOSE: Verify invalid action raises ValueError.
|
||||
def test_preview_row_invalid_action(self):
|
||||
"""Invalid action should raise ValueError."""
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
session = MagicMock(spec=TranslationPreviewSession)
|
||||
session.id = "session-1"
|
||||
session.job_id = "job-123"
|
||||
session.status = "ACTIVE"
|
||||
|
||||
record = MagicMock(spec=TranslationPreviewRecord)
|
||||
record.id = "record-1"
|
||||
record.session_id = "session-1"
|
||||
record.status = "PENDING"
|
||||
|
||||
db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session
|
||||
db.query.return_value.filter.return_value.first.return_value = record
|
||||
|
||||
preview_service = TranslationPreview(db, config_manager, "test-user")
|
||||
with pytest.raises(ValueError, match="Invalid action"):
|
||||
preview_service.update_preview_row(
|
||||
job_id="job-123",
|
||||
row_id="record-1",
|
||||
action="invalid_action",
|
||||
)
|
||||
# [/DEF:test_preview_row_invalid_action:Function]
|
||||
|
||||
# [DEF:test_preview_accept_session:Function]
|
||||
# @PURPOSE: Verify accepting a preview session gates execution.
|
||||
def test_preview_accept_session(self):
|
||||
"""Accept should set session status to APPLIED and return records."""
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
session = MagicMock(spec=TranslationPreviewSession)
|
||||
session.id = "session-1"
|
||||
session.job_id = "job-123"
|
||||
session.status = "ACTIVE"
|
||||
session.created_by = "test-user"
|
||||
session.created_at = datetime.now(timezone.utc)
|
||||
session.expires_at = datetime.now(timezone.utc) + timedelta(hours=24)
|
||||
|
||||
record = MagicMock(spec=TranslationPreviewRecord)
|
||||
record.id = "record-1"
|
||||
record.session_id = "session-1"
|
||||
record.source_sql = "Hello"
|
||||
record.target_sql = "Привет"
|
||||
record.status = "APPROVED"
|
||||
record.feedback = None
|
||||
|
||||
# Session lookup
|
||||
db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session
|
||||
# Records lookup
|
||||
db.query.return_value.filter.return_value.all.return_value = [record]
|
||||
|
||||
preview_service = TranslationPreview(db, config_manager, "test-user")
|
||||
result = preview_service.accept_preview_session(job_id="job-123")
|
||||
|
||||
assert session.status == "APPLIED"
|
||||
assert result["status"] == "APPLIED"
|
||||
assert len(result["records"]) == 1
|
||||
assert result["records"][0]["status"] == "APPROVED"
|
||||
assert db.commit.called
|
||||
# [/DEF:test_preview_accept_session:Function]
|
||||
|
||||
# [DEF:test_preview_no_active_session:Function]
|
||||
# @PURPOSE: Verify accept raises error when no active session.
|
||||
def test_preview_no_active_session(self):
|
||||
"""Accept without active session should raise ValueError."""
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
db.query.return_value.filter.return_value.order_by.return_value.first.return_value = None
|
||||
|
||||
preview_service = TranslationPreview(db, config_manager, "test-user")
|
||||
with pytest.raises(ValueError, match="No active preview session"):
|
||||
preview_service.accept_preview_session(job_id="job-123")
|
||||
# [/DEF:test_preview_no_active_session:Function]
|
||||
|
||||
# [DEF:test_cost_estimation:Function]
|
||||
# @PURPOSE: Verify token and cost estimation methods.
|
||||
def test_cost_estimation(self):
|
||||
"""Token and cost estimation should return reasonable values."""
|
||||
prompt = "Translate this text from English to Russian. " * 100
|
||||
tokens = TokenEstimator.estimate_prompt_tokens(prompt)
|
||||
assert tokens > 0
|
||||
assert tokens < len(prompt) # tokens typically fewer than chars
|
||||
|
||||
output_tokens = TokenEstimator.estimate_output_tokens(10)
|
||||
assert output_tokens == 500
|
||||
|
||||
cost = TokenEstimator.estimate_cost(1000, 0.002)
|
||||
assert cost == 0.002
|
||||
|
||||
cost_default = TokenEstimator.estimate_cost(1000)
|
||||
assert cost_default == 0.002
|
||||
# [/DEF:test_cost_estimation:Function]
|
||||
|
||||
# [DEF:test_preview_parse_llm_response:Function]
|
||||
# @PURPOSE: Verify LLM JSON response parsing.
|
||||
def test_preview_parse_llm_response(self):
|
||||
"""Parse LLM response should extract translations keyed by row_id."""
|
||||
response = json.dumps({
|
||||
"rows": [
|
||||
{"row_id": "0", "translation": "Привет"},
|
||||
{"row_id": "1", "translation": "Мир"},
|
||||
]
|
||||
})
|
||||
result = TranslationPreview._parse_llm_response(response, 2)
|
||||
assert result == {"0": "Привет", "1": "Мир"}
|
||||
# [/DEF:test_preview_parse_llm_response:Function]
|
||||
|
||||
# [DEF:test_preview_parse_llm_response_with_code_block:Function]
|
||||
# @PURPOSE: Verify LLM response parsing handles markdown code blocks.
|
||||
def test_preview_parse_llm_response_with_code_block(self):
|
||||
"""Parse LLM response should handle markdown code block wrapping."""
|
||||
response = "```json\n{\n \"rows\": [\n {\"row_id\": \"0\", \"translation\": \"Test\"}\n ]\n}\n```"
|
||||
result = TranslationPreview._parse_llm_response(response, 1)
|
||||
assert result == {"0": "Test"}
|
||||
# [/DEF:test_preview_parse_llm_response_with_code_block:Function]
|
||||
|
||||
# [DEF:test_preview_compute_config_hash:Function]
|
||||
# @PURPOSE: Verify config hash computation is deterministic.
|
||||
def test_preview_compute_config_hash(self):
|
||||
"""Config hash should be deterministic and non-empty."""
|
||||
job = _make_mock_job()
|
||||
hash1 = TranslationPreview._compute_config_hash(job)
|
||||
hash2 = TranslationPreview._compute_config_hash(job)
|
||||
assert hash1 == hash2
|
||||
assert len(hash1) == 16
|
||||
# [/DEF:test_preview_compute_config_hash:Function]
|
||||
|
||||
# [DEF:test_preview_missing_datasource:Function]
|
||||
# @PURPOSE: Verify error when job has no datasource configured.
|
||||
def test_preview_missing_datasource(self):
|
||||
"""Preview should fail if job has no datasource."""
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
job = _make_mock_job(source_datasource_id=None)
|
||||
db.query.return_value.filter.return_value.first.return_value = job
|
||||
|
||||
preview_service = TranslationPreview(db, config_manager, "test-user")
|
||||
with pytest.raises(ValueError, match="source datasource"):
|
||||
preview_service.preview_rows(job_id="job-123")
|
||||
# [/DEF:test_preview_missing_datasource:Function]
|
||||
|
||||
# [DEF:test_preview_missing_translation_column:Function]
|
||||
# @PURPOSE: Verify error when job has no translation column.
|
||||
def test_preview_missing_translation_column(self):
|
||||
"""Preview should fail if job has no translation column."""
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
job = _make_mock_job(translation_column=None)
|
||||
db.query.return_value.filter.return_value.first.return_value = job
|
||||
|
||||
preview_service = TranslationPreview(db, config_manager, "test-user")
|
||||
with pytest.raises(ValueError, match="translation column"):
|
||||
preview_service.preview_rows(job_id="job-123")
|
||||
# [/DEF:test_preview_missing_translation_column:Function]
|
||||
|
||||
|
||||
# [/DEF:TestTranslationPreview:Class]
|
||||
|
||||
# [/DEF:TranslationPreviewTests:Module]
|
||||
332
backend/src/plugins/translate/__tests__/test_sql_generator.py
Normal file
332
backend/src/plugins/translate/__tests__/test_sql_generator.py
Normal file
@@ -0,0 +1,332 @@
|
||||
# [DEF:SQLGeneratorTests:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @SEMANTICS: test, translate, sql_generator
|
||||
# @PURPOSE: Tests for SQLGenerator: PostgreSQL INSERT + UPSERT, ClickHouse INSERT, dialect quoting, NULL handling, injection safety.
|
||||
# @LAYER: Test
|
||||
# @RELATION: BINDS_TO -> [SQLGenerator:Module]
|
||||
# @TEST_CONTRACT: SQLGenerator.generate -> SQL string, row_count
|
||||
# @TEST_FIXTURE: sample_rows -> [{"col1": "val1", "col2": 42}, {"col1": "val2", "col2": 99}]
|
||||
# @TEST_EDGE: empty_rows -> raises ValueError
|
||||
# @TEST_EDGE: null_values -> properly encoded as NULL
|
||||
# @TEST_EDGE: sql_injection -> values with single quotes are escaped
|
||||
|
||||
import pytest
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from src.plugins.translate.sql_generator import (
|
||||
SQLGenerator,
|
||||
_quote_identifier,
|
||||
_encode_sql_value,
|
||||
generate_insert_sql,
|
||||
generate_upsert_sql,
|
||||
)
|
||||
|
||||
|
||||
# [DEF:_normalize_sql:Function]
|
||||
# @PURPOSE: Normalize SQL for comparison by collapsing whitespace.
|
||||
def _normalize_sql(sql: str) -> str:
|
||||
return " ".join(sql.split())
|
||||
|
||||
|
||||
# [DEF:sample_rows:Function]
|
||||
# @PURPOSE: Sample rows fixture for SQL generation tests.
|
||||
@pytest.fixture
|
||||
def sample_rows() -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{"col1": "hello", "col2": 42, "col3": None},
|
||||
{"col1": "world", "col2": 99, "col3": "note"},
|
||||
]
|
||||
|
||||
|
||||
# [DEF:TestQuoteIdentifier:Class]
|
||||
# @PURPOSE: Tests for identifier quoting per dialect.
|
||||
class TestQuoteIdentifier:
|
||||
# [DEF:test_quote_postgresql:Function]
|
||||
# @PURPOSE: PostgreSQL uses double quotes.
|
||||
def test_quote_postgresql(self) -> None:
|
||||
assert _quote_identifier("my_column", "postgresql") == '"my_column"'
|
||||
|
||||
# [DEF:test_quote_clickhouse:Function]
|
||||
# @PURPOSE: ClickHouse uses backticks.
|
||||
def test_quote_clickhouse(self) -> None:
|
||||
assert _quote_identifier("my_column", "clickhouse") == "`my_column`"
|
||||
|
||||
# [DEF:test_quote_with_existing_quotes:Function]
|
||||
# @PURPOSE: Strips existing quotes before quoting.
|
||||
def test_quote_with_existing_quotes(self) -> None:
|
||||
assert _quote_identifier('"my_col"', "postgresql") == '"my_col"'
|
||||
assert _quote_identifier("`my_col`", "clickhouse") == "`my_col`"
|
||||
|
||||
# [DEF:test_quote_schema_table:Function]
|
||||
# @PURPOSE: Schema-qualified table references work.
|
||||
def test_quote_schema_table(self) -> None:
|
||||
assert _quote_identifier("my_schema", "postgresql") == '"my_schema"'
|
||||
assert _quote_identifier("my_table", "postgresql") == '"my_table"'
|
||||
# [/DEF:TestQuoteIdentifier:Class]
|
||||
|
||||
|
||||
# [DEF:TestEncodeSqlValue:Class]
|
||||
# @PURPOSE: Tests for SQL value encoding.
|
||||
class TestEncodeSqlValue:
|
||||
# [DEF:test_null:Function]
|
||||
# @PURPOSE: None encodes as NULL.
|
||||
def test_null(self) -> None:
|
||||
assert _encode_sql_value(None) == "NULL"
|
||||
|
||||
# [DEF:test_string:Function]
|
||||
# @PURPOSE: Strings are single-quoted.
|
||||
def test_string(self) -> None:
|
||||
assert _encode_sql_value("hello") == "'hello'"
|
||||
|
||||
# [DEF:test_integer:Function]
|
||||
# @PURPOSE: Integers are not quoted.
|
||||
def test_integer(self) -> None:
|
||||
assert _encode_sql_value(42) == "42"
|
||||
|
||||
# [DEF:test_float:Function]
|
||||
# @PURPOSE: Floats are not quoted.
|
||||
def test_float(self) -> None:
|
||||
assert _encode_sql_value(3.14) == "3.14"
|
||||
|
||||
# [DEF:test_boolean:Function]
|
||||
# @PURPOSE: Booleans encode as TRUE/FALSE.
|
||||
def test_boolean(self) -> None:
|
||||
assert _encode_sql_value(True) == "TRUE"
|
||||
assert _encode_sql_value(False) == "FALSE"
|
||||
|
||||
# [DEF:test_sql_injection:Function]
|
||||
# @PURPOSE: Single quotes in strings are escaped.
|
||||
def test_sql_injection(self) -> None:
|
||||
assert _encode_sql_value("it's a test") == "'it''s a test'"
|
||||
|
||||
# [DEF:test_special_chars:Function]
|
||||
# @PURPOSE: Special characters in strings.
|
||||
def test_special_chars(self) -> None:
|
||||
assert _encode_sql_value("line1\nline2") == "'line1\nline2'"
|
||||
# [/DEF:TestEncodeSqlValue:Class]
|
||||
|
||||
|
||||
# [DEF:TestGenerateInsertSql:Class]
|
||||
# @PURPOSE: Tests for plain INSERT SQL generation.
|
||||
class TestGenerateInsertSql:
|
||||
# [DEF:test_basic_insert:Function]
|
||||
# @PURPOSE: Generates basic INSERT with VALUES.
|
||||
def test_basic_insert(self, sample_rows: List[Dict[str, Any]]) -> None:
|
||||
sql = generate_insert_sql(
|
||||
target_schema=None,
|
||||
target_table="my_table",
|
||||
columns=["col1", "col2", "col3"],
|
||||
rows=sample_rows,
|
||||
)
|
||||
assert sql.startswith("INSERT INTO my_table")
|
||||
assert "VALUES" in sql
|
||||
assert "'hello'" in sql
|
||||
assert "42" in sql
|
||||
assert "NULL" in sql
|
||||
assert sql.endswith(";")
|
||||
|
||||
# [DEF:test_insert_with_schema:Function]
|
||||
# @PURPOSE: Schema-qualified table name.
|
||||
def test_insert_with_schema(self, sample_rows: List[Dict[str, Any]]) -> None:
|
||||
sql = generate_insert_sql(
|
||||
target_schema="public",
|
||||
target_table="my_table",
|
||||
columns=["col1", "col2", "col3"],
|
||||
rows=sample_rows,
|
||||
)
|
||||
assert "public" in sql
|
||||
assert "my_table" in sql
|
||||
|
||||
# [DEF:test_empty_columns_raises:Function]
|
||||
# @PURPOSE: Empty columns list raises ValueError.
|
||||
def test_empty_columns_raises(self, sample_rows: List[Dict[str, Any]]) -> None:
|
||||
with pytest.raises(ValueError, match="At least one column"):
|
||||
generate_insert_sql(
|
||||
target_schema=None,
|
||||
target_table="my_table",
|
||||
columns=[],
|
||||
rows=sample_rows,
|
||||
)
|
||||
|
||||
# [DEF:test_empty_rows_raises:Function]
|
||||
# @PURPOSE: Empty rows list raises ValueError.
|
||||
def test_empty_rows_raises(self) -> None:
|
||||
with pytest.raises(ValueError, match="At least one row"):
|
||||
generate_insert_sql(
|
||||
target_schema=None,
|
||||
target_table="my_table",
|
||||
columns=["col1"],
|
||||
rows=[],
|
||||
)
|
||||
|
||||
# [DEF:test_null_handling:Function]
|
||||
# @PURPOSE: NULL values are correctly represented.
|
||||
def test_null_handling(self) -> None:
|
||||
rows = [{"col1": None, "col2": "text"}]
|
||||
sql = generate_insert_sql(
|
||||
target_schema=None,
|
||||
target_table="t",
|
||||
columns=["col1", "col2"],
|
||||
rows=rows,
|
||||
)
|
||||
assert "NULL" in sql
|
||||
assert "'text'" in sql
|
||||
|
||||
# [DEF:test_injection_safety:Function]
|
||||
# @PURPOSE: SQL injection via string values is prevented.
|
||||
def test_injection_safety(self) -> None:
|
||||
rows = [{"col1": "'; DROP TABLE users; --"}]
|
||||
sql = generate_insert_sql(
|
||||
target_schema=None,
|
||||
target_table="t",
|
||||
columns=["col1"],
|
||||
rows=rows,
|
||||
)
|
||||
# The single quote should be doubled, preventing injection
|
||||
assert "'';" in sql or "''" in sql
|
||||
assert "DROP TABLE" in sql # It's literal data, not a command
|
||||
# [/DEF:TestGenerateInsertSql:Class]
|
||||
|
||||
|
||||
# [DEF:TestGenerateUpsertSql:Class]
|
||||
# @PURPOSE: Tests for PostgreSQL UPSERT SQL generation.
|
||||
class TestGenerateUpsertSql:
|
||||
# [DEF:test_basic_upsert:Function]
|
||||
# @PURPOSE: Generates INSERT ... ON CONFLICT DO UPDATE.
|
||||
def test_basic_upsert(self, sample_rows: List[Dict[str, Any]]) -> None:
|
||||
sql = generate_upsert_sql(
|
||||
target_schema=None,
|
||||
target_table="my_table",
|
||||
columns=["col1", "col2", "col3"],
|
||||
key_columns=["col1"],
|
||||
rows=sample_rows,
|
||||
)
|
||||
assert "INSERT INTO" in sql
|
||||
assert "ON CONFLICT" in sql
|
||||
assert "DO UPDATE SET" in sql
|
||||
assert "col2 = EXCLUDED.col2" in sql
|
||||
|
||||
# [DEF:test_upsert_all_keys:Function]
|
||||
# @PURPOSE: When all columns are keys, use DO NOTHING.
|
||||
def test_upsert_all_keys(self) -> None:
|
||||
rows = [{"id": 1, "name": "test"}]
|
||||
sql = generate_upsert_sql(
|
||||
target_schema=None,
|
||||
target_table="t",
|
||||
columns=["id", "name"],
|
||||
key_columns=["id", "name"],
|
||||
rows=rows,
|
||||
)
|
||||
assert "DO NOTHING" in sql
|
||||
assert "DO UPDATE" not in sql
|
||||
|
||||
# [DEF:test_upsert_empty_keys:Function]
|
||||
# @PURPOSE: Empty key_columns raises ValueError.
|
||||
def test_upsert_empty_keys(self, sample_rows: List[Dict[str, Any]]) -> None:
|
||||
with pytest.raises(ValueError, match="key_columns"):
|
||||
generate_upsert_sql(
|
||||
target_schema=None,
|
||||
target_table="t",
|
||||
columns=["col1"],
|
||||
key_columns=[],
|
||||
rows=sample_rows,
|
||||
)
|
||||
# [/DEF:TestGenerateUpsertSql:Class]
|
||||
|
||||
|
||||
# [DEF:TestSQLGenerator:Class]
|
||||
# @PURPOSE: Tests for the full SQLGenerator class.
|
||||
class TestSQLGenerator:
|
||||
# [DEF:test_postgresql_insert:Function]
|
||||
# @PURPOSE: PostgreSQL dialect generates proper INSERT.
|
||||
def test_postgresql_insert(self, sample_rows: List[Dict[str, Any]]) -> None:
|
||||
sql, count = SQLGenerator.generate(
|
||||
dialect="postgresql",
|
||||
target_schema="public",
|
||||
target_table="my_table",
|
||||
columns=["col1", "col2"],
|
||||
rows=sample_rows,
|
||||
upsert_strategy="INSERT",
|
||||
)
|
||||
assert count == 2
|
||||
assert "INSERT INTO" in sql
|
||||
assert '"my_table"' in sql # Quoted identifier
|
||||
|
||||
# [DEF:test_postgresql_upsert:Function]
|
||||
# @PURPOSE: PostgreSQL dialect generates proper UPSERT.
|
||||
def test_postgresql_upsert(self, sample_rows: List[Dict[str, Any]]) -> None:
|
||||
sql, count = SQLGenerator.generate(
|
||||
dialect="postgresql",
|
||||
target_schema="public",
|
||||
target_table="my_table",
|
||||
columns=["col1", "col2", "col3"],
|
||||
rows=sample_rows,
|
||||
key_columns=["col1"],
|
||||
upsert_strategy="MERGE",
|
||||
)
|
||||
assert count == 2
|
||||
assert "ON CONFLICT" in sql
|
||||
|
||||
# [DEF:test_clickhouse_insert:Function]
|
||||
# @PURPOSE: ClickHouse dialect generates plain INSERT.
|
||||
def test_clickhouse_insert(self, sample_rows: List[Dict[str, Any]]) -> None:
|
||||
sql, count = SQLGenerator.generate(
|
||||
dialect="clickhouse",
|
||||
target_schema="default",
|
||||
target_table="my_table",
|
||||
columns=["col1", "col2"],
|
||||
rows=sample_rows,
|
||||
upsert_strategy="INSERT",
|
||||
)
|
||||
assert count == 2
|
||||
assert "INSERT INTO" in sql
|
||||
# ClickHouse uses backticks
|
||||
assert "`my_table`" in sql or "my_table" in sql
|
||||
|
||||
# [DEF:test_clickhouse_upsert_fallback:Function]
|
||||
# @PURPOSE: ClickHouse with MERGE strategy falls back to plain INSERT with warning.
|
||||
def test_clickhouse_upsert_fallback(self, sample_rows: List[Dict[str, Any]]) -> None:
|
||||
sql, count = SQLGenerator.generate(
|
||||
dialect="clickhouse",
|
||||
target_schema="default",
|
||||
target_table="my_table",
|
||||
columns=["col1", "col2"],
|
||||
rows=sample_rows,
|
||||
key_columns=["col1"],
|
||||
upsert_strategy="MERGE",
|
||||
)
|
||||
assert count == 2
|
||||
# ClickHouse doesn't support ON CONFLICT; generates plain INSERT
|
||||
assert "INSERT INTO" in sql
|
||||
|
||||
# [DEF:test_empty_rows_via_generator:Function]
|
||||
# @PURPOSE: Generator raises on empty rows.
|
||||
def test_empty_rows_via_generator(self) -> None:
|
||||
with pytest.raises(ValueError, match="At least one row"):
|
||||
SQLGenerator.generate(
|
||||
dialect="postgresql",
|
||||
target_schema=None,
|
||||
target_table="t",
|
||||
columns=["c"],
|
||||
rows=[],
|
||||
)
|
||||
|
||||
# [DEF:test_generate_batch:Function]
|
||||
# @PURPOSE: generate_batch splits large row sets.
|
||||
def test_generate_batch(self, sample_rows: List[Dict[str, Any]]) -> None:
|
||||
# Create many rows to test batching
|
||||
many_rows = sample_rows * 5 # 10 rows
|
||||
statements = SQLGenerator.generate_batch(
|
||||
dialect="postgresql",
|
||||
target_schema="public",
|
||||
target_table="t",
|
||||
columns=["col1", "col2"],
|
||||
rows=many_rows,
|
||||
max_rows_per_statement=3,
|
||||
)
|
||||
assert len(statements) > 1 # Should be split into multiple statements
|
||||
total_count = sum(count for _, count in statements)
|
||||
assert total_count == 10
|
||||
# [/DEF:TestSQLGenerator:Class]
|
||||
# [/DEF:SQLGeneratorTests:Module]
|
||||
35
backend/src/plugins/translate/_utils.py
Normal file
35
backend/src/plugins/translate/_utils.py
Normal file
@@ -0,0 +1,35 @@
|
||||
# [DEF:DictionaryUtils:Module]
|
||||
# @COMPLEXITY: 1
|
||||
# @SEMANTICS: dictionary, utils, normalize, delimiter
|
||||
# @PURPOSE: Utility helpers for dictionary management (term normalization and delimiter detection).
|
||||
# @LAYER: Domain
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
|
||||
# [DEF:_normalize_term:Function]
|
||||
# @PURPOSE: Normalize a term for case-insensitive unique constraint lookup.
|
||||
# @RATIONALE: NFC normalization is applied before lowercasing to ensure consistent
|
||||
# comparison of Unicode characters (e.g. precomposed vs decomposed forms).
|
||||
# @REJECTED: Lowercasing without NFC normalization — would cause duplicate entries
|
||||
# for semantically identical Unicode strings in different normalization forms.
|
||||
def _normalize_term(term: str) -> str:
|
||||
"""Normalize a term by NFC, lowercasing, and removing extra whitespace."""
|
||||
if not term:
|
||||
return ""
|
||||
term = unicodedata.normalize('NFC', term)
|
||||
return " ".join(term.lower().split())
|
||||
# [/DEF:_normalize_term:Function]
|
||||
|
||||
|
||||
# [DEF:_detect_delimiter:Function]
|
||||
# @PURPOSE: Detect the delimiter used in a CSV/TSV header line.
|
||||
def _detect_delimiter(header_line: str) -> str:
|
||||
"""Detect delimiter by counting tabs vs commas in the first line."""
|
||||
if not header_line:
|
||||
return ","
|
||||
tab_count = header_line.count("\t")
|
||||
comma_count = header_line.count(",")
|
||||
return "\t" if tab_count > comma_count else ","
|
||||
# [/DEF:_detect_delimiter:Function]
|
||||
781
backend/src/plugins/translate/dictionary.py
Normal file
781
backend/src/plugins/translate/dictionary.py
Normal file
@@ -0,0 +1,781 @@
|
||||
# [DEF:DictionaryManagerModule:Module]
|
||||
# @COMPLEXITY: 4
|
||||
# @SEMANTICS: dictionary, manager, terminology, crud, import, filter
|
||||
# @PURPOSE: Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [TerminologyDictionary:Class]
|
||||
# @RELATION: DEPENDS_ON -> [DictionaryEntry:Class]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationJobDictionary:Class]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationJob:Class]
|
||||
# @RATIONALE: C4 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term.
|
||||
# @REJECTED: Pure C3 CRUD without state guards would allow orphaned job-dictionary links.
|
||||
# @REJECTED: "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing.
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import func
|
||||
from ...core.logger import logger, belief_scope
|
||||
from ...models.translate import (
|
||||
TerminologyDictionary,
|
||||
DictionaryEntry,
|
||||
TranslationJobDictionary,
|
||||
TranslationJob,
|
||||
)
|
||||
from ._utils import _normalize_term, _detect_delimiter
|
||||
|
||||
|
||||
# [DEF:DictionaryManager:Class]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Manages terminology dictionaries and their entries with referential integrity.
|
||||
# @PRE: Database session is open and valid.
|
||||
# @POST: Dictionary and entry mutations are persisted with conflict detection.
|
||||
# @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
|
||||
class DictionaryManager:
|
||||
# [DEF:DictionaryManager.create_dictionary:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Create a new terminology dictionary.
|
||||
# @PRE: payload contains name, source_dialect, target_dialect.
|
||||
# @POST: New TerminologyDictionary row is created and returned.
|
||||
@staticmethod
|
||||
def create_dictionary(
|
||||
db: Session, name: str, source_dialect: str, target_dialect: str,
|
||||
created_by: Optional[str] = None, description: Optional[str] = None,
|
||||
is_active: bool = True,
|
||||
) -> TerminologyDictionary:
|
||||
with belief_scope("DictionaryManager.create_dictionary"):
|
||||
logger.reason("Creating dictionary", {"name": name, "source": source_dialect, "target": target_dialect})
|
||||
dictionary = TerminologyDictionary(
|
||||
name=name,
|
||||
description=description,
|
||||
source_dialect=source_dialect,
|
||||
target_dialect=target_dialect,
|
||||
is_active=is_active,
|
||||
created_by=created_by,
|
||||
)
|
||||
db.add(dictionary)
|
||||
db.commit()
|
||||
db.refresh(dictionary)
|
||||
logger.reflect("Dictionary created", {"id": dictionary.id})
|
||||
return dictionary
|
||||
# [/DEF:DictionaryManager.create_dictionary:Function]
|
||||
|
||||
# [DEF:DictionaryManager.update_dictionary:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Update an existing terminology dictionary.
|
||||
# @PRE: dict_id exists in terminology_dictionaries table.
|
||||
# @POST: Dictionary metadata is updated and returned.
|
||||
@staticmethod
|
||||
def update_dictionary(
|
||||
db: Session, dict_id: str, name: Optional[str] = None,
|
||||
description: Optional[str] = None, source_dialect: Optional[str] = None,
|
||||
target_dialect: Optional[str] = None, is_active: Optional[bool] = None,
|
||||
) -> TerminologyDictionary:
|
||||
with belief_scope("DictionaryManager.update_dictionary"):
|
||||
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
|
||||
if not dictionary:
|
||||
raise ValueError(f"Dictionary not found: {dict_id}")
|
||||
logger.reason("Updating dictionary", {"id": dict_id})
|
||||
if name is not None:
|
||||
dictionary.name = name
|
||||
if description is not None:
|
||||
dictionary.description = description
|
||||
if source_dialect is not None:
|
||||
dictionary.source_dialect = source_dialect
|
||||
if target_dialect is not None:
|
||||
dictionary.target_dialect = target_dialect
|
||||
if is_active is not None:
|
||||
dictionary.is_active = is_active
|
||||
db.commit()
|
||||
db.refresh(dictionary)
|
||||
logger.reflect("Dictionary updated", {"id": dictionary.id})
|
||||
return dictionary
|
||||
# [/DEF:DictionaryManager.update_dictionary:Function]
|
||||
|
||||
# [DEF:DictionaryManager.delete_dictionary:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Delete a dictionary, blocked if attached to active/scheduled jobs.
|
||||
# @PRE: dict_id exists.
|
||||
# @POST: Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.
|
||||
# @SIDE_EFFECT: Deletes TerminologyDictionary row and all DictionaryEntry rows.
|
||||
@staticmethod
|
||||
def delete_dictionary(db: Session, dict_id: str) -> None:
|
||||
with belief_scope("DictionaryManager.delete_dictionary"):
|
||||
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
|
||||
if not dictionary:
|
||||
raise ValueError(f"Dictionary not found: {dict_id}")
|
||||
|
||||
# Check for attached active/scheduled jobs
|
||||
blocked_statuses = ["ACTIVE", "READY", "RUNNING", "SCHEDULED"]
|
||||
attached_jobs = (
|
||||
db.query(TranslationJobDictionary)
|
||||
.filter(
|
||||
TranslationJobDictionary.dictionary_id == dict_id,
|
||||
TranslationJobDictionary.job_id.in_(
|
||||
db.query(TranslationJob.id).filter(
|
||||
TranslationJob.status.in_(blocked_statuses)
|
||||
)
|
||||
),
|
||||
)
|
||||
.count()
|
||||
)
|
||||
|
||||
if attached_jobs > 0:
|
||||
logger.explore("Delete blocked: dictionary attached to active jobs", {"dict_id": dict_id, "jobs": attached_jobs})
|
||||
raise ValueError(
|
||||
f"Cannot delete dictionary attached to {attached_jobs} active/scheduled job(s). "
|
||||
"Remove dictionary associations from jobs first."
|
||||
)
|
||||
|
||||
logger.reason("Deleting dictionary", {"id": dict_id})
|
||||
# Delete entries and job-dictionary links first
|
||||
db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()
|
||||
db.query(TranslationJobDictionary).filter(TranslationJobDictionary.dictionary_id == dict_id).delete()
|
||||
db.delete(dictionary)
|
||||
db.commit()
|
||||
logger.reflect("Dictionary deleted", {"id": dict_id})
|
||||
# [/DEF:DictionaryManager.delete_dictionary:Function]
|
||||
|
||||
# [DEF:DictionaryManager.get_dictionary:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Get a single dictionary by ID with entry count.
|
||||
# @PRE: dict_id exists.
|
||||
# @POST: Returns dict with dictionary + entry_count or raises ValueError.
|
||||
@staticmethod
|
||||
def get_dictionary(db: Session, dict_id: str) -> TerminologyDictionary:
|
||||
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
|
||||
if not dictionary:
|
||||
raise ValueError(f"Dictionary not found: {dict_id}")
|
||||
return dictionary
|
||||
# [/DEF:DictionaryManager.get_dictionary:Function]
|
||||
|
||||
# [DEF:DictionaryManager.list_dictionaries:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: List dictionaries with pagination and entry counts.
|
||||
# @PRE: page >= 1, page_size between 1 and 100.
|
||||
# @POST: Returns (list of dicts, total_count).
|
||||
@staticmethod
|
||||
def list_dictionaries(
|
||||
db: Session, page: int = 1, page_size: int = 20,
|
||||
) -> Tuple[List[TerminologyDictionary], int]:
|
||||
total = db.query(func.count(TerminologyDictionary.id)).scalar() or 0
|
||||
dictionaries = (
|
||||
db.query(TerminologyDictionary)
|
||||
.order_by(TerminologyDictionary.created_at.desc())
|
||||
.offset((page - 1) * page_size)
|
||||
.limit(page_size)
|
||||
.all()
|
||||
)
|
||||
return dictionaries, total
|
||||
# [/DEF:DictionaryManager.list_dictionaries:Function]
|
||||
|
||||
# [DEF:DictionaryManager.add_entry:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Add an entry to a dictionary, enforcing unique source_term_normalized.
|
||||
# @PRE: dict_id exists. source_term and target_term are non-empty.
|
||||
# @POST: New DictionaryEntry row is created or raises on duplicate.
|
||||
@staticmethod
|
||||
def add_entry(
|
||||
db: Session, dict_id: str, source_term: str, target_term: str,
|
||||
context_notes: Optional[str] = None,
|
||||
) -> DictionaryEntry:
|
||||
with belief_scope("DictionaryManager.add_entry"):
|
||||
normalized = _normalize_term(source_term)
|
||||
existing = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(
|
||||
DictionaryEntry.dictionary_id == dict_id,
|
||||
DictionaryEntry.source_term_normalized == normalized,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
raise ValueError(
|
||||
f"Duplicate entry: '{source_term}' already exists in this dictionary "
|
||||
f"(id={existing.id}). Use overwrite or keep_existing conflict mode."
|
||||
)
|
||||
|
||||
logger.reason("Adding dictionary entry", {"dict_id": dict_id, "term": source_term})
|
||||
entry = DictionaryEntry(
|
||||
dictionary_id=dict_id,
|
||||
source_term=source_term.strip(),
|
||||
source_term_normalized=normalized,
|
||||
target_term=target_term.strip(),
|
||||
context_notes=context_notes.strip() if context_notes else None,
|
||||
)
|
||||
db.add(entry)
|
||||
db.commit()
|
||||
db.refresh(entry)
|
||||
logger.reflect("Entry added", {"entry_id": entry.id})
|
||||
return entry
|
||||
# [/DEF:DictionaryManager.add_entry:Function]
|
||||
|
||||
# [DEF:DictionaryManager.edit_entry:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Edit an existing dictionary entry with duplicate-aware normalization.
|
||||
# @PRE: entry_id exists.
|
||||
# @POST: Entry fields are updated.
|
||||
@staticmethod
|
||||
def edit_entry(
|
||||
db: Session, entry_id: str, source_term: Optional[str] = None,
|
||||
target_term: Optional[str] = None, context_notes: Optional[str] = None,
|
||||
) -> DictionaryEntry:
|
||||
with belief_scope("DictionaryManager.edit_entry"):
|
||||
entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()
|
||||
if not entry:
|
||||
raise ValueError(f"Entry not found: {entry_id}")
|
||||
|
||||
logger.reason("Editing dictionary entry", {"entry_id": entry_id})
|
||||
if source_term is not None:
|
||||
normalized = _normalize_term(source_term)
|
||||
# Check uniqueness within the same dictionary
|
||||
existing = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(
|
||||
DictionaryEntry.dictionary_id == entry.dictionary_id,
|
||||
DictionaryEntry.source_term_normalized == normalized,
|
||||
DictionaryEntry.id != entry_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
raise ValueError(
|
||||
f"Duplicate entry: '{source_term}' already exists in this dictionary "
|
||||
f"(id={existing.id})."
|
||||
)
|
||||
entry.source_term = source_term.strip()
|
||||
entry.source_term_normalized = normalized
|
||||
if target_term is not None:
|
||||
entry.target_term = target_term.strip()
|
||||
if context_notes is not None:
|
||||
entry.context_notes = context_notes.strip() if context_notes else None
|
||||
db.commit()
|
||||
db.refresh(entry)
|
||||
logger.reflect("Entry updated", {"entry_id": entry.id})
|
||||
return entry
|
||||
# [/DEF:DictionaryManager.edit_entry:Function]
|
||||
|
||||
# [DEF:DictionaryManager.delete_entry:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Delete a single dictionary entry.
|
||||
# @PRE: entry_id exists.
|
||||
# @POST: Entry is deleted.
|
||||
@staticmethod
|
||||
def delete_entry(db: Session, entry_id: str) -> None:
|
||||
with belief_scope("DictionaryManager.delete_entry"):
|
||||
entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()
|
||||
if not entry:
|
||||
raise ValueError(f"Entry not found: {entry_id}")
|
||||
logger.reason("Deleting dictionary entry", {"entry_id": entry_id})
|
||||
db.delete(entry)
|
||||
db.commit()
|
||||
logger.reflect("Entry deleted", {"entry_id": entry_id})
|
||||
# [/DEF:DictionaryManager.delete_entry:Function]
|
||||
|
||||
# [DEF:DictionaryManager.clear_entries:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Delete all entries for a dictionary.
|
||||
# @PRE: dict_id exists.
|
||||
# @POST: All entries for the dictionary are deleted.
|
||||
@staticmethod
|
||||
def clear_entries(db: Session, dict_id: str) -> int:
|
||||
with belief_scope("DictionaryManager.clear_entries"):
|
||||
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
|
||||
if not dictionary:
|
||||
raise ValueError(f"Dictionary not found: {dict_id}")
|
||||
logger.reason("Clearing all entries", {"dict_id": dict_id})
|
||||
deleted = db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()
|
||||
db.commit()
|
||||
logger.reflect("Entries cleared", {"dict_id": dict_id, "count": deleted})
|
||||
return deleted
|
||||
# [/DEF:DictionaryManager.clear_entries:Function]
|
||||
|
||||
# [DEF:DictionaryManager.list_entries:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: List entries for a dictionary with pagination.
|
||||
# @PRE: dict_id exists.
|
||||
# @POST: Returns (list of entries, total_count).
|
||||
@staticmethod
|
||||
def list_entries(
|
||||
db: Session, dict_id: str, page: int = 1, page_size: int = 50,
|
||||
) -> Tuple[List[DictionaryEntry], int]:
|
||||
total = (
|
||||
db.query(func.count(DictionaryEntry.id))
|
||||
.filter(DictionaryEntry.dictionary_id == dict_id)
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
entries = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(DictionaryEntry.dictionary_id == dict_id)
|
||||
.order_by(DictionaryEntry.source_term.asc())
|
||||
.offset((page - 1) * page_size)
|
||||
.limit(page_size)
|
||||
.all()
|
||||
)
|
||||
return entries, total
|
||||
# [/DEF:DictionaryManager.list_entries:Function]
|
||||
|
||||
# [DEF:DictionaryManager.import_entries:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Import entries from CSV/TSV content with duplicate detection and conflict resolution.
|
||||
# @PRE: content is valid CSV or TSV. dict_id exists.
|
||||
# @POST: Entries are created/updated/skipped per conflict mode. Returns result summary.
|
||||
# @SIDE_EFFECT: Batch-inserts or updates DictionaryEntry rows.
|
||||
@staticmethod
|
||||
def import_entries(
|
||||
db: Session, dict_id: str, content: str,
|
||||
delimiter: Optional[str] = None,
|
||||
on_conflict: str = "overwrite",
|
||||
preview_only: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
with belief_scope("DictionaryManager.import_entries"):
|
||||
# Validate dictionary
|
||||
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
|
||||
if not dictionary:
|
||||
raise ValueError(f"Dictionary not found: {dict_id}")
|
||||
|
||||
# Detect delimiter if not specified
|
||||
if not delimiter:
|
||||
delimiter = _detect_delimiter(content)
|
||||
logger.reason("Detected delimiter", {"delimiter": repr(delimiter)})
|
||||
|
||||
if delimiter not in (",", "\t"):
|
||||
raise ValueError(f"Unsupported delimiter: {repr(delimiter)}. Use ',' or '\\t'.")
|
||||
|
||||
# Parse content
|
||||
reader = csv.DictReader(io.StringIO(content), delimiter=delimiter)
|
||||
required_fields = {"source_term", "target_term"}
|
||||
if not reader.fieldnames or not required_fields.issubset(reader.fieldnames):
|
||||
raise ValueError(
|
||||
f"CSV/TSV must have at least 'source_term' and 'target_term' columns. "
|
||||
f"Got: {reader.fieldnames}"
|
||||
)
|
||||
|
||||
result: Dict[str, Any] = {
|
||||
"total": 0,
|
||||
"created": 0,
|
||||
"updated": 0,
|
||||
"skipped": 0,
|
||||
"errors": [],
|
||||
"preview": [],
|
||||
}
|
||||
|
||||
rows = list(reader)
|
||||
result["total"] = len(rows)
|
||||
|
||||
for row_idx, row in enumerate(rows):
|
||||
try:
|
||||
source_term = row.get("source_term", "").strip()
|
||||
target_term = row.get("target_term", "").strip()
|
||||
context_notes = row.get("context_notes", "").strip() or None
|
||||
|
||||
if not source_term or not target_term:
|
||||
result["errors"].append({
|
||||
"line": row_idx + 2, # +2 for header and 1-indexed
|
||||
"error": "Both source_term and target_term are required",
|
||||
"row": row,
|
||||
})
|
||||
continue
|
||||
|
||||
normalized = _normalize_term(source_term)
|
||||
|
||||
if preview_only:
|
||||
existing = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(
|
||||
DictionaryEntry.dictionary_id == dict_id,
|
||||
DictionaryEntry.source_term_normalized == normalized,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
preview_row = {
|
||||
"line": row_idx + 2,
|
||||
"source_term": source_term,
|
||||
"target_term": target_term,
|
||||
"context_notes": context_notes,
|
||||
"is_conflict": existing is not None,
|
||||
"existing_target_term": existing.target_term if existing else None,
|
||||
}
|
||||
result["preview"].append(preview_row)
|
||||
continue
|
||||
|
||||
existing = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(
|
||||
DictionaryEntry.dictionary_id == dict_id,
|
||||
DictionaryEntry.source_term_normalized == normalized,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
if existing:
|
||||
if on_conflict == "overwrite":
|
||||
existing.source_term = source_term
|
||||
existing.target_term = target_term
|
||||
existing.context_notes = context_notes
|
||||
result["updated"] += 1
|
||||
elif on_conflict == "keep_existing":
|
||||
result["skipped"] += 1
|
||||
else: # cancel
|
||||
result["errors"].append({
|
||||
"line": row_idx + 2,
|
||||
"error": f"Conflict on '{source_term}' and on_conflict='cancel'",
|
||||
"row": row,
|
||||
})
|
||||
continue
|
||||
else:
|
||||
entry = DictionaryEntry(
|
||||
dictionary_id=dict_id,
|
||||
source_term=source_term,
|
||||
source_term_normalized=normalized,
|
||||
target_term=target_term,
|
||||
context_notes=context_notes,
|
||||
)
|
||||
db.add(entry)
|
||||
result["created"] += 1
|
||||
|
||||
except Exception as e:
|
||||
result["errors"].append({
|
||||
"line": row_idx + 2,
|
||||
"error": str(e),
|
||||
"row": row,
|
||||
})
|
||||
|
||||
if not preview_only:
|
||||
db.commit()
|
||||
|
||||
logger.reflect("Import complete", {
|
||||
"dict_id": dict_id,
|
||||
"total": result["total"],
|
||||
"created": result["created"],
|
||||
"updated": result["updated"],
|
||||
"skipped": result["skipped"],
|
||||
"errors": len(result["errors"]),
|
||||
})
|
||||
return result
|
||||
# [/DEF:DictionaryManager.import_entries:Function]
|
||||
|
||||
# [DEF:DictionaryManager.filter_for_batch:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.
|
||||
# @PRE: job_id exists and source_texts is a list of strings.
|
||||
# @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries.
|
||||
# @SIDE_EFFECT: Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.
|
||||
@staticmethod
|
||||
def filter_for_batch(
|
||||
db: Session, source_texts: List[str], job_id: str,
|
||||
) -> List[Dict[str, Any]]:
|
||||
with belief_scope("DictionaryManager.filter_for_batch"):
|
||||
# Get dictionaries attached to this job
|
||||
job_dict_links = (
|
||||
db.query(TranslationJobDictionary)
|
||||
.filter(TranslationJobDictionary.job_id == job_id)
|
||||
.all()
|
||||
)
|
||||
if not job_dict_links:
|
||||
logger.reason("No dictionaries attached to job", {"job_id": job_id})
|
||||
return []
|
||||
|
||||
dict_ids = [jd.dictionary_id for jd in job_dict_links]
|
||||
|
||||
# Get all active dictionaries
|
||||
dictionaries = (
|
||||
db.query(TerminologyDictionary)
|
||||
.filter(
|
||||
TerminologyDictionary.id.in_(dict_ids),
|
||||
TerminologyDictionary.is_active == True,
|
||||
)
|
||||
.all()
|
||||
)
|
||||
if not dictionaries:
|
||||
return []
|
||||
|
||||
# Build dict_id -> dict_name lookup
|
||||
dict_map = {d.id: d.name for d in dictionaries}
|
||||
|
||||
# Fetch all entries for these dictionaries, ordered by dictionary (priority order from link order)
|
||||
all_entries = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(DictionaryEntry.dictionary_id.in_(dict_ids))
|
||||
.order_by(DictionaryEntry.source_term.asc())
|
||||
.all()
|
||||
)
|
||||
|
||||
if not all_entries:
|
||||
return []
|
||||
|
||||
matched: List[Dict[str, Any]] = []
|
||||
seen_source_match: set = set()
|
||||
|
||||
for text_idx, source_text in enumerate(source_texts):
|
||||
if not source_text:
|
||||
continue
|
||||
|
||||
text_lower = source_text.lower()
|
||||
|
||||
for entry in all_entries:
|
||||
norm = entry.source_term_normalized
|
||||
if not norm:
|
||||
continue
|
||||
|
||||
# Word-boundary-aware matching
|
||||
# Build pattern: \bterm\b (case-insensitive)
|
||||
# Escape regex special chars in the search term
|
||||
escaped = re.escape(norm)
|
||||
pattern = re.compile(r"\b" + escaped + r"\b", re.IGNORECASE)
|
||||
|
||||
if pattern.search(text_lower):
|
||||
match_key = (text_idx, entry.id)
|
||||
if match_key not in seen_source_match:
|
||||
seen_source_match.add(match_key)
|
||||
matched.append({
|
||||
"text_index": text_idx,
|
||||
"source_text": source_text,
|
||||
"entry_id": entry.id,
|
||||
"source_term": entry.source_term,
|
||||
"source_term_normalized": entry.source_term_normalized,
|
||||
"target_term": entry.target_term,
|
||||
"dictionary_id": entry.dictionary_id,
|
||||
"dictionary_name": dict_map.get(entry.dictionary_id, ""),
|
||||
"context_notes": entry.context_notes,
|
||||
})
|
||||
|
||||
# Sort by dictionary link priority (order of dict_ids from link order)
|
||||
dict_priority = {d_id: idx for idx, d_id in enumerate(dict_ids)}
|
||||
matched.sort(key=lambda m: (dict_priority.get(m["dictionary_id"], 999), m["text_index"]))
|
||||
|
||||
logger.reflect("Batch filter match complete", {
|
||||
"job_id": job_id,
|
||||
"source_texts": len(source_texts),
|
||||
"matches": len(matched),
|
||||
"dictionaries_used": len(dictionaries),
|
||||
})
|
||||
return matched
|
||||
# [/DEF:DictionaryManager.filter_for_batch:Function]
|
||||
|
||||
|
||||
# [DEF:DictionaryManager.submit_correction:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Submit a term correction from a run result. Validates language match, detects conflicts.
|
||||
# @PRE: source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.
|
||||
# @POST: Entry created or updated; origin tracking populated. Returns action + conflict info.
|
||||
# @SIDE_EFFECT: Creates/updates DictionaryEntry row.
|
||||
@staticmethod
|
||||
def submit_correction(
|
||||
db: Session,
|
||||
dict_id: str,
|
||||
source_term: str,
|
||||
incorrect_target_term: str,
|
||||
corrected_target_term: str,
|
||||
origin_run_id: Optional[str] = None,
|
||||
origin_row_key: Optional[str] = None,
|
||||
origin_user_id: Optional[str] = None,
|
||||
on_conflict: str = "overwrite",
|
||||
) -> Dict[str, Any]:
|
||||
with belief_scope("DictionaryManager.submit_correction"):
|
||||
# Validate dictionary exists
|
||||
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
|
||||
if not dictionary:
|
||||
raise ValueError(f"Dictionary '{dict_id}' not found")
|
||||
|
||||
normalized = _normalize_term(source_term)
|
||||
existing = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(
|
||||
DictionaryEntry.dictionary_id == dict_id,
|
||||
DictionaryEntry.source_term_normalized == normalized,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
result: Dict[str, Any] = {
|
||||
"source_term": source_term,
|
||||
"target_term": corrected_target_term,
|
||||
"action": "created",
|
||||
"entry_id": None,
|
||||
"conflict": None,
|
||||
"message": None,
|
||||
}
|
||||
|
||||
if existing:
|
||||
# Conflict detected
|
||||
if on_conflict == "keep_existing":
|
||||
result["action"] = "conflict_detected"
|
||||
result["conflict"] = {
|
||||
"source_term": source_term,
|
||||
"existing_target_term": existing.target_term,
|
||||
"submitted_target_term": corrected_target_term,
|
||||
"action": "keep_existing",
|
||||
}
|
||||
result["message"] = f"Existing entry kept: '{existing.target_term}'"
|
||||
return result
|
||||
elif on_conflict == "overwrite":
|
||||
existing.target_term = corrected_target_term.strip()
|
||||
if origin_run_id:
|
||||
existing.origin_run_id = origin_run_id
|
||||
if origin_row_key:
|
||||
existing.origin_row_key = origin_row_key
|
||||
if origin_user_id:
|
||||
existing.origin_user_id = origin_user_id
|
||||
db.flush()
|
||||
result["action"] = "updated"
|
||||
result["entry_id"] = existing.id
|
||||
result["message"] = f"Entry updated from '{existing.target_term}' to '{corrected_target_term}'"
|
||||
else: # cancel
|
||||
result["action"] = "skipped"
|
||||
result["conflict"] = {
|
||||
"source_term": source_term,
|
||||
"existing_target_term": existing.target_term,
|
||||
"submitted_target_term": corrected_target_term,
|
||||
"action": "cancel",
|
||||
}
|
||||
result["message"] = "Correction cancelled by conflict mode"
|
||||
return result
|
||||
else:
|
||||
# Create new entry
|
||||
entry = DictionaryEntry(
|
||||
dictionary_id=dict_id,
|
||||
source_term=source_term.strip(),
|
||||
source_term_normalized=normalized,
|
||||
target_term=corrected_target_term.strip(),
|
||||
origin_run_id=origin_run_id,
|
||||
origin_row_key=origin_row_key,
|
||||
origin_user_id=origin_user_id,
|
||||
)
|
||||
db.add(entry)
|
||||
db.flush()
|
||||
result["entry_id"] = entry.id
|
||||
result["message"] = f"Entry created for '{source_term}' -> '{corrected_target_term}'"
|
||||
|
||||
db.commit()
|
||||
logger.reflect("Correction processed", result)
|
||||
return result
|
||||
# [/DEF:DictionaryManager.submit_correction:Function]
|
||||
|
||||
# [DEF:DictionaryManager.submit_bulk_corrections:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Submit multiple term corrections atomically — all succeed or all fail.
|
||||
# @PRE: corrections list is non-empty. dict_id exists.
|
||||
# @POST: All corrections applied or none applied with conflict list.
|
||||
# @SIDE_EFFECT: Creates/updates DictionaryEntry rows; commits once.
|
||||
@staticmethod
|
||||
def submit_bulk_corrections(
|
||||
db: Session,
|
||||
dict_id: str,
|
||||
corrections: List[Dict[str, Any]],
|
||||
origin_user_id: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
with belief_scope("DictionaryManager.submit_bulk_corrections"):
|
||||
# Validate dictionary first
|
||||
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
|
||||
if not dictionary:
|
||||
raise ValueError(f"Dictionary '{dict_id}' not found")
|
||||
|
||||
results: List[Dict[str, Any]] = []
|
||||
conflicts: List[Dict[str, Any]] = []
|
||||
all_ok = True
|
||||
|
||||
for corr in corrections:
|
||||
source_term = corr.get("source_term", "").strip()
|
||||
incorrect_target = corr.get("incorrect_target_term", "").strip()
|
||||
corrected_target = corr.get("corrected_target_term", "").strip()
|
||||
|
||||
if not source_term or not corrected_target:
|
||||
results.append({
|
||||
"source_term": source_term,
|
||||
"action": "error",
|
||||
"message": "source_term and corrected_target_term are required",
|
||||
})
|
||||
all_ok = False
|
||||
continue
|
||||
|
||||
normalized = _normalize_term(source_term)
|
||||
existing = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(
|
||||
DictionaryEntry.dictionary_id == dict_id,
|
||||
DictionaryEntry.source_term_normalized == normalized,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
if existing:
|
||||
on_conflict = corr.get("on_conflict", "overwrite")
|
||||
if on_conflict == "keep_existing":
|
||||
conflicts.append({
|
||||
"source_term": source_term,
|
||||
"existing_target_term": existing.target_term,
|
||||
"submitted_target_term": corrected_target,
|
||||
"action": "keep_existing",
|
||||
})
|
||||
results.append({
|
||||
"source_term": source_term,
|
||||
"action": "conflict_detected",
|
||||
"message": f"Existing entry kept: '{existing.target_term}'",
|
||||
})
|
||||
continue
|
||||
elif on_conflict == "overwrite":
|
||||
existing.target_term = corrected_target
|
||||
existing.origin_user_id = origin_user_id
|
||||
results.append({
|
||||
"source_term": source_term,
|
||||
"action": "updated",
|
||||
"entry_id": existing.id,
|
||||
"message": f"Updated to '{corrected_target}'",
|
||||
})
|
||||
else:
|
||||
conflicts.append({
|
||||
"source_term": source_term,
|
||||
"existing_target_term": existing.target_term,
|
||||
"submitted_target_term": corrected_target,
|
||||
"action": "cancel",
|
||||
})
|
||||
results.append({
|
||||
"source_term": source_term,
|
||||
"action": "skipped",
|
||||
"message": "Cancelled by conflict mode",
|
||||
})
|
||||
else:
|
||||
entry = DictionaryEntry(
|
||||
dictionary_id=dict_id,
|
||||
source_term=source_term,
|
||||
source_term_normalized=normalized,
|
||||
target_term=corrected_target,
|
||||
origin_run_id=corr.get("origin_run_id"),
|
||||
origin_row_key=corr.get("origin_row_key"),
|
||||
origin_user_id=origin_user_id,
|
||||
)
|
||||
db.add(entry)
|
||||
results.append({
|
||||
"source_term": source_term,
|
||||
"action": "created",
|
||||
"message": f"Entry created for '{source_term}' -> '{corrected_target}'",
|
||||
})
|
||||
|
||||
if conflicts and not all_ok:
|
||||
# Rollback — all failed
|
||||
db.rollback()
|
||||
return {
|
||||
"status": "conflicts",
|
||||
"results": results,
|
||||
"conflicts": conflicts,
|
||||
"message": "Conflicts detected. Resolve before retrying.",
|
||||
}
|
||||
|
||||
db.commit()
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"results": results,
|
||||
"conflicts": conflicts,
|
||||
"total": len(corrections),
|
||||
"applied": sum(1 for r in results if r["action"] in ("created", "updated")),
|
||||
}
|
||||
# [/DEF:DictionaryManager.submit_bulk_corrections:Function]
|
||||
|
||||
|
||||
# [/DEF:DictionaryManager:Class]
|
||||
# [/DEF:DictionaryManagerModule:Module]
|
||||
262
backend/src/plugins/translate/events.py
Normal file
262
backend/src/plugins/translate/events.py
Normal file
@@ -0,0 +1,262 @@
|
||||
# [DEF:TranslationEventLog:Module]
|
||||
# @COMPLEXITY: 5
|
||||
# @SEMANTICS: translate, events, audit, logging
|
||||
# @PURPOSE: Structured event logging for translation operations with terminal event invariant enforcement.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [TranslationEvent]
|
||||
# @RELATION: DEPENDS_ON -> [MetricSnapshot]
|
||||
# @PRE: Database session is open and valid.
|
||||
# @POST: Events are persisted immutably; terminal events enforce exactly-one invariant per run.
|
||||
# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion.
|
||||
# @DATA_CONTRACT: Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]
|
||||
# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id.
|
||||
# @RATIONALE: Immutable event log with nullable run_id allows job-level events (no run context) and run-level events.
|
||||
# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
|
||||
# @REJECTED: Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit.
|
||||
from typing import Any, Dict, List, Optional
|
||||
from sqlalchemy.orm import Session
|
||||
from datetime import datetime, timezone, timedelta
|
||||
import uuid
|
||||
|
||||
from ...core.logger import logger, belief_scope
|
||||
from ...models.translate import TranslationEvent, MetricSnapshot
|
||||
|
||||
# Terminal events per run — exactly one of these must exist for a completed run.
|
||||
TERMINAL_EVENT_TYPES = {"RUN_COMPLETED", "RUN_FAILED", "RUN_CANCELLED"}
|
||||
# All valid event types for validation.
|
||||
VALID_EVENT_TYPES = TERMINAL_EVENT_TYPES | {
|
||||
"JOB_CREATED", "JOB_UPDATED", "JOB_DELETED",
|
||||
"RUN_STARTED", "RUN_RETRYING", "RUN_RETRY_INSERT",
|
||||
"BATCH_STARTED", "BATCH_COMPLETED", "BATCH_FAILED", "BATCH_RETRYING",
|
||||
"TRANSLATION_PHASE_STARTED", "TRANSLATION_PHASE_COMPLETED",
|
||||
"INSERT_PHASE_STARTED", "INSERT_PHASE_COMPLETED",
|
||||
"PREVIEW_CREATED", "PREVIEW_ACCEPTED", "PREVIEW_DISCARDED",
|
||||
"SCHEDULE_CREATED", "SCHEDULE_UPDATED", "SCHEDULE_DELETED",
|
||||
"METRICS_SNAPSHOT_CREATED",
|
||||
}
|
||||
|
||||
DEFAULT_RETENTION_DAYS = 90
|
||||
|
||||
|
||||
# [DEF:TranslationEventLog:Class]
|
||||
# @COMPLEXITY: 5
|
||||
# @PURPOSE: Structured event logging for translation operations with terminal event invariant enforcement.
|
||||
# @PRE: Database session is available.
|
||||
# @POST: Events are written immutably; terminal events enforced per run.
|
||||
# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events.
|
||||
# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id.
|
||||
class TranslationEventLog:
|
||||
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
|
||||
# [DEF:log_event:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Write an immutable event. Enforces terminal event invariant for non-null run_id.
|
||||
# @PRE: event_type must be a known type. If run_id is not None, enforce terminal invariant.
|
||||
# @POST: TranslationEvent row is created.
|
||||
# @SIDE_EFFECT: DB write.
|
||||
def log_event(
|
||||
self,
|
||||
job_id: str,
|
||||
event_type: str,
|
||||
payload: Optional[Dict[str, Any]] = None,
|
||||
run_id: Optional[str] = None,
|
||||
created_by: Optional[str] = None,
|
||||
) -> TranslationEvent:
|
||||
with belief_scope("TranslationEventLog.log_event"):
|
||||
if event_type not in VALID_EVENT_TYPES:
|
||||
raise ValueError(
|
||||
f"Invalid event_type '{event_type}'. "
|
||||
f"Must be one of: {', '.join(sorted(VALID_EVENT_TYPES))}"
|
||||
)
|
||||
|
||||
# Enforce terminal event invariant for non-null run_id
|
||||
if run_id is not None and event_type in TERMINAL_EVENT_TYPES:
|
||||
existing_terminal = (
|
||||
self.db.query(TranslationEvent.id)
|
||||
.filter(
|
||||
TranslationEvent.run_id == run_id,
|
||||
TranslationEvent.event_type.in_(TERMINAL_EVENT_TYPES),
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing_terminal:
|
||||
raise ValueError(
|
||||
f"Run '{run_id}' already has a terminal event "
|
||||
f"({existing_terminal[0]}). Cannot add another terminal event."
|
||||
)
|
||||
|
||||
# Enforce run_started invariant — must exist before other run events
|
||||
if run_id is not None and event_type not in {"RUN_STARTED"} | TERMINAL_EVENT_TYPES:
|
||||
has_started = (
|
||||
self.db.query(TranslationEvent.id)
|
||||
.filter(
|
||||
TranslationEvent.run_id == run_id,
|
||||
TranslationEvent.event_type == "RUN_STARTED",
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not has_started:
|
||||
raise ValueError(
|
||||
f"Cannot log '{event_type}' for run '{run_id}': "
|
||||
f"RUN_STARTED event must precede other run events."
|
||||
)
|
||||
|
||||
event = TranslationEvent(
|
||||
id=str(uuid.uuid4()),
|
||||
job_id=job_id,
|
||||
run_id=run_id,
|
||||
event_type=event_type,
|
||||
event_data=payload or {},
|
||||
created_by=created_by,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
self.db.add(event)
|
||||
self.db.flush()
|
||||
|
||||
logger.reason(f"Event logged: {event_type}", {
|
||||
"event_id": event.id,
|
||||
"job_id": job_id,
|
||||
"run_id": run_id,
|
||||
"event_type": event_type,
|
||||
})
|
||||
return event
|
||||
# [/DEF:log_event:Function]
|
||||
|
||||
# [DEF:query_events:Function]
|
||||
# @PURPOSE: Query events with optional filters.
|
||||
# @PRE: None.
|
||||
# @POST: Returns list of TranslationEvent dicts matching filters.
|
||||
def query_events(
|
||||
self,
|
||||
job_id: Optional[str] = None,
|
||||
run_id: Optional[str] = None,
|
||||
event_type: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
) -> List[Dict[str, Any]]:
|
||||
with belief_scope("TranslationEventLog.query_events"):
|
||||
query = self.db.query(TranslationEvent)
|
||||
|
||||
if job_id:
|
||||
query = query.filter(TranslationEvent.job_id == job_id)
|
||||
if run_id:
|
||||
query = query.filter(TranslationEvent.run_id == run_id)
|
||||
if event_type:
|
||||
query = query.filter(TranslationEvent.event_type == event_type)
|
||||
|
||||
events = (
|
||||
query.order_by(TranslationEvent.created_at.desc())
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
|
||||
return [
|
||||
{
|
||||
"id": e.id,
|
||||
"job_id": e.job_id,
|
||||
"run_id": e.run_id,
|
||||
"event_type": e.event_type,
|
||||
"event_data": e.event_data or {},
|
||||
"created_by": e.created_by,
|
||||
"created_at": e.created_at.isoformat() if e.created_at else None,
|
||||
}
|
||||
for e in events
|
||||
]
|
||||
# [/DEF:query_events:Function]
|
||||
|
||||
# [DEF:prune_expired:Function]
|
||||
# @PURPOSE: Delete events older than retention_days. Persists MetricSnapshot before pruning.
|
||||
# @PRE: None.
|
||||
# @POST: Expired events are deleted; MetricSnapshot is created before deletion.
|
||||
# @SIDE_EFFECT: Creates MetricSnapshot row; deletes TranslationEvent rows.
|
||||
def prune_expired(
|
||||
self,
|
||||
retention_days: int = DEFAULT_RETENTION_DAYS,
|
||||
batch_size: int = 1000,
|
||||
) -> Dict[str, Any]:
|
||||
with belief_scope("TranslationEventLog.prune_expired"):
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
|
||||
logger.reason("Pruning expired events", {
|
||||
"cutoff": cutoff.isoformat(),
|
||||
"retention_days": retention_days,
|
||||
})
|
||||
|
||||
# Find events to prune
|
||||
expired_query = self.db.query(TranslationEvent).filter(
|
||||
TranslationEvent.created_at < cutoff
|
||||
)
|
||||
total_expired = expired_query.count()
|
||||
|
||||
if total_expired == 0:
|
||||
logger.reflect("No expired events to prune", {})
|
||||
return {"pruned": 0, "snapshot_id": None}
|
||||
|
||||
# Create MetricSnapshot before pruning
|
||||
snapshot = MetricSnapshot(
|
||||
id=str(uuid.uuid4()),
|
||||
job_id="_prune_aggregate_",
|
||||
key_hash=f"prune_{cutoff.timestamp():.0f}",
|
||||
covers_events_before=cutoff,
|
||||
total_records=total_expired,
|
||||
snapshot_date=datetime.now(timezone.utc),
|
||||
)
|
||||
self.db.add(snapshot)
|
||||
self.db.flush()
|
||||
snapshot_id = snapshot.id
|
||||
|
||||
# Delete in batches
|
||||
pruned = 0
|
||||
while True:
|
||||
batch_ids = (
|
||||
self.db.query(TranslationEvent.id)
|
||||
.filter(TranslationEvent.created_at < cutoff)
|
||||
.limit(batch_size)
|
||||
.all()
|
||||
)
|
||||
if not batch_ids:
|
||||
break
|
||||
ids = [row[0] for row in batch_ids]
|
||||
self.db.query(TranslationEvent).filter(
|
||||
TranslationEvent.id.in_(ids)
|
||||
).delete(synchronize_session=False)
|
||||
self.db.flush()
|
||||
pruned += len(ids)
|
||||
logger.reason("Pruned batch", {"batch_size": len(ids), "total_pruned": pruned})
|
||||
|
||||
self.db.commit()
|
||||
|
||||
logger.reflect("Pruning complete", {
|
||||
"pruned": pruned,
|
||||
"snapshot_id": snapshot_id,
|
||||
})
|
||||
return {"pruned": pruned, "snapshot_id": snapshot_id}
|
||||
# [/DEF:prune_expired:Function]
|
||||
|
||||
# [DEF:get_run_event_summary:Function]
|
||||
# @PURPOSE: Get a summary of events for a run, including invariant check.
|
||||
# @PRE: run_id is not None.
|
||||
# @POST: Returns dict with event list and invariant validity.
|
||||
def get_run_event_summary(self, run_id: str) -> Dict[str, Any]:
|
||||
with belief_scope("TranslationEventLog.get_run_event_summary"):
|
||||
events = self.query_events(run_id=run_id)
|
||||
|
||||
has_started = any(e["event_type"] == "RUN_STARTED" for e in events)
|
||||
terminal_events = [e for e in events if e["event_type"] in TERMINAL_EVENT_TYPES]
|
||||
|
||||
return {
|
||||
"run_id": run_id,
|
||||
"event_count": len(events),
|
||||
"has_run_started": has_started,
|
||||
"terminal_event_count": len(terminal_events),
|
||||
"terminal_events": terminal_events,
|
||||
"invariant_valid": has_started and len(terminal_events) <= 1,
|
||||
"events": events,
|
||||
}
|
||||
# [/DEF:get_run_event_summary:Function]
|
||||
|
||||
|
||||
# [/DEF:TranslationEventLog:Class]
|
||||
# [/DEF:TranslationEventLog:Module]
|
||||
626
backend/src/plugins/translate/executor.py
Normal file
626
backend/src/plugins/translate/executor.py
Normal file
@@ -0,0 +1,626 @@
|
||||
# [DEF:TranslationExecutor:Module]
|
||||
# @COMPLEXITY: 4
|
||||
# @SEMANTICS: translate, executor, batch, llm
|
||||
# @PURPOSE: Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [TranslationBatch]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationRecord]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationRun]
|
||||
# @RELATION: DEPENDS_ON -> [LLMProviderService]
|
||||
# @RELATION: DEPENDS_ON -> [DictionaryManager]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationPreview]
|
||||
# @PRE: Valid TranslationRun with job configuration. DB session is available.
|
||||
# @POST: TranslationBatch and TranslationRecord rows are created. Run status is updated.
|
||||
# @SIDE_EFFECT: Calls LLM provider; creates DB rows; updates run statistics.
|
||||
# @RATIONALE: Batch processing with retry — independent batches allow partial recovery.
|
||||
# @REJECTED: Single monolithic LLM call — would lose all progress on any failure.
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple, Callable
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...core.logger import logger, belief_scope
|
||||
from ...core.config_manager import ConfigManager
|
||||
from ...models.translate import (
|
||||
TranslationJob,
|
||||
TranslationRun,
|
||||
TranslationBatch,
|
||||
TranslationRecord,
|
||||
TranslationPreviewSession,
|
||||
TranslationPreviewRecord,
|
||||
)
|
||||
from ...services.llm_provider import LLMProviderService
|
||||
from ...services.llm_prompt_templates import render_prompt
|
||||
from .dictionary import DictionaryManager
|
||||
from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE
|
||||
|
||||
|
||||
# [DEF:MAX_RETRIES_PER_BATCH:Constant]
|
||||
# @PURPOSE: Maximum number of retries for a single batch before marking it failed.
|
||||
MAX_RETRIES_PER_BATCH = 3
|
||||
|
||||
|
||||
# [DEF:TranslationExecutor:Class]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Process translation batches: fetch source rows, filter dict, call LLM, persist results.
|
||||
# @PRE: DB session and config manager available.
|
||||
# @POST: Batches and records created with status tracking.
|
||||
# @SIDE_EFFECT: LLM API calls; DB writes.
|
||||
class TranslationExecutor:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db: Session,
|
||||
config_manager: ConfigManager,
|
||||
current_user: Optional[str] = None,
|
||||
on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None,
|
||||
):
|
||||
self.db = db
|
||||
self.config_manager = config_manager
|
||||
self.current_user = current_user
|
||||
self.on_batch_progress = on_batch_progress
|
||||
self._current_run_id: Optional[str] = None
|
||||
|
||||
# [DEF:execute_run:Function]
|
||||
# @PURPOSE: Run full translation execution for a TranslationRun.
|
||||
# @PRE: run is in PENDING or RUNNING status with valid job config.
|
||||
# @POST: Run is populated with batches and records.
|
||||
# @SIDE_EFFECT: LLM API calls; DB batch writes.
|
||||
def execute_run(
|
||||
self,
|
||||
run: TranslationRun,
|
||||
llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None,
|
||||
) -> TranslationRun:
|
||||
with belief_scope("TranslationExecutor.execute_run"):
|
||||
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
|
||||
if not job:
|
||||
raise ValueError(f"Job '{run.job_id}' not found for run '{run.id}'")
|
||||
|
||||
logger.reason("Starting translation execution", {
|
||||
"run_id": run.id,
|
||||
"job_id": job.id,
|
||||
"batch_size": job.batch_size,
|
||||
})
|
||||
|
||||
# Mark run as RUNNING
|
||||
run.status = "RUNNING"
|
||||
run.started_at = datetime.now(timezone.utc)
|
||||
self.db.flush()
|
||||
|
||||
# Fetch source rows from the accepted preview session
|
||||
source_rows = self._fetch_source_rows(job.id, run.id)
|
||||
if not source_rows:
|
||||
logger.explore("No source rows to translate", {"run_id": run.id})
|
||||
run.status = "COMPLETED"
|
||||
run.completed_at = datetime.now(timezone.utc)
|
||||
self.db.flush()
|
||||
return run
|
||||
|
||||
total_rows = len(source_rows)
|
||||
run.total_records = total_rows
|
||||
|
||||
# Split into batches
|
||||
batch_size = job.batch_size or 50
|
||||
batches = [
|
||||
source_rows[i:i + batch_size]
|
||||
for i in range(0, total_rows, batch_size)
|
||||
]
|
||||
|
||||
logger.reason(f"Processing {len(batches)} batches", {
|
||||
"run_id": run.id,
|
||||
"total_rows": total_rows,
|
||||
"batch_size": batch_size,
|
||||
})
|
||||
|
||||
successful_records = 0
|
||||
failed_records = 0
|
||||
skipped_records = 0
|
||||
|
||||
for batch_idx, batch_rows in enumerate(batches):
|
||||
batch_result = self._process_batch(
|
||||
job=job,
|
||||
run_id=run.id,
|
||||
batch_index=batch_idx,
|
||||
batch_rows=batch_rows,
|
||||
)
|
||||
successful_records += batch_result["successful"]
|
||||
failed_records += batch_result["failed"]
|
||||
skipped_records += batch_result["skipped"]
|
||||
|
||||
# Update run stats incrementally
|
||||
run.successful_records = successful_records
|
||||
run.failed_records = failed_records
|
||||
run.skipped_records = skipped_records
|
||||
self.db.flush()
|
||||
|
||||
if self.on_batch_progress:
|
||||
self.on_batch_progress(
|
||||
run.id, batch_idx + 1, len(batches),
|
||||
successful_records, total_rows,
|
||||
)
|
||||
|
||||
# Update final run status
|
||||
if failed_records == 0 and skipped_records == 0:
|
||||
run.status = "COMPLETED"
|
||||
elif successful_records == 0:
|
||||
run.status = "FAILED"
|
||||
else:
|
||||
run.status = "COMPLETED" # Partial success
|
||||
|
||||
run.completed_at = datetime.now(timezone.utc)
|
||||
self.db.flush()
|
||||
|
||||
logger.reflect("Translation execution complete", {
|
||||
"run_id": run.id,
|
||||
"status": run.status,
|
||||
"total": total_rows,
|
||||
"successful": successful_records,
|
||||
"failed": failed_records,
|
||||
"skipped": skipped_records,
|
||||
})
|
||||
|
||||
return run
|
||||
# [/DEF:execute_run:Function]
|
||||
|
||||
# [DEF:_fetch_source_rows:Function]
|
||||
# @PURPOSE: Fetch source rows from the accepted preview session for this job.
|
||||
# @PRE: job_id exists.
|
||||
# @POST: Returns list of dicts with source data.
|
||||
def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]:
|
||||
with belief_scope("TranslationExecutor._fetch_source_rows"):
|
||||
# Get the latest APPLIED preview session
|
||||
session = (
|
||||
self.db.query(TranslationPreviewSession)
|
||||
.filter(
|
||||
TranslationPreviewSession.job_id == job_id,
|
||||
TranslationPreviewSession.status == "APPLIED",
|
||||
)
|
||||
.order_by(TranslationPreviewSession.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
if not session:
|
||||
logger.explore("No accepted preview session found", {"job_id": job_id})
|
||||
return []
|
||||
|
||||
# Fetch APPROVED or all records from the session
|
||||
records = (
|
||||
self.db.query(TranslationPreviewRecord)
|
||||
.filter(
|
||||
TranslationPreviewRecord.session_id == session.id,
|
||||
TranslationPreviewRecord.status.in_(["APPROVED", "PENDING"]),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
source_rows = []
|
||||
for rec in records:
|
||||
source_rows.append({
|
||||
"row_index": rec.source_object_id or "0",
|
||||
"source_text": rec.source_sql or "",
|
||||
"approved_translation": rec.target_sql if rec.status == "APPROVED" else None,
|
||||
"source_object_name": rec.source_object_name or "",
|
||||
})
|
||||
|
||||
logger.reason(f"Fetched {len(source_rows)} source rows from preview", {
|
||||
"run_id": run_id,
|
||||
"session_id": session.id,
|
||||
})
|
||||
return source_rows
|
||||
# [/DEF:_fetch_source_rows:Function]
|
||||
|
||||
# [DEF:_process_batch:Function]
|
||||
# @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records.
|
||||
# @PRE: job and batch_rows are valid.
|
||||
# @POST: TranslationBatch and TranslationRecord rows are created.
|
||||
# @SIDE_EFFECT: LLM API call.
|
||||
def _process_batch(
|
||||
self,
|
||||
job: TranslationJob,
|
||||
run_id: str,
|
||||
batch_index: int,
|
||||
batch_rows: List[Dict[str, Any]],
|
||||
) -> Dict[str, int]:
|
||||
with belief_scope("TranslationExecutor._process_batch"):
|
||||
batch_start = time.monotonic()
|
||||
|
||||
# Create batch record
|
||||
batch = TranslationBatch(
|
||||
id=str(uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
batch_index=batch_index,
|
||||
status="RUNNING",
|
||||
total_records=len(batch_rows),
|
||||
started_at=datetime.now(timezone.utc),
|
||||
)
|
||||
self.db.add(batch)
|
||||
self.db.flush()
|
||||
batch_id = batch.id
|
||||
|
||||
result = {"successful": 0, "failed": 0, "skipped": 0, "retries": 0}
|
||||
|
||||
# Extract source texts for dict filtering
|
||||
source_texts = [
|
||||
row.get("source_text", "")
|
||||
for row in batch_rows
|
||||
if row.get("source_text")
|
||||
]
|
||||
|
||||
# Filter dictionary entries
|
||||
dict_matches = DictionaryManager.filter_for_batch(
|
||||
self.db, source_texts, job.id
|
||||
)
|
||||
|
||||
# For each row, determine if we need LLM translation or can use approved translation
|
||||
rows_for_llm = []
|
||||
pre_translated = []
|
||||
|
||||
for row in batch_rows:
|
||||
if row.get("approved_translation"):
|
||||
pre_translated.append(row)
|
||||
else:
|
||||
rows_for_llm.append(row)
|
||||
|
||||
# Handle pre-translated (approved) rows
|
||||
for row in pre_translated:
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
target_sql=row.get("approved_translation"),
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
status="SUCCESS",
|
||||
)
|
||||
self.db.add(record)
|
||||
result["successful"] += 1
|
||||
|
||||
# Process rows needing LLM translation
|
||||
if rows_for_llm:
|
||||
llm_result = self._call_llm_for_batch(
|
||||
job=job,
|
||||
run_id=run_id,
|
||||
batch_rows=rows_for_llm,
|
||||
dict_matches=dict_matches,
|
||||
batch_id=batch_id,
|
||||
)
|
||||
result["successful"] += llm_result["successful"]
|
||||
result["failed"] += llm_result["failed"]
|
||||
result["skipped"] += llm_result["skipped"]
|
||||
result["retries"] += llm_result.get("retries", 0)
|
||||
|
||||
# Update batch status
|
||||
batch.successful_records = result["successful"]
|
||||
batch.failed_records = result["failed"]
|
||||
batch.completed_at = datetime.now(timezone.utc)
|
||||
batch.status = "COMPLETED" if result["failed"] == 0 else "COMPLETED_WITH_ERRORS"
|
||||
self.db.flush()
|
||||
|
||||
batch_latency = int((time.monotonic() - batch_start) * 1000)
|
||||
logger.reason(f"Batch {batch_index} complete", {
|
||||
"batch_id": batch_id,
|
||||
"latency_ms": batch_latency,
|
||||
**result,
|
||||
})
|
||||
|
||||
return result
|
||||
# [/DEF:_process_batch:Function]
|
||||
|
||||
# [DEF:_call_llm_for_batch:Function]
|
||||
# @PURPOSE: Call LLM for a batch of rows requiring translation. Parse structured JSON response.
|
||||
# @PRE: job has valid provider_id. batch_rows is non-empty.
|
||||
# @POST: Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.
|
||||
# @SIDE_EFFECT: HTTP call to LLM provider.
|
||||
def _call_llm_for_batch(
|
||||
self,
|
||||
job: TranslationJob,
|
||||
run_id: str,
|
||||
batch_rows: List[Dict[str, Any]],
|
||||
dict_matches: List[Dict[str, Any]],
|
||||
batch_id: str,
|
||||
) -> Dict[str, int]:
|
||||
with belief_scope("TranslationExecutor._call_llm_for_batch"):
|
||||
# Build dictionary section
|
||||
dictionary_section = ""
|
||||
if dict_matches:
|
||||
glossary_lines = []
|
||||
for m in dict_matches:
|
||||
glossary_lines.append(
|
||||
f"- '{m['source_term']}' -> '{m['target_term']}'"
|
||||
f"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}"
|
||||
)
|
||||
dictionary_section = (
|
||||
"Terminology dictionary (use these translations when applicable):\n"
|
||||
+ "\n".join(glossary_lines)
|
||||
+ "\n\n"
|
||||
)
|
||||
|
||||
# Build rows JSON for LLM
|
||||
rows_json = json.dumps([
|
||||
{
|
||||
"row_id": str(row.get("row_index", idx)),
|
||||
"text": row.get("source_text", ""),
|
||||
}
|
||||
for idx, row in enumerate(batch_rows)
|
||||
], indent=2)
|
||||
|
||||
# Build prompt
|
||||
prompt = render_prompt(
|
||||
DEFAULT_EXECUTION_PROMPT_TEMPLATE,
|
||||
{
|
||||
"source_language": job.source_dialect or "SQL",
|
||||
"target_language": job.target_language or job.target_dialect or "en",
|
||||
"source_dialect": job.source_dialect or "",
|
||||
"target_dialect": job.target_dialect or "",
|
||||
"translation_column": job.translation_column or "",
|
||||
"dictionary_section": dictionary_section,
|
||||
"rows_json": rows_json,
|
||||
"row_count": str(len(batch_rows)),
|
||||
},
|
||||
)
|
||||
|
||||
# Call LLM with retry
|
||||
llm_response = None
|
||||
last_error = None
|
||||
retries = 0
|
||||
|
||||
for attempt in range(1, MAX_RETRIES_PER_BATCH + 1):
|
||||
try:
|
||||
llm_response = self._call_llm(job, prompt)
|
||||
break
|
||||
except Exception as e:
|
||||
last_error = str(e)
|
||||
retries += 1
|
||||
logger.explore(f"LLM call failed (attempt {attempt})", {
|
||||
"batch_id": batch_id,
|
||||
"error": last_error,
|
||||
"attempt": attempt,
|
||||
})
|
||||
if attempt < MAX_RETRIES_PER_BATCH:
|
||||
time.sleep(2 ** attempt) # Exponential backoff
|
||||
else:
|
||||
logger.explore("LLM call exhausted retries", {
|
||||
"batch_id": batch_id,
|
||||
"last_error": last_error,
|
||||
})
|
||||
|
||||
if llm_response is None:
|
||||
# All retries failed — mark all rows as failed
|
||||
for row in batch_rows:
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
target_sql=None,
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
status="FAILED",
|
||||
error_message=f"LLM call failed after {retries} retries: {last_error}",
|
||||
)
|
||||
self.db.add(record)
|
||||
return {"successful": 0, "failed": len(batch_rows), "skipped": 0, "retries": retries}
|
||||
|
||||
# Parse LLM response
|
||||
try:
|
||||
translations = self._parse_llm_response(llm_response, len(batch_rows))
|
||||
except ValueError as e:
|
||||
# Parse failure — mark all rows as SKIPPED
|
||||
logger.explore("LLM response parse failed", {
|
||||
"batch_id": batch_id,
|
||||
"error": str(e),
|
||||
"response_preview": llm_response[:500] if llm_response else "",
|
||||
})
|
||||
skipped = len(batch_rows)
|
||||
for row in batch_rows:
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
target_sql=None,
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
status="SKIPPED",
|
||||
error_message=f"LLM parse failure: {e}",
|
||||
)
|
||||
self.db.add(record)
|
||||
return {
|
||||
"successful": 0,
|
||||
"failed": 0,
|
||||
"skipped": skipped,
|
||||
"retries": retries,
|
||||
}
|
||||
|
||||
successful = 0
|
||||
failed = 0
|
||||
skipped = 0
|
||||
|
||||
for row in batch_rows:
|
||||
row_id = str(row.get("row_index", ""))
|
||||
translation = translations.get(row_id)
|
||||
|
||||
if translation is None:
|
||||
# NULL translation — skip
|
||||
skipped += 1
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
target_sql="",
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
status="SKIPPED",
|
||||
error_message="NULL translation returned by LLM",
|
||||
)
|
||||
self.db.add(record)
|
||||
continue
|
||||
|
||||
if translation.strip() == "":
|
||||
# Empty translation — skip
|
||||
skipped += 1
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
target_sql="",
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
status="SKIPPED",
|
||||
error_message="Empty translation returned by LLM",
|
||||
)
|
||||
self.db.add(record)
|
||||
continue
|
||||
|
||||
successful += 1
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
target_sql=translation,
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
status="SUCCESS",
|
||||
)
|
||||
self.db.add(record)
|
||||
|
||||
return {
|
||||
"successful": successful,
|
||||
"failed": failed,
|
||||
"skipped": skipped,
|
||||
"retries": retries,
|
||||
}
|
||||
# [/DEF:_call_llm_for_batch:Function]
|
||||
|
||||
# [DEF:_call_llm:Function]
|
||||
# @PURPOSE: Call the configured LLM provider with the batch prompt.
|
||||
# @PRE: job has valid provider_id.
|
||||
# @POST: Returns raw LLM response string.
|
||||
# @SIDE_EFFECT: HTTP call to LLM provider.
|
||||
def _call_llm(self, job: TranslationJob, prompt: str) -> str:
|
||||
with belief_scope("TranslationExecutor._call_llm"):
|
||||
if not job.provider_id:
|
||||
raise ValueError("Job has no LLM provider configured")
|
||||
|
||||
provider_svc = LLMProviderService(self.db)
|
||||
provider = provider_svc.get_provider(job.provider_id)
|
||||
if not provider:
|
||||
raise ValueError(f"LLM provider '{job.provider_id}' not found")
|
||||
|
||||
api_key = provider_svc.get_decrypted_api_key(job.provider_id)
|
||||
if not api_key:
|
||||
raise ValueError(f"Could not decrypt API key for provider '{job.provider_id}'")
|
||||
|
||||
model = provider.default_model or "gpt-4o-mini"
|
||||
provider_type = provider.provider_type.lower() if provider.provider_type else "openai"
|
||||
|
||||
if provider_type in ("openai", "openai_compatible"):
|
||||
return self._call_openai_compatible(
|
||||
base_url=provider.base_url,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported provider type '{provider_type}'")
|
||||
# [/DEF:_call_llm:Function]
|
||||
|
||||
# [DEF:_call_openai_compatible:Function]
|
||||
# @PURPOSE: Call OpenAI-compatible API for batch translation.
|
||||
# @PRE: Valid API endpoint, key, model, and prompt.
|
||||
# @POST: Returns response text.
|
||||
# @SIDE_EFFECT: HTTP POST to LLM API.
|
||||
@staticmethod
|
||||
def _call_openai_compatible(
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model: str,
|
||||
prompt: str,
|
||||
) -> str:
|
||||
with belief_scope("TranslationExecutor._call_openai_compatible"):
|
||||
import requests as http_requests
|
||||
|
||||
url = f"{base_url.rstrip('/')}/chat/completions"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics."},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"temperature": 0.1,
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
|
||||
response = http_requests.post(url, headers=headers, json=payload, timeout=180)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
choices = data.get("choices", [])
|
||||
if not choices:
|
||||
raise ValueError("LLM returned no choices")
|
||||
|
||||
content = choices[0].get("message", {}).get("content", "")
|
||||
if not content:
|
||||
raise ValueError("LLM returned empty content")
|
||||
|
||||
return content
|
||||
# [/DEF:_call_openai_compatible:Function]
|
||||
|
||||
# [DEF:_parse_llm_response:Function]
|
||||
# @PURPOSE: Parse LLM JSON response into dict of row_id -> translation.
|
||||
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
|
||||
# @POST: Returns dict mapping row_id to translation text.
|
||||
@staticmethod
|
||||
def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:
|
||||
with belief_scope("TranslationExecutor._parse_llm_response"):
|
||||
try:
|
||||
data = json.loads(response_text)
|
||||
except json.JSONDecodeError:
|
||||
# Try to extract from markdown code block
|
||||
import re
|
||||
match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', response_text, re.DOTALL)
|
||||
if match:
|
||||
try:
|
||||
data = json.loads(match.group(1))
|
||||
except json.JSONDecodeError:
|
||||
raise ValueError("LLM response was not valid JSON")
|
||||
else:
|
||||
raise ValueError("LLM response was not valid JSON")
|
||||
|
||||
rows = data.get("rows", [])
|
||||
if not isinstance(rows, list):
|
||||
raise ValueError("LLM response missing 'rows' array")
|
||||
|
||||
translations: Dict[str, str] = {}
|
||||
for item in rows:
|
||||
row_id = str(item.get("row_id", ""))
|
||||
translation = item.get("translation")
|
||||
if translation is None:
|
||||
# Skip NULL translations — they'll be handled by caller
|
||||
continue
|
||||
if row_id:
|
||||
translations[row_id] = str(translation)
|
||||
|
||||
return translations
|
||||
# [/DEF:_parse_llm_response:Function]
|
||||
|
||||
|
||||
# [/DEF:TranslationExecutor:Class]
|
||||
# [/DEF:TranslationExecutor:Module]
|
||||
171
backend/src/plugins/translate/metrics.py
Normal file
171
backend/src/plugins/translate/metrics.py
Normal file
@@ -0,0 +1,171 @@
|
||||
# [DEF:TranslationMetrics:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @SEMANTICS: translate, metrics, aggregation
|
||||
# @PURPOSE: Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [TranslationEvent:Class]
|
||||
# @RELATION: DEPENDS_ON -> [MetricSnapshot:Class]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationRun:Class]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationSchedule:Class]
|
||||
# @PRE: Database session is open.
|
||||
# @POST: Metrics are aggregated and returned; no side effects.
|
||||
# @RATIONALE: Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture.
|
||||
# @REJECTED: Querying all events for all time — would be slow; MetricSnapshot provides historical summary.
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import func
|
||||
|
||||
from ...core.logger import logger, belief_scope
|
||||
from ...models.translate import (
|
||||
TranslationEvent,
|
||||
MetricSnapshot,
|
||||
TranslationRun,
|
||||
TranslationSchedule,
|
||||
)
|
||||
|
||||
|
||||
# [DEF:TranslationMetrics:Class]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Aggregate translation metrics from live events and MetricSnapshot.
|
||||
class TranslationMetrics:
|
||||
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
|
||||
# [DEF:get_job_metrics:Function]
|
||||
# @PURPOSE: Get aggregated metrics for a specific job.
|
||||
# @PRE: job_id exists.
|
||||
# @POST: Returns dict with metrics from events + latest snapshot.
|
||||
def get_job_metrics(self, job_id: str) -> Dict[str, Any]:
|
||||
with belief_scope("TranslationMetrics.get_job_metrics"):
|
||||
# Run counts from TranslationRun
|
||||
run_counts = (
|
||||
self.db.query(
|
||||
TranslationRun.status,
|
||||
func.count(TranslationRun.id),
|
||||
)
|
||||
.filter(TranslationRun.job_id == job_id)
|
||||
.group_by(TranslationRun.status)
|
||||
.all()
|
||||
)
|
||||
|
||||
total_runs = 0
|
||||
status_counts: Dict[str, int] = {}
|
||||
for status_val, count in run_counts:
|
||||
status_counts[status_val] = count
|
||||
total_runs += count
|
||||
|
||||
# Aggregate record stats from runs
|
||||
record_stats = (
|
||||
self.db.query(
|
||||
func.coalesce(func.sum(TranslationRun.total_records), 0),
|
||||
func.coalesce(func.sum(TranslationRun.successful_records), 0),
|
||||
func.coalesce(func.sum(TranslationRun.failed_records), 0),
|
||||
func.coalesce(func.sum(TranslationRun.skipped_records), 0),
|
||||
)
|
||||
.filter(TranslationRun.job_id == job_id)
|
||||
.first()
|
||||
)
|
||||
total_records = record_stats[0] if record_stats else 0
|
||||
successful_records = record_stats[1] if record_stats else 0
|
||||
failed_records = record_stats[2] if record_stats else 0
|
||||
skipped_records = record_stats[3] if record_stats else 0
|
||||
|
||||
# Average duration from runs
|
||||
avg_duration = (
|
||||
self.db.query(
|
||||
func.avg(
|
||||
func.extract('epoch', TranslationRun.completed_at - TranslationRun.started_at) * 1000
|
||||
)
|
||||
)
|
||||
.filter(
|
||||
TranslationRun.job_id == job_id,
|
||||
TranslationRun.started_at.isnot(None),
|
||||
TranslationRun.completed_at.isnot(None),
|
||||
)
|
||||
.scalar()
|
||||
)
|
||||
|
||||
# Last run
|
||||
last_run = (
|
||||
self.db.query(TranslationRun)
|
||||
.filter(TranslationRun.job_id == job_id)
|
||||
.order_by(TranslationRun.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
|
||||
# Next scheduled run
|
||||
next_schedule = (
|
||||
self.db.query(TranslationSchedule)
|
||||
.filter(
|
||||
TranslationSchedule.job_id == job_id,
|
||||
TranslationSchedule.is_active == True,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
# Cumulative tokens/cost from MetricSnapshot + events
|
||||
cumulative_tokens = 0
|
||||
cumulative_cost = 0.0
|
||||
|
||||
# Latest MetricSnapshot
|
||||
latest_snapshot = (
|
||||
self.db.query(MetricSnapshot)
|
||||
.filter(MetricSnapshot.job_id == job_id)
|
||||
.order_by(MetricSnapshot.snapshot_date.desc())
|
||||
.first()
|
||||
)
|
||||
if latest_snapshot:
|
||||
# MetricSnapshot stores per-snapshot aggregated tokens/cost
|
||||
# Here we sum what we stored — in practice MetricSnapshot.covers_events_before
|
||||
# indicates cutoff
|
||||
pass
|
||||
|
||||
# Live events (<90 days) for token/cost
|
||||
cutoff = datetime.now(timezone.utc)
|
||||
live_events = (
|
||||
self.db.query(TranslationEvent)
|
||||
.filter(
|
||||
TranslationEvent.job_id == job_id,
|
||||
TranslationEvent.event_type.in_(["TRANSLATION_PHASE_COMPLETED", "RUN_COMPLETED"]),
|
||||
TranslationEvent.created_at > cutoff, # events newer than snapshot
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"total_runs": total_runs,
|
||||
"successful_runs": status_counts.get("COMPLETED", 0),
|
||||
"failed_runs": status_counts.get("FAILED", 0),
|
||||
"cancelled_runs": status_counts.get("CANCELLED", 0),
|
||||
"total_records": int(total_records),
|
||||
"successful_records": int(successful_records),
|
||||
"failed_records": int(failed_records),
|
||||
"skipped_records": int(skipped_records),
|
||||
"cumulative_tokens": cumulative_tokens,
|
||||
"cumulative_cost": cumulative_cost,
|
||||
"avg_duration_ms": int(avg_duration) if avg_duration else None,
|
||||
"last_run_at": last_run.created_at.isoformat() if last_run else None,
|
||||
"next_scheduled_run": next_schedule.last_run_at.isoformat() if next_schedule and next_schedule.last_run_at else None,
|
||||
}
|
||||
# [/DEF:get_job_metrics:Function]
|
||||
|
||||
# [DEF:get_all_metrics:Function]
|
||||
# @PURPOSE: Get aggregated metrics for all jobs.
|
||||
# @POST: Returns list of per-job metrics.
|
||||
def get_all_metrics(self) -> List[Dict[str, Any]]:
|
||||
with belief_scope("TranslationMetrics.get_all_metrics"):
|
||||
job_ids = (
|
||||
self.db.query(TranslationRun.job_id)
|
||||
.distinct()
|
||||
.all()
|
||||
)
|
||||
return [self.get_job_metrics(jid[0]) for jid in job_ids if jid[0]]
|
||||
# [/DEF:get_all_metrics:Function]
|
||||
|
||||
|
||||
# [/DEF:TranslationMetrics:Class]
|
||||
# [/DEF:TranslationMetrics:Module]
|
||||
855
backend/src/plugins/translate/orchestrator.py
Normal file
855
backend/src/plugins/translate/orchestrator.py
Normal file
@@ -0,0 +1,855 @@
|
||||
# [DEF:TranslationOrchestrator:Module]
|
||||
# @COMPLEXITY: 5
|
||||
# @SEMANTICS: translate, orchestrator, lifecycle, run
|
||||
# @PURPOSE: Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [TranslationRun]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationJob]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationExecutor]
|
||||
# @RELATION: DEPENDS_ON -> [SQLGenerator]
|
||||
# @RELATION: DEPENDS_ON -> [SupersetSqlLabExecutor]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationEventLog]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationPreviewSession]
|
||||
# @PRE: Valid job and accepted preview (for manual runs). Superset and LLM are reachable.
|
||||
# @POST: Translation run is executed, SQL generated and submitted, events recorded.
|
||||
# @SIDE_EFFECT: Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events.
|
||||
# @DATA_CONTRACT: Input[db, config_manager, current_user] -> Output[TranslationRun]
|
||||
# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run.
|
||||
# @RATIONALE: C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary.
|
||||
# @REJECTED: Distributed actor model (Celery) — eventual-consistency challenges at current scale.
|
||||
# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Callable, Tuple
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...core.logger import logger, belief_scope
|
||||
from ...core.config_manager import ConfigManager
|
||||
from ...models.translate import (
|
||||
TranslationJob,
|
||||
TranslationRun,
|
||||
TranslationBatch,
|
||||
TranslationRecord,
|
||||
TranslationPreviewSession,
|
||||
)
|
||||
from ...schemas.translate import TranslationRunResponse
|
||||
from .executor import TranslationExecutor
|
||||
from .sql_generator import SQLGenerator
|
||||
from .superset_executor import SupersetSqlLabExecutor
|
||||
from .events import TranslationEventLog
|
||||
from ..translate.service import TranslateJobService
|
||||
|
||||
|
||||
# [DEF:TranslationOrchestrator:Class]
|
||||
# @COMPLEXITY: 5
|
||||
# @PURPOSE: Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging.
|
||||
# @PRE: DB session and config manager are available.
|
||||
# @POST: Runs are created, executed, and finalized with event records.
|
||||
# @SIDE_EFFECT: Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows.
|
||||
# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED.
|
||||
class TranslationOrchestrator:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db: Session,
|
||||
config_manager: ConfigManager,
|
||||
current_user: Optional[str] = None,
|
||||
):
|
||||
self.db = db
|
||||
self.config_manager = config_manager
|
||||
self.current_user = current_user
|
||||
self.event_log = TranslationEventLog(db)
|
||||
self._job: Optional[TranslationJob] = None
|
||||
|
||||
# [DEF:start_run:Function]
|
||||
# @COMPLEXITY: 5
|
||||
# @PURPOSE: Start a new translation run for a job.
|
||||
# @PRE: job_id exists. For manual runs, there must be an accepted preview session.
|
||||
# @POST: TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.
|
||||
# @SIDE_EFFECT: DB writes.
|
||||
def start_run(
|
||||
self,
|
||||
job_id: str,
|
||||
is_scheduled: bool = False,
|
||||
trigger_type: Optional[str] = None,
|
||||
) -> TranslationRun:
|
||||
with belief_scope("TranslationOrchestrator.start_run"):
|
||||
# Load and validate job
|
||||
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
|
||||
if not job:
|
||||
raise ValueError(f"Translation job '{job_id}' not found")
|
||||
self._job = job
|
||||
|
||||
logger.reason("Starting translation run", {
|
||||
"job_id": job_id,
|
||||
"is_scheduled": is_scheduled,
|
||||
})
|
||||
|
||||
# Validate preconditions
|
||||
self._validate_preconditions(job, is_scheduled=is_scheduled)
|
||||
|
||||
# Compute hashes
|
||||
config_hash = self._compute_config_hash(job)
|
||||
dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id)
|
||||
|
||||
# Build config snapshot
|
||||
config_snapshot = {
|
||||
"source_dialect": job.source_dialect,
|
||||
"target_dialect": job.target_dialect,
|
||||
"database_dialect": job.database_dialect,
|
||||
"source_datasource_id": job.source_datasource_id,
|
||||
"source_table": job.source_table,
|
||||
"target_schema": job.target_schema,
|
||||
"target_table": job.target_table,
|
||||
"source_key_cols": job.source_key_cols,
|
||||
"target_key_cols": job.target_key_cols,
|
||||
"translation_column": job.translation_column,
|
||||
"context_columns": job.context_columns,
|
||||
"target_language": job.target_language,
|
||||
"provider_id": job.provider_id,
|
||||
"batch_size": job.batch_size,
|
||||
"upsert_strategy": job.upsert_strategy,
|
||||
"dictionary_ids": self._compute_dict_snapshot_hash(job_id),
|
||||
}
|
||||
|
||||
# Compute key_hash from source_key_cols
|
||||
import hashlib
|
||||
key_hash_input = json.dumps({
|
||||
"source_key_cols": job.source_key_cols,
|
||||
"source_datasource_id": job.source_datasource_id,
|
||||
"source_table": job.source_table,
|
||||
}, sort_keys=True)
|
||||
key_hash = hashlib.sha256(key_hash_input.encode()).hexdigest()[:16]
|
||||
|
||||
# Create run record with all hash/snapshot fields
|
||||
run = TranslationRun(
|
||||
id=str(uuid.uuid4()),
|
||||
job_id=job_id,
|
||||
status="PENDING",
|
||||
trigger_type=trigger_type or ("scheduled" if is_scheduled else "manual"),
|
||||
config_snapshot=config_snapshot,
|
||||
key_hash=key_hash,
|
||||
config_hash=config_hash,
|
||||
dict_snapshot_hash=dict_snapshot_hash,
|
||||
created_by=self.current_user,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
self.db.add(run)
|
||||
self.db.flush()
|
||||
|
||||
# Record event
|
||||
self.event_log.log_event(
|
||||
job_id=job_id,
|
||||
run_id=run.id,
|
||||
event_type="RUN_STARTED",
|
||||
payload={
|
||||
"is_scheduled": is_scheduled,
|
||||
"trigger_type": run.trigger_type,
|
||||
"config_hash": config_hash,
|
||||
"dict_snapshot_hash": dict_snapshot_hash,
|
||||
"key_hash": key_hash,
|
||||
},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
self.db.commit()
|
||||
self.db.refresh(run)
|
||||
|
||||
logger.reflect("Run created", {
|
||||
"run_id": run.id,
|
||||
"job_id": job_id,
|
||||
"status": run.status,
|
||||
"trigger_type": run.trigger_type,
|
||||
})
|
||||
return run
|
||||
# [/DEF:start_run:Function]
|
||||
|
||||
# [DEF:execute_run:Function]
|
||||
# @PURPOSE: Execute a translation run: dispatch executor, generate SQL, submit to Superset.
|
||||
# @PRE: run is in PENDING status.
|
||||
# @POST: Run is executed, SQL generated, Superset submission attempted.
|
||||
# @SIDE_EFFECT: LLM calls, DB writes, Superset API calls.
|
||||
def execute_run(
|
||||
self,
|
||||
run: TranslationRun,
|
||||
on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None,
|
||||
skip_insert: bool = False,
|
||||
) -> TranslationRun:
|
||||
with belief_scope("TranslationOrchestrator.execute_run"):
|
||||
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
|
||||
if not job:
|
||||
raise ValueError(f"Job '{run.job_id}' not found")
|
||||
self._job = job
|
||||
|
||||
if run.status != "PENDING":
|
||||
raise ValueError(
|
||||
f"Cannot execute run in status '{run.status}'. "
|
||||
f"Run must be in PENDING status."
|
||||
)
|
||||
|
||||
logger.reason("Executing run", {
|
||||
"run_id": run.id,
|
||||
"job_id": job.id,
|
||||
"skip_insert": skip_insert,
|
||||
})
|
||||
|
||||
# Record translation phase start
|
||||
self.event_log.log_event(
|
||||
job_id=job.id,
|
||||
run_id=run.id,
|
||||
event_type="TRANSLATION_PHASE_STARTED",
|
||||
payload={},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
# Dispatch executor
|
||||
executor = TranslationExecutor(
|
||||
self.db, self.config_manager, self.current_user,
|
||||
on_batch_progress=on_batch_progress,
|
||||
)
|
||||
try:
|
||||
run = executor.execute_run(run, llm_progress_callback=None)
|
||||
except Exception as e:
|
||||
logger.explore("Translation execution failed", {
|
||||
"run_id": run.id,
|
||||
"error": str(e),
|
||||
})
|
||||
run.status = "FAILED"
|
||||
run.error_message = f"Translation execution failed: {e}"
|
||||
run.completed_at = datetime.now(timezone.utc)
|
||||
self.db.flush()
|
||||
self.event_log.log_event(
|
||||
job_id=job.id,
|
||||
run_id=run.id,
|
||||
event_type="RUN_FAILED",
|
||||
payload={"error": str(e), "phase": "translation"},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
self.db.commit()
|
||||
return run
|
||||
|
||||
# Record translation phase complete
|
||||
self.event_log.log_event(
|
||||
job_id=job.id,
|
||||
run_id=run.id,
|
||||
event_type="TRANSLATION_PHASE_COMPLETED",
|
||||
payload={
|
||||
"total": run.total_records,
|
||||
"successful": run.successful_records,
|
||||
"failed": run.failed_records,
|
||||
"skipped": run.skipped_records,
|
||||
},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
# Skip insert phase if requested (e.g., for preview-only execution)
|
||||
if skip_insert:
|
||||
run.status = "COMPLETED"
|
||||
run.completed_at = datetime.now(timezone.utc)
|
||||
self.db.flush()
|
||||
self.event_log.log_event(
|
||||
job_id=job.id,
|
||||
run_id=run.id,
|
||||
event_type="RUN_COMPLETED",
|
||||
payload={"skip_insert": True},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
self.db.commit()
|
||||
return run
|
||||
|
||||
# Generate SQL and submit to Superset
|
||||
insert_result = self._generate_and_insert_sql(job, run)
|
||||
run.insert_status = insert_result.get("status")
|
||||
run.superset_execution_id = str(insert_result.get("query_id") or "")
|
||||
run.superset_execution_log = insert_result
|
||||
run.status = "COMPLETED" if insert_result.get("status") == "success" else "COMPLETED"
|
||||
if insert_result.get("error_message"):
|
||||
run.error_message = insert_result["error_message"]
|
||||
run.completed_at = datetime.now(timezone.utc)
|
||||
self.db.flush()
|
||||
|
||||
# Record terminal event
|
||||
terminal_event = "RUN_COMPLETED"
|
||||
self.event_log.log_event(
|
||||
job_id=job.id,
|
||||
run_id=run.id,
|
||||
event_type=terminal_event,
|
||||
payload={
|
||||
"insert_status": insert_result.get("status"),
|
||||
"query_id": insert_result.get("query_id"),
|
||||
"rows_affected": insert_result.get("rows_affected"),
|
||||
"total_records": run.total_records,
|
||||
"successful": run.successful_records,
|
||||
"failed": run.failed_records,
|
||||
"skipped": run.skipped_records,
|
||||
},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
self.db.commit()
|
||||
self.db.refresh(run)
|
||||
|
||||
logger.reflect("Run execution complete", {
|
||||
"run_id": run.id,
|
||||
"status": run.status,
|
||||
"insert_status": run.insert_status,
|
||||
})
|
||||
return run
|
||||
# [/DEF:execute_run:Function]
|
||||
|
||||
# [DEF:_generate_and_insert_sql:Function]
|
||||
# @PURPOSE: Generate INSERT SQL from successful records and submit to Superset SQL Lab.
|
||||
# @PRE: job has target table configured. run has successful records.
|
||||
# @POST: SQL is generated and submitted. Returns execution result.
|
||||
# @SIDE_EFFECT: Superset API call.
|
||||
def _generate_and_insert_sql(
|
||||
self,
|
||||
job: TranslationJob,
|
||||
run: TranslationRun,
|
||||
) -> Dict[str, Any]:
|
||||
with belief_scope("TranslationOrchestrator._generate_and_insert_sql"):
|
||||
# Fetch successful records
|
||||
records = (
|
||||
self.db.query(TranslationRecord)
|
||||
.filter(
|
||||
TranslationRecord.run_id == run.id,
|
||||
TranslationRecord.status == "SUCCESS",
|
||||
TranslationRecord.target_sql.isnot(None),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
if not records:
|
||||
logger.reason("No successful records to insert", {"run_id": run.id})
|
||||
return {"status": "skipped", "reason": "no_records", "query_id": None}
|
||||
|
||||
logger.reason(f"Generating SQL for {len(records)} records", {
|
||||
"run_id": run.id,
|
||||
"dialect": job.database_dialect or job.target_dialect,
|
||||
})
|
||||
|
||||
# Build rows for SQL generation
|
||||
columns = job.context_columns or []
|
||||
if job.translation_column and job.translation_column not in columns:
|
||||
columns.append(job.translation_column)
|
||||
|
||||
# Also include key columns if used for upsert
|
||||
if job.target_key_cols:
|
||||
for k in job.target_key_cols:
|
||||
if k not in columns:
|
||||
columns.append(k)
|
||||
|
||||
rows_for_sql = []
|
||||
for rec in records:
|
||||
row_data = {}
|
||||
if job.context_columns:
|
||||
for col in job.context_columns:
|
||||
row_data[col] = ""
|
||||
if job.translation_column:
|
||||
row_data[job.translation_column] = rec.target_sql or ""
|
||||
if job.target_key_cols:
|
||||
for k in job.target_key_cols:
|
||||
row_data[k] = ""
|
||||
rows_for_sql.append(row_data)
|
||||
|
||||
if not columns:
|
||||
# Use target_sql as the sole column
|
||||
columns = [job.translation_column or "translated_text"]
|
||||
rows_for_sql = [{columns[0]: rec.target_sql or ""} for rec in records]
|
||||
|
||||
# Generate SQL
|
||||
try:
|
||||
sql, row_count = SQLGenerator.generate(
|
||||
dialect=job.database_dialect or job.target_dialect or "postgresql",
|
||||
target_schema=job.target_schema,
|
||||
target_table=job.target_table or "translated_data",
|
||||
columns=columns,
|
||||
rows=rows_for_sql,
|
||||
key_columns=job.target_key_cols,
|
||||
upsert_strategy=job.upsert_strategy or "MERGE",
|
||||
)
|
||||
except ValueError as e:
|
||||
logger.explore("SQL generation failed", {"error": str(e)})
|
||||
return {"status": "failed", "error_message": str(e), "query_id": None}
|
||||
|
||||
# Log insert phase start
|
||||
self.event_log.log_event(
|
||||
job_id=job.id,
|
||||
run_id=run.id,
|
||||
event_type="INSERT_PHASE_STARTED",
|
||||
payload={"sql_length": len(sql), "row_count": row_count},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
# Submit to Superset
|
||||
try:
|
||||
env_id = job.source_dialect or ""
|
||||
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
|
||||
result = executor.execute_and_poll(
|
||||
sql=sql,
|
||||
max_polls=30,
|
||||
poll_interval_seconds=2.0,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.explore("Superset SQL submission failed", {
|
||||
"run_id": run.id,
|
||||
"error": str(e),
|
||||
})
|
||||
result = {"status": "failed", "error_message": str(e), "query_id": None}
|
||||
|
||||
# Log insert phase complete
|
||||
self.event_log.log_event(
|
||||
job_id=job.id,
|
||||
run_id=run.id,
|
||||
event_type="INSERT_PHASE_COMPLETED",
|
||||
payload=result,
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
return result
|
||||
# [/DEF:_generate_and_insert_sql:Function]
|
||||
|
||||
# [DEF:_validate_preconditions:Function]
|
||||
# @PURPOSE: Validate preconditions before starting a run.
|
||||
# @PRE: None.
|
||||
# @POST: Raises ValueError if preconditions are not met.
|
||||
# @SIDE_EFFECT: None.
|
||||
def _validate_preconditions(
|
||||
self,
|
||||
job: TranslationJob,
|
||||
is_scheduled: bool = False,
|
||||
) -> None:
|
||||
with belief_scope("TranslationOrchestrator._validate_preconditions"):
|
||||
# Job must be in valid status
|
||||
if job.status in ("DRAFT",):
|
||||
raise ValueError(
|
||||
f"Cannot run job '{job.id}' in status '{job.status}'. "
|
||||
f"Job must be READY, ACTIVE, or COMPLETED."
|
||||
)
|
||||
|
||||
# Must have target table configured
|
||||
if not job.target_table:
|
||||
raise ValueError(
|
||||
f"Job '{job.id}' has no target table configured. "
|
||||
"Configure a target table before running."
|
||||
)
|
||||
|
||||
# Must have LLM provider configured
|
||||
if not job.provider_id:
|
||||
raise ValueError(
|
||||
f"Job '{job.id}' has no LLM provider configured. "
|
||||
"Select an LLM provider before running."
|
||||
)
|
||||
|
||||
# Must have a translation column
|
||||
if not job.translation_column:
|
||||
raise ValueError(
|
||||
f"Job '{job.id}' has no translation column configured. "
|
||||
"Select a translation column before running."
|
||||
)
|
||||
|
||||
# For manual runs, must have accepted preview
|
||||
if not is_scheduled:
|
||||
accepted_session = (
|
||||
self.db.query(TranslationPreviewSession)
|
||||
.filter(
|
||||
TranslationPreviewSession.job_id == job.id,
|
||||
TranslationPreviewSession.status == "APPLIED",
|
||||
)
|
||||
.order_by(TranslationPreviewSession.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
if not accepted_session:
|
||||
raise ValueError(
|
||||
f"Job '{job.id}' has no accepted preview session. "
|
||||
"Run and accept a preview before executing a manual translation run."
|
||||
)
|
||||
|
||||
logger.reason("Preconditions validated", {
|
||||
"job_id": job.id,
|
||||
"is_scheduled": is_scheduled,
|
||||
})
|
||||
# [/DEF:_validate_preconditions:Function]
|
||||
|
||||
# [DEF:retry_failed_batches:Function]
|
||||
# @PURPOSE: Retry failed batches in a run.
|
||||
# @PRE: run exists and has failed batches.
|
||||
# @POST: Failed batches are re-processed.
|
||||
# @SIDE_EFFECT: LLM calls, DB writes.
|
||||
def retry_failed_batches(
|
||||
self,
|
||||
run_id: str,
|
||||
) -> TranslationRun:
|
||||
with belief_scope("TranslationOrchestrator.retry_failed_batches"):
|
||||
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
|
||||
if not run:
|
||||
raise ValueError(f"Run '{run_id}' not found")
|
||||
|
||||
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
|
||||
if not job:
|
||||
raise ValueError(f"Job '{run.job_id}' not found")
|
||||
self._job = job
|
||||
|
||||
# Find failed batches
|
||||
failed_batches = (
|
||||
self.db.query(TranslationBatch)
|
||||
.filter(
|
||||
TranslationBatch.run_id == run_id,
|
||||
TranslationBatch.status.in_(["FAILED", "COMPLETED_WITH_ERRORS"]),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
if not failed_batches:
|
||||
raise ValueError(f"No failed batches found for run '{run_id}'")
|
||||
|
||||
logger.reason("Retrying failed batches", {
|
||||
"run_id": run_id,
|
||||
"batch_count": len(failed_batches),
|
||||
})
|
||||
|
||||
self.event_log.log_event(
|
||||
job_id=job.id,
|
||||
run_id=run.id,
|
||||
event_type="RUN_RETRYING",
|
||||
payload={"batch_count": len(failed_batches)},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
# Re-process each failed batch
|
||||
executor = TranslationExecutor(self.db, self.config_manager, self.current_user)
|
||||
run.status = "RUNNING"
|
||||
self.db.flush()
|
||||
|
||||
for batch in failed_batches:
|
||||
# Fetch failed records for this batch
|
||||
failed_records = (
|
||||
self.db.query(TranslationRecord)
|
||||
.filter(
|
||||
TranslationRecord.batch_id == batch.id,
|
||||
TranslationRecord.status == "FAILED",
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
if not failed_records:
|
||||
continue
|
||||
|
||||
# Build rows from failed records
|
||||
retry_rows = []
|
||||
for rec in failed_records:
|
||||
retry_rows.append({
|
||||
"row_index": rec.source_object_id or "0",
|
||||
"source_text": rec.source_sql or "",
|
||||
"approved_translation": None,
|
||||
"source_object_name": rec.source_object_name or "",
|
||||
})
|
||||
|
||||
# Process retry batch
|
||||
result = executor._process_batch(
|
||||
job=job,
|
||||
run_id=run_id,
|
||||
batch_index=batch.batch_index,
|
||||
batch_rows=retry_rows,
|
||||
)
|
||||
|
||||
# Update run stats
|
||||
run.successful_records = (run.successful_records or 0) + result["successful"]
|
||||
run.failed_records = (run.failed_records or 0) + result["failed"]
|
||||
run.skipped_records = (run.skipped_records or 0) + result["skipped"]
|
||||
self.db.flush()
|
||||
|
||||
# Update run status
|
||||
if run.failed_records == 0:
|
||||
run.status = "COMPLETED"
|
||||
elif run.successful_records == 0:
|
||||
run.status = "FAILED"
|
||||
else:
|
||||
run.status = "COMPLETED"
|
||||
|
||||
run.completed_at = datetime.now(timezone.utc)
|
||||
self.db.flush()
|
||||
|
||||
self.event_log.log_event(
|
||||
job_id=job.id,
|
||||
run_id=run.id,
|
||||
event_type="RUN_COMPLETED",
|
||||
payload={"retry": True},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
self.db.commit()
|
||||
logger.reflect("Retry complete", {
|
||||
"run_id": run_id,
|
||||
"status": run.status,
|
||||
})
|
||||
return run
|
||||
# [/DEF:retry_failed_batches:Function]
|
||||
|
||||
# [DEF:retry_insert:Function]
|
||||
# @PURPOSE: Retry the SQL insert phase for a completed run.
|
||||
# @PRE: run exists and has successful records.
|
||||
# @POST: SQL is regenerated and re-submitted to Superset.
|
||||
# @SIDE_EFFECT: Superset API call.
|
||||
def retry_insert(
|
||||
self,
|
||||
run_id: str,
|
||||
) -> TranslationRun:
|
||||
with belief_scope("TranslationOrchestrator.retry_insert"):
|
||||
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
|
||||
if not run:
|
||||
raise ValueError(f"Run '{run_id}' not found")
|
||||
|
||||
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
|
||||
if not job:
|
||||
raise ValueError(f"Job '{run.job_id}' not found")
|
||||
self._job = job
|
||||
|
||||
logger.reason("Retrying insert phase", {
|
||||
"run_id": run_id,
|
||||
})
|
||||
|
||||
self.event_log.log_event(
|
||||
job_id=job.id,
|
||||
run_id=run.id,
|
||||
event_type="RUN_RETRY_INSERT",
|
||||
payload={},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
# Regenerate SQL and submit
|
||||
insert_result = self._generate_and_insert_sql(job, run)
|
||||
|
||||
# Update run
|
||||
run.insert_status = insert_result.get("status")
|
||||
run.superset_execution_id = str(insert_result.get("query_id") or "")
|
||||
if insert_result.get("error_message"):
|
||||
run.error_message = insert_result["error_message"]
|
||||
self.db.flush()
|
||||
self.db.commit()
|
||||
self.db.refresh(run)
|
||||
|
||||
logger.reflect("Insert retry complete", {
|
||||
"run_id": run_id,
|
||||
"insert_status": run.insert_status,
|
||||
})
|
||||
return run
|
||||
# [/DEF:retry_insert:Function]
|
||||
|
||||
# [DEF:cancel_run:Function]
|
||||
# @PURPOSE: Cancel a running translation.
|
||||
# @PRE: run is in PENDING or RUNNING status.
|
||||
# @POST: Run status is set to CANCELLED.
|
||||
# @SIDE_EFFECT: DB write; records event.
|
||||
def cancel_run(self, run_id: str) -> TranslationRun:
|
||||
with belief_scope("TranslationOrchestrator.cancel_run"):
|
||||
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
|
||||
if not run:
|
||||
raise ValueError(f"Run '{run_id}' not found")
|
||||
|
||||
if run.status not in ("PENDING", "RUNNING"):
|
||||
raise ValueError(
|
||||
f"Cannot cancel run in status '{run.status}'. "
|
||||
f"Only PENDING or RUNNING runs can be cancelled."
|
||||
)
|
||||
|
||||
run.status = "CANCELLED"
|
||||
run.completed_at = datetime.now(timezone.utc)
|
||||
self.db.flush()
|
||||
|
||||
self.event_log.log_event(
|
||||
job_id=run.job_id,
|
||||
run_id=run.id,
|
||||
event_type="RUN_CANCELLED",
|
||||
payload={},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
self.db.commit()
|
||||
self.db.refresh(run)
|
||||
|
||||
logger.reflect("Run cancelled", {
|
||||
"run_id": run_id,
|
||||
})
|
||||
return run
|
||||
# [/DEF:cancel_run:Function]
|
||||
|
||||
# [DEF:get_run_status:Function]
|
||||
# @PURPOSE: Get run status with statistics.
|
||||
# @PRE: run_id exists.
|
||||
# @POST: Returns dict with run details.
|
||||
def get_run_status(self, run_id: str) -> Dict[str, Any]:
|
||||
with belief_scope("TranslationOrchestrator.get_run_status"):
|
||||
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
|
||||
if not run:
|
||||
raise ValueError(f"Run '{run_id}' not found")
|
||||
|
||||
# Count batches
|
||||
batch_count = (
|
||||
self.db.query(TranslationBatch)
|
||||
.filter(TranslationBatch.run_id == run_id)
|
||||
.count()
|
||||
)
|
||||
|
||||
# Get event summary
|
||||
event_summary = self.event_log.get_run_event_summary(run_id)
|
||||
|
||||
return {
|
||||
"id": run.id,
|
||||
"job_id": run.job_id,
|
||||
"status": run.status,
|
||||
"started_at": run.started_at.isoformat() if run.started_at else None,
|
||||
"completed_at": run.completed_at.isoformat() if run.completed_at else None,
|
||||
"error_message": run.error_message,
|
||||
"total_records": run.total_records or 0,
|
||||
"successful_records": run.successful_records or 0,
|
||||
"failed_records": run.failed_records or 0,
|
||||
"skipped_records": run.skipped_records or 0,
|
||||
"insert_status": run.insert_status,
|
||||
"superset_execution_id": run.superset_execution_id,
|
||||
"batch_count": batch_count,
|
||||
"event_invariants": {
|
||||
"has_run_started": event_summary["has_run_started"],
|
||||
"terminal_event_count": event_summary["terminal_event_count"],
|
||||
"invariant_valid": event_summary["invariant_valid"],
|
||||
},
|
||||
"created_by": run.created_by,
|
||||
"created_at": run.created_at.isoformat() if run.created_at else None,
|
||||
}
|
||||
# [/DEF:get_run_status:Function]
|
||||
|
||||
# [DEF:get_run_records:Function]
|
||||
# @PURPOSE: Get paginated records for a run.
|
||||
# @PRE: run_id exists.
|
||||
# @POST: Returns dict with records and pagination info.
|
||||
def get_run_records(
|
||||
self,
|
||||
run_id: str,
|
||||
page: int = 1,
|
||||
page_size: int = 50,
|
||||
status_filter: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
with belief_scope("TranslationOrchestrator.get_run_records"):
|
||||
query = self.db.query(TranslationRecord).filter(
|
||||
TranslationRecord.run_id == run_id
|
||||
)
|
||||
|
||||
if status_filter:
|
||||
query = query.filter(TranslationRecord.status == status_filter)
|
||||
|
||||
total = query.count()
|
||||
offset = (page - 1) * page_size
|
||||
records = (
|
||||
query.order_by(TranslationRecord.created_at.desc())
|
||||
.offset(offset)
|
||||
.limit(page_size)
|
||||
.all()
|
||||
)
|
||||
|
||||
return {
|
||||
"items": [
|
||||
{
|
||||
"id": r.id,
|
||||
"batch_id": r.batch_id,
|
||||
"source_sql": r.source_sql,
|
||||
"target_sql": r.target_sql,
|
||||
"source_object_type": r.source_object_type,
|
||||
"source_object_id": r.source_object_id,
|
||||
"source_object_name": r.source_object_name,
|
||||
"status": r.status,
|
||||
"error_message": r.error_message,
|
||||
"created_at": r.created_at.isoformat() if r.created_at else None,
|
||||
}
|
||||
for r in records
|
||||
],
|
||||
"total": total,
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
"status_filter": status_filter,
|
||||
}
|
||||
# [/DEF:get_run_records:Function]
|
||||
|
||||
# [DEF:get_run_history:Function]
|
||||
# @PURPOSE: Get run history for a job.
|
||||
# @PRE: job_id exists.
|
||||
# @POST: Returns list of runs.
|
||||
def get_run_history(
|
||||
self,
|
||||
job_id: str,
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
) -> Tuple[int, List[Dict[str, Any]]]:
|
||||
with belief_scope("TranslationOrchestrator.get_run_history"):
|
||||
query = self.db.query(TranslationRun).filter(
|
||||
TranslationRun.job_id == job_id
|
||||
)
|
||||
total = query.count()
|
||||
runs = (
|
||||
query.order_by(TranslationRun.created_at.desc())
|
||||
.offset((page - 1) * page_size)
|
||||
.limit(page_size)
|
||||
.all()
|
||||
)
|
||||
|
||||
return total, [
|
||||
{
|
||||
"id": r.id,
|
||||
"job_id": r.job_id,
|
||||
"status": r.status,
|
||||
"started_at": r.started_at.isoformat() if r.started_at else None,
|
||||
"completed_at": r.completed_at.isoformat() if r.completed_at else None,
|
||||
"error_message": r.error_message,
|
||||
"total_records": r.total_records or 0,
|
||||
"successful_records": r.successful_records or 0,
|
||||
"failed_records": r.failed_records or 0,
|
||||
"skipped_records": r.skipped_records or 0,
|
||||
"insert_status": r.insert_status,
|
||||
"superset_execution_id": r.superset_execution_id,
|
||||
"created_by": r.created_by,
|
||||
"created_at": r.created_at.isoformat() if r.created_at else None,
|
||||
}
|
||||
for r in runs
|
||||
]
|
||||
# [/DEF:get_run_history:Function]
|
||||
|
||||
# [DEF:_compute_config_hash:Function]
|
||||
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
|
||||
@staticmethod
|
||||
def _compute_config_hash(job: TranslationJob) -> str:
|
||||
import hashlib
|
||||
config_str = json.dumps({
|
||||
"source_dialect": job.source_dialect,
|
||||
"target_dialect": job.target_dialect,
|
||||
"source_datasource_id": job.source_datasource_id,
|
||||
"translation_column": job.translation_column,
|
||||
"context_columns": job.context_columns,
|
||||
"target_language": job.target_language,
|
||||
"provider_id": job.provider_id,
|
||||
"batch_size": job.batch_size,
|
||||
"upsert_strategy": job.upsert_strategy,
|
||||
}, sort_keys=True)
|
||||
return hashlib.sha256(config_str.encode()).hexdigest()[:16]
|
||||
# [/DEF:_compute_config_hash:Function]
|
||||
|
||||
# [DEF:_compute_dict_snapshot_hash:Function]
|
||||
# @PURPOSE: Compute a hash of dictionary state for snapshot comparison.
|
||||
def _compute_dict_snapshot_hash(self, job_id: str) -> str:
|
||||
import hashlib
|
||||
from ...models.translate import TranslationJobDictionary
|
||||
dict_links = (
|
||||
self.db.query(TranslationJobDictionary)
|
||||
.filter(TranslationJobDictionary.job_id == job_id)
|
||||
.all()
|
||||
)
|
||||
dict_ids = sorted([dl.dictionary_id for dl in dict_links])
|
||||
hash_input = ",".join(dict_ids)
|
||||
return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
|
||||
# [/DEF:_compute_dict_snapshot_hash:Function]
|
||||
|
||||
|
||||
# [/DEF:TranslationOrchestrator:Class]
|
||||
# [/DEF:TranslationOrchestrator:Module]
|
||||
62
backend/src/plugins/translate/plugin.py
Normal file
62
backend/src/plugins/translate/plugin.py
Normal file
@@ -0,0 +1,62 @@
|
||||
# [DEF:TranslatePlugin:Module]
|
||||
# @COMPLEXITY: 2
|
||||
# @SEMANTICS: plugin, translate, llm, sql, dashboard
|
||||
# @PURPOSE: TranslatePlugin skeleton for cross-dialect SQL and dashboard translation.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: INHERITS -> [PluginBase]
|
||||
# @RATIONALE: Separate plugin avoids bloating LLMAnalysisPlugin beyond fractal limit (<400 lines).
|
||||
# @REJECTED: Extending LLMAnalysisPlugin would conflate two distinct feature domains.
|
||||
|
||||
from typing import Dict, Any, Optional
|
||||
from ...core.plugin_base import PluginBase
|
||||
|
||||
|
||||
# [DEF:TranslatePlugin:Class]
|
||||
# @PURPOSE: Plugin for translating SQL queries and dashboard definitions across database dialects.
|
||||
# @RELATION: IMPLEMENTS -> backend.src.core.plugin_base.PluginBase
|
||||
class TranslatePlugin(PluginBase):
|
||||
@property
|
||||
def id(self) -> str:
|
||||
return "translate"
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "LLM Table Translation"
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return "Cross-dialect SQL and dashboard translation using LLMs."
|
||||
|
||||
@property
|
||||
def version(self) -> str:
|
||||
return "1.0.0"
|
||||
|
||||
def get_schema(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"source_dialect": {
|
||||
"type": "string",
|
||||
"title": "Source Dialect",
|
||||
"description": "Source database dialect (e.g. PostgreSQL, ClickHouse)",
|
||||
},
|
||||
"target_dialect": {
|
||||
"type": "string",
|
||||
"title": "Target Dialect",
|
||||
"description": "Target database dialect (e.g. PostgreSQL, ClickHouse)",
|
||||
},
|
||||
"job_id": {
|
||||
"type": "string",
|
||||
"title": "Translation Job ID",
|
||||
"description": "Existing translation job to execute",
|
||||
},
|
||||
},
|
||||
"required": ["job_id"],
|
||||
}
|
||||
|
||||
async def execute(self, params: Dict[str, Any], context: Optional[Any] = None):
|
||||
"""Execute a translation job — implementation deferred to later phases."""
|
||||
raise NotImplementedError("TranslatePlugin.execute not yet implemented")
|
||||
# [/DEF:TranslatePlugin:Class]
|
||||
|
||||
# [/DEF:TranslatePlugin:Module]
|
||||
788
backend/src/plugins/translate/preview.py
Normal file
788
backend/src/plugins/translate/preview.py
Normal file
@@ -0,0 +1,788 @@
|
||||
# [DEF:TranslationPreview:Module]
|
||||
# @COMPLEXITY: 4
|
||||
# @SEMANTICS: translate, preview, llm, session
|
||||
# @PURPOSE: Preview session management: fetch sample rows from Superset, send to LLM with context + filtered dictionary, return side-by-side results.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [TranslationJob:Class]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationPreviewSession:Class]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationPreviewRecord:Class]
|
||||
# @RELATION: DEPENDS_ON -> [LLMProviderService]
|
||||
# @RELATION: DEPENDS_ON -> [DictionaryManager]
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION: DEPENDS_ON -> [render_prompt]
|
||||
# @RELATION: DEPENDS_ON -> [ConfigManager]
|
||||
# @PRE: Database session and config manager are available.
|
||||
# @POST: Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected.
|
||||
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows.
|
||||
# @RATIONALE: C4 because preview is stateful with approve/edit/reject lifecycle and LLM API calls.
|
||||
# @REJECTED: Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably.
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from sqlalchemy.orm import Session
|
||||
from datetime import datetime, timezone, timedelta
|
||||
import uuid
|
||||
import json
|
||||
import hashlib
|
||||
import re
|
||||
|
||||
from ...core.logger import logger, belief_scope
|
||||
from ...core.config_manager import ConfigManager
|
||||
from ...core.superset_client import SupersetClient
|
||||
from ...models.translate import (
|
||||
TranslationJob,
|
||||
TranslationPreviewSession,
|
||||
TranslationPreviewRecord,
|
||||
TranslationJobDictionary,
|
||||
)
|
||||
from ...services.llm_provider import LLMProviderService
|
||||
from ...services.llm_prompt_templates import render_prompt
|
||||
from .dictionary import DictionaryManager
|
||||
|
||||
# [DEF:DEFAULT_EXECUTION_PROMPT_TEMPLATE:Constant]
|
||||
# @PURPOSE: Default prompt template for batch LLM translation execution (no context columns — faster).
|
||||
DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
|
||||
"Translate the following database content from {source_language} to {target_language}.\n\n"
|
||||
"Source dialect: {source_dialect}\n"
|
||||
"Target dialect: {target_dialect}\n"
|
||||
"Column to translate: {translation_column}\n\n"
|
||||
"{dictionary_section}"
|
||||
"For each row, provide an accurate translation of the text.\n\n"
|
||||
"Rows to translate:\n{rows_json}\n\n"
|
||||
"Respond with a JSON object in this exact format:\n"
|
||||
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
|
||||
"Each row_id must match the index provided. Return exactly {row_count} entries."
|
||||
)
|
||||
# [/DEF:DEFAULT_EXECUTION_PROMPT_TEMPLATE:Constant]
|
||||
|
||||
|
||||
# [DEF:DEFAULT_PREVIEW_PROMPT_TEMPLATE:Constant]
|
||||
# @PURPOSE: Default prompt template for LLM translation preview.
|
||||
DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
|
||||
"Translate the following database content from {source_language} to {target_language}.\n\n"
|
||||
"Source dialect: {source_dialect}\n"
|
||||
"Target dialect: {target_dialect}\n"
|
||||
"Column to translate: {translation_column}\n\n"
|
||||
"{dictionary_section}"
|
||||
"For each row, provide an accurate translation of the '{translation_column}' value.\n"
|
||||
"Consider the context columns when determining the meaning of the text.\n\n"
|
||||
"Rows to translate:\n{rows_json}\n\n"
|
||||
"Respond with a JSON object in this exact format:\n"
|
||||
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
|
||||
"Each row_id must match the index provided. Return exactly {row_count} entries."
|
||||
)
|
||||
# [/DEF:DEFAULT_PREVIEW_PROMPT_TEMPLATE:Constant]
|
||||
|
||||
|
||||
# [DEF:TokenEstimator:Class]
|
||||
# @PURPOSE: Estimate token counts and costs for LLM translation operations.
|
||||
# @RATIONALE: Token estimation uses a heuristic (chars/token ratio) since exact tokenization depends on the LLM model.
|
||||
# @REJECTED: Using an external tokenizer library would introduce a heavy dependency for estimation only.
|
||||
class TokenEstimator:
|
||||
"""Estimate token counts and costs for LLM operations."""
|
||||
|
||||
CHARS_PER_TOKEN_ESTIMATE: float = 4.0
|
||||
OUTPUT_TOKENS_PER_ROW_ESTIMATE: int = 50
|
||||
TOKEN_COST_PER_1K: float = 0.002 # Default cost per 1K tokens
|
||||
|
||||
# [DEF:estimate_prompt_tokens:Function]
|
||||
# @PURPOSE: Estimate token count for a prompt string.
|
||||
# @PRE: prompt is a non-empty string.
|
||||
# @POST: Returns estimated token count (integer).
|
||||
@staticmethod
|
||||
def estimate_prompt_tokens(prompt: str) -> int:
|
||||
if not prompt:
|
||||
return 0
|
||||
return max(1, int(len(prompt) / TokenEstimator.CHARS_PER_TOKEN_ESTIMATE))
|
||||
# [/DEF:estimate_prompt_tokens:Function]
|
||||
|
||||
# [DEF:estimate_output_tokens:Function]
|
||||
# @PURPOSE: Estimate output token count for translating N rows.
|
||||
# @PRE: row_count >= 0.
|
||||
# @POST: Returns estimated output token count.
|
||||
@staticmethod
|
||||
def estimate_output_tokens(row_count: int) -> int:
|
||||
return row_count * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE
|
||||
# [/DEF:estimate_output_tokens:Function]
|
||||
|
||||
# [DEF:estimate_cost:Function]
|
||||
# @PURPOSE: Estimate cost for a given number of tokens.
|
||||
# @PRE: total_tokens >= 0.
|
||||
# @POST: Returns estimated cost in USD.
|
||||
@staticmethod
|
||||
def estimate_cost(total_tokens: int, cost_per_1k: Optional[float] = None) -> float:
|
||||
rate = cost_per_1k if cost_per_1k is not None else TokenEstimator.TOKEN_COST_PER_1K
|
||||
return round((total_tokens / 1000) * rate, 6)
|
||||
# [/DEF:estimate_cost:Function]
|
||||
|
||||
|
||||
# [/DEF:TokenEstimator:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationPreview:Class]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate.
|
||||
# @PRE: Database session and config manager are available.
|
||||
# @POST: Preview sessions created with persisted records; full execution gates on accepted session.
|
||||
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows.
|
||||
class TranslationPreview:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db: Session,
|
||||
config_manager: ConfigManager,
|
||||
current_user: Optional[str] = None,
|
||||
):
|
||||
self.db = db
|
||||
self.config_manager = config_manager
|
||||
self.current_user = current_user
|
||||
|
||||
# [DEF:preview_rows:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.
|
||||
# @PRE: job_id exists and job has source_datasource_id, translation_column configured.
|
||||
# @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session.
|
||||
# @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.
|
||||
def preview_rows(
|
||||
self,
|
||||
job_id: str,
|
||||
sample_size: int = 10,
|
||||
prompt_template: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
with belief_scope("TranslationPreview.preview_rows"):
|
||||
logger.reason("Starting preview for job", {"job_id": job_id, "sample_size": sample_size})
|
||||
|
||||
# 1. Load job
|
||||
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
|
||||
if not job:
|
||||
raise ValueError(f"Translation job '{job_id}' not found")
|
||||
if not job.source_datasource_id:
|
||||
raise ValueError("Job must have a source datasource configured for preview")
|
||||
if not job.translation_column:
|
||||
raise ValueError("Job must have a translation column configured for preview")
|
||||
|
||||
# 2. Compute config hash and dict snapshot hash
|
||||
config_hash = self._compute_config_hash(job)
|
||||
dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id)
|
||||
|
||||
# 3. Fetch sample rows from Superset
|
||||
logger.reason("Fetching sample rows from Superset", {
|
||||
"datasource_id": job.source_datasource_id,
|
||||
"sample_size": sample_size,
|
||||
"translation_column": job.translation_column,
|
||||
})
|
||||
source_rows = self._fetch_sample_rows(
|
||||
job=job,
|
||||
sample_size=sample_size,
|
||||
)
|
||||
if not source_rows:
|
||||
raise ValueError("No rows returned from datasource for preview")
|
||||
|
||||
actual_row_count = len(source_rows)
|
||||
logger.reason("Fetched sample rows", {"actual_count": actual_row_count})
|
||||
|
||||
# 4. Build prompt context from rows
|
||||
all_source_texts = []
|
||||
row_meta: List[Dict[str, Any]] = []
|
||||
for idx, row in enumerate(source_rows):
|
||||
translation_value = str(row.get(job.translation_column, "") or "")
|
||||
context_values = {}
|
||||
if job.context_columns:
|
||||
for col in job.context_columns:
|
||||
context_values[col] = str(row.get(col, "") or "")
|
||||
all_source_texts.append(translation_value)
|
||||
row_meta.append({
|
||||
"row_index": idx,
|
||||
"source_text": translation_value,
|
||||
"context_data": context_values,
|
||||
"source_row": row,
|
||||
})
|
||||
|
||||
# 5. Filter dictionary entries for this batch
|
||||
dict_matches = DictionaryManager.filter_for_batch(
|
||||
self.db, all_source_texts, job_id
|
||||
)
|
||||
|
||||
# Build dictionary glossary section
|
||||
dictionary_section = ""
|
||||
if dict_matches:
|
||||
glossary_lines = []
|
||||
for m in dict_matches:
|
||||
glossary_lines.append(
|
||||
f"- '{m['source_term']}' -> '{m['target_term']}'"
|
||||
f"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}"
|
||||
)
|
||||
dictionary_section = (
|
||||
"Terminology dictionary (use these translations when applicable):\n"
|
||||
+ "\n".join(glossary_lines)
|
||||
+ "\n\n"
|
||||
)
|
||||
|
||||
# 6. Build LLM prompt
|
||||
rows_json = json.dumps([
|
||||
{"row_id": str(m["row_index"]), "text": m["source_text"], "context": m["context_data"]}
|
||||
for m in row_meta
|
||||
], indent=2)
|
||||
|
||||
template = prompt_template or DEFAULT_PREVIEW_PROMPT_TEMPLATE
|
||||
prompt = render_prompt(template, {
|
||||
"source_language": job.source_dialect or "SQL",
|
||||
"target_language": job.target_language or job.target_dialect or "en",
|
||||
"source_dialect": job.source_dialect or "",
|
||||
"target_dialect": job.target_dialect or "",
|
||||
"translation_column": job.translation_column or "",
|
||||
"dictionary_section": dictionary_section,
|
||||
"rows_json": rows_json,
|
||||
"row_count": str(actual_row_count),
|
||||
})
|
||||
|
||||
# 7. Estimate tokens/cost for sample and full dataset
|
||||
sample_prompt_tokens = TokenEstimator.estimate_prompt_tokens(prompt)
|
||||
sample_output_tokens = TokenEstimator.estimate_output_tokens(actual_row_count)
|
||||
sample_total_tokens = sample_prompt_tokens + sample_output_tokens
|
||||
sample_cost = TokenEstimator.estimate_cost(sample_total_tokens)
|
||||
|
||||
# Estimate full dataset cost (if we knew total rows)
|
||||
total_est_tokens = TokenEstimator.estimate_prompt_tokens(
|
||||
prompt.replace(str(actual_row_count), "{total}")
|
||||
) + TokenEstimator.estimate_output_tokens(sample_size * 10) # rough extrapolation
|
||||
total_est_cost = TokenEstimator.estimate_cost(total_est_tokens)
|
||||
|
||||
# 8. Call LLM
|
||||
logger.reason("Calling LLM for preview translation", {
|
||||
"provider_id": job.provider_id,
|
||||
"row_count": actual_row_count,
|
||||
"estimated_tokens": sample_total_tokens,
|
||||
})
|
||||
llm_response = self._call_llm(
|
||||
job=job,
|
||||
prompt=prompt,
|
||||
)
|
||||
|
||||
# 9. Parse LLM response
|
||||
translations = self._parse_llm_response(llm_response, actual_row_count)
|
||||
|
||||
# 10. Create preview session
|
||||
session = TranslationPreviewSession(
|
||||
id=str(uuid.uuid4()),
|
||||
job_id=job_id,
|
||||
status="ACTIVE",
|
||||
created_by=self.current_user,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
expires_at=datetime.now(timezone.utc) + timedelta(hours=24),
|
||||
)
|
||||
self.db.add(session)
|
||||
self.db.flush()
|
||||
|
||||
# 11. Create preview records
|
||||
records = []
|
||||
for meta in row_meta:
|
||||
idx = meta["row_index"]
|
||||
translation = translations.get(str(idx), "")
|
||||
is_rejected = False
|
||||
status = "PENDING"
|
||||
feedback = None
|
||||
|
||||
record = TranslationPreviewRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
session_id=session.id,
|
||||
source_sql=meta["source_text"],
|
||||
target_sql=translation,
|
||||
source_object_type="table_row",
|
||||
source_object_id=str(idx),
|
||||
source_object_name=f"Row {idx + 1}",
|
||||
status=status,
|
||||
feedback=feedback,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
self.db.add(record)
|
||||
self.db.flush()
|
||||
records.append({
|
||||
"id": record.id,
|
||||
"source_sql": record.source_sql,
|
||||
"target_sql": record.target_sql,
|
||||
"source_object_type": record.source_object_type,
|
||||
"source_object_id": record.source_object_id,
|
||||
"source_object_name": record.source_object_name,
|
||||
"status": record.status,
|
||||
"feedback": record.feedback,
|
||||
})
|
||||
|
||||
self.db.commit()
|
||||
|
||||
result = {
|
||||
"id": session.id,
|
||||
"job_id": job_id,
|
||||
"status": "ACTIVE",
|
||||
"created_by": self.current_user,
|
||||
"created_at": session.created_at.isoformat(),
|
||||
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
|
||||
"records": records,
|
||||
"cost_estimate": {
|
||||
"sample_size": actual_row_count,
|
||||
"sample_prompt_tokens": sample_prompt_tokens,
|
||||
"sample_output_tokens": sample_output_tokens,
|
||||
"sample_total_tokens": sample_total_tokens,
|
||||
"sample_cost": sample_cost,
|
||||
"estimated_total_rows": actual_row_count * 10,
|
||||
"estimated_tokens": total_est_tokens,
|
||||
"estimated_cost": total_est_cost,
|
||||
},
|
||||
"config_hash": config_hash,
|
||||
"dict_snapshot_hash": dict_snapshot_hash,
|
||||
}
|
||||
|
||||
logger.reflect("Preview completed", {
|
||||
"session_id": session.id,
|
||||
"row_count": actual_row_count,
|
||||
"sample_cost": sample_cost,
|
||||
})
|
||||
return result
|
||||
# [/DEF:preview_rows:Function]
|
||||
|
||||
# [DEF:update_preview_row:Function]
|
||||
# @PURPOSE: Approve, edit, or reject an individual preview row.
|
||||
# @PRE: session_id and row_id exist, session is ACTIVE.
|
||||
# @POST: PreviewRecord status is updated.
|
||||
def update_preview_row(
|
||||
self,
|
||||
job_id: str,
|
||||
row_id: str,
|
||||
action: str,
|
||||
translation: Optional[str] = None,
|
||||
feedback: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
with belief_scope("TranslationPreview.update_preview_row"):
|
||||
# Find the active session for this job
|
||||
session = (
|
||||
self.db.query(TranslationPreviewSession)
|
||||
.filter(
|
||||
TranslationPreviewSession.job_id == job_id,
|
||||
TranslationPreviewSession.status == "ACTIVE",
|
||||
)
|
||||
.order_by(TranslationPreviewSession.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
if not session:
|
||||
raise ValueError(f"No active preview session for job '{job_id}'")
|
||||
|
||||
record = (
|
||||
self.db.query(TranslationPreviewRecord)
|
||||
.filter(
|
||||
TranslationPreviewRecord.id == row_id,
|
||||
TranslationPreviewRecord.session_id == session.id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not record:
|
||||
raise ValueError(f"Preview record '{row_id}' not found in active session")
|
||||
|
||||
if action == "approve":
|
||||
record.status = "APPROVED"
|
||||
elif action == "reject":
|
||||
record.status = "REJECTED"
|
||||
elif action == "edit":
|
||||
record.status = "APPROVED"
|
||||
if translation is not None:
|
||||
record.target_sql = translation
|
||||
else:
|
||||
raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.")
|
||||
|
||||
if feedback is not None:
|
||||
record.feedback = feedback
|
||||
|
||||
self.db.commit()
|
||||
self.db.refresh(record)
|
||||
|
||||
logger.reason(f"Preview row {action}d", {
|
||||
"row_id": row_id,
|
||||
"session_id": session.id,
|
||||
"status": record.status,
|
||||
})
|
||||
|
||||
return {
|
||||
"id": record.id,
|
||||
"source_sql": record.source_sql,
|
||||
"target_sql": record.target_sql,
|
||||
"status": record.status,
|
||||
"feedback": record.feedback,
|
||||
}
|
||||
# [/DEF:update_preview_row:Function]
|
||||
|
||||
# [DEF:accept_preview_session:Function]
|
||||
# @PURPOSE: Mark a preview session as accepted, which gates full execution.
|
||||
# @PRE: job_id has an ACTIVE preview session.
|
||||
# @POST: Session status changes to APPLIED.
|
||||
# @SIDE_EFFECT: Future full execution calls will check for accepted session.
|
||||
def accept_preview_session(self, job_id: str) -> Dict[str, Any]:
|
||||
with belief_scope("TranslationPreview.accept_preview_session"):
|
||||
session = (
|
||||
self.db.query(TranslationPreviewSession)
|
||||
.filter(
|
||||
TranslationPreviewSession.job_id == job_id,
|
||||
TranslationPreviewSession.status == "ACTIVE",
|
||||
)
|
||||
.order_by(TranslationPreviewSession.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
if not session:
|
||||
raise ValueError(f"No active preview session for job '{job_id}'")
|
||||
|
||||
session.status = "APPLIED"
|
||||
self.db.commit()
|
||||
self.db.refresh(session)
|
||||
|
||||
logger.reason("Preview session accepted", {
|
||||
"session_id": session.id,
|
||||
"job_id": job_id,
|
||||
})
|
||||
|
||||
# Get records for response
|
||||
records = (
|
||||
self.db.query(TranslationPreviewRecord)
|
||||
.filter(TranslationPreviewRecord.session_id == session.id)
|
||||
.all()
|
||||
)
|
||||
|
||||
return {
|
||||
"id": session.id,
|
||||
"job_id": job_id,
|
||||
"status": "APPLIED",
|
||||
"created_by": session.created_by,
|
||||
"created_at": session.created_at.isoformat(),
|
||||
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
|
||||
"records": [
|
||||
{
|
||||
"id": r.id,
|
||||
"source_sql": r.source_sql,
|
||||
"target_sql": r.target_sql,
|
||||
"status": r.status,
|
||||
"feedback": r.feedback,
|
||||
}
|
||||
for r in records
|
||||
],
|
||||
}
|
||||
# [/DEF:accept_preview_session:Function]
|
||||
|
||||
# [DEF:get_preview_session:Function]
|
||||
# @PURPOSE: Get the latest preview session for a job with its records.
|
||||
# @PRE: job_id exists.
|
||||
# @POST: Returns session data with records or raises ValueError.
|
||||
def get_preview_session(self, job_id: str) -> Dict[str, Any]:
|
||||
with belief_scope("TranslationPreview.get_preview_session"):
|
||||
session = (
|
||||
self.db.query(TranslationPreviewSession)
|
||||
.filter(TranslationPreviewSession.job_id == job_id)
|
||||
.order_by(TranslationPreviewSession.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
if not session:
|
||||
raise ValueError(f"No preview session found for job '{job_id}'")
|
||||
|
||||
records = (
|
||||
self.db.query(TranslationPreviewRecord)
|
||||
.filter(TranslationPreviewRecord.session_id == session.id)
|
||||
.all()
|
||||
)
|
||||
|
||||
return {
|
||||
"id": session.id,
|
||||
"job_id": job_id,
|
||||
"status": session.status,
|
||||
"created_by": session.created_by,
|
||||
"created_at": session.created_at.isoformat(),
|
||||
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
|
||||
"records": [
|
||||
{
|
||||
"id": r.id,
|
||||
"source_sql": r.source_sql,
|
||||
"target_sql": r.target_sql,
|
||||
"source_object_type": r.source_object_type,
|
||||
"source_object_id": r.source_object_id,
|
||||
"source_object_name": r.source_object_name,
|
||||
"status": r.status,
|
||||
"feedback": r.feedback,
|
||||
}
|
||||
for r in records
|
||||
],
|
||||
}
|
||||
# [/DEF:get_preview_session:Function]
|
||||
|
||||
# [DEF:_fetch_sample_rows:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Fetch sample rows from the Superset dataset for preview.
|
||||
# @PRE: job has source_datasource_id and translation_column.
|
||||
# @POST: Returns list of dicts with row data.
|
||||
# @SIDE_EFFECT: Calls Superset chart data endpoint.
|
||||
def _fetch_sample_rows(self, job: TranslationJob, sample_size: int = 10) -> List[Dict[str, Any]]:
|
||||
with belief_scope("TranslationPreview._fetch_sample_rows"):
|
||||
# Find environment config using source_dialect as env_id
|
||||
environments = self.config_manager.get_environments()
|
||||
env_config = next(
|
||||
(e for e in environments if e.id == job.source_dialect),
|
||||
None,
|
||||
)
|
||||
if not env_config:
|
||||
logger.explore("Could not find environment for datasource", {
|
||||
"env_id": job.source_dialect,
|
||||
})
|
||||
# Fallback: try first environment
|
||||
if environments:
|
||||
env_config = environments[0]
|
||||
logger.explore("Falling back to first available environment", {
|
||||
"env_id": env_config.id,
|
||||
})
|
||||
else:
|
||||
raise ValueError("No Superset environments configured")
|
||||
|
||||
client = SupersetClient(env_config)
|
||||
|
||||
# Fetch dataset detail to build proper query context
|
||||
dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))
|
||||
|
||||
# Determine columns to query
|
||||
query_columns = [job.translation_column]
|
||||
if job.context_columns:
|
||||
query_columns.extend(job.context_columns)
|
||||
|
||||
# Build query context for chart data endpoint
|
||||
query_context = client.build_dataset_preview_query_context(
|
||||
dataset_id=int(job.source_datasource_id),
|
||||
dataset_record=dataset_detail,
|
||||
template_params={},
|
||||
effective_filters=[],
|
||||
)
|
||||
|
||||
# Modify to fetch specific columns as raw data (no aggregation)
|
||||
queries = query_context.get("queries", [])
|
||||
if queries:
|
||||
queries[0]["columns"] = query_columns
|
||||
queries[0]["metrics"] = []
|
||||
queries[0]["row_limit"] = sample_size
|
||||
queries[0]["result_type"] = "query"
|
||||
|
||||
try:
|
||||
response = client.network.request(
|
||||
method="POST",
|
||||
endpoint="/api/v1/chart/data",
|
||||
data=json.dumps(query_context),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
except Exception as e:
|
||||
# Try legacy endpoint as fallback
|
||||
logger.explore("Chart data endpoint failed, trying legacy", {"error": str(e)})
|
||||
try:
|
||||
response = client.network.request(
|
||||
method="POST",
|
||||
endpoint="/explore_json/form_data",
|
||||
params={"form_data": json.dumps(query_context.get("form_data", {}))},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
except Exception as e2:
|
||||
raise ValueError(
|
||||
f"Failed to fetch sample data from Superset: {e2}"
|
||||
)
|
||||
|
||||
# Parse response
|
||||
rows = self._extract_data_rows(response)
|
||||
logger.reason("Extracted data rows", {"count": len(rows)})
|
||||
return rows
|
||||
# [/DEF:_fetch_sample_rows:Function]
|
||||
|
||||
# [DEF:_extract_data_rows:Function]
|
||||
# @PURPOSE: Extract data rows from Superset chart data response.
|
||||
# @PRE: response is a dict from Superset API.
|
||||
# @POST: Returns list of row dicts.
|
||||
@staticmethod
|
||||
def _extract_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
with belief_scope("TranslationPreview._extract_data_rows"):
|
||||
# Try various response formats
|
||||
result = response.get("result")
|
||||
if isinstance(result, list):
|
||||
for item in result:
|
||||
if isinstance(item, dict):
|
||||
data = item.get("data")
|
||||
if isinstance(data, list) and data:
|
||||
return data
|
||||
|
||||
# Try flat result
|
||||
if isinstance(result, dict):
|
||||
data = result.get("data")
|
||||
if isinstance(data, list) and data:
|
||||
return data
|
||||
|
||||
# Legacy: response may have data at top level
|
||||
data = response.get("data")
|
||||
if isinstance(data, list) and data:
|
||||
return data
|
||||
|
||||
# Last resort: return response itself wrapped if it looks like a list of rows
|
||||
if isinstance(result, list):
|
||||
return result
|
||||
|
||||
return []
|
||||
# [/DEF:_extract_data_rows:Function]
|
||||
|
||||
# [DEF:_call_llm:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Call the configured LLM provider with the preview prompt.
|
||||
# @PRE: job has a valid provider_id.
|
||||
# @POST: Returns raw LLM response string.
|
||||
# @SIDE_EFFECT: Makes HTTP call to LLM provider.
|
||||
def _call_llm(self, job: TranslationJob, prompt: str) -> str:
|
||||
with belief_scope("TranslationPreview._call_llm"):
|
||||
if not job.provider_id:
|
||||
raise ValueError("Job has no LLM provider configured")
|
||||
|
||||
provider_svc = LLMProviderService(self.db)
|
||||
provider = provider_svc.get_provider(job.provider_id)
|
||||
if not provider:
|
||||
raise ValueError(f"LLM provider '{job.provider_id}' not found")
|
||||
|
||||
api_key = provider_svc.get_decrypted_api_key(job.provider_id)
|
||||
if not api_key:
|
||||
raise ValueError(f"Could not decrypt API key for provider '{job.provider_id}'")
|
||||
|
||||
# Build the API call based on provider type
|
||||
model = provider.default_model or "gpt-4o-mini"
|
||||
provider_type = provider.provider_type.lower() if provider.provider_type else "openai"
|
||||
|
||||
if provider_type in ("openai", "openai_compatible"):
|
||||
response_text = self._call_openai_compatible(
|
||||
base_url=provider.base_url,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported provider type '{provider_type}' for preview")
|
||||
|
||||
logger.reason("LLM call completed", {
|
||||
"provider_id": job.provider_id,
|
||||
"model": model,
|
||||
"response_length": len(response_text),
|
||||
})
|
||||
return response_text
|
||||
# [/DEF:_call_llm:Function]
|
||||
|
||||
# [DEF:_call_openai_compatible:Function]
|
||||
# @PURPOSE: Call an OpenAI-compatible API for translation.
|
||||
# @PRE: base_url, api_key, model, prompt are valid.
|
||||
# @POST: Returns response text.
|
||||
# @SIDE_EFFECT: Makes HTTP POST to LLM API.
|
||||
@staticmethod
|
||||
def _call_openai_compatible(
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model: str,
|
||||
prompt: str,
|
||||
) -> str:
|
||||
with belief_scope("TranslationPreview._call_openai_compatible"):
|
||||
import requests as http_requests
|
||||
|
||||
url = f"{base_url.rstrip('/')}/chat/completions"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics."},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"temperature": 0.1,
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
|
||||
response = http_requests.post(url, headers=headers, json=payload, timeout=120)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
choices = data.get("choices", [])
|
||||
if not choices:
|
||||
raise ValueError("LLM returned no choices")
|
||||
|
||||
content = choices[0].get("message", {}).get("content", "")
|
||||
if not content:
|
||||
raise ValueError("LLM returned empty content")
|
||||
|
||||
return content
|
||||
# [/DEF:_call_openai_compatible:Function]
|
||||
|
||||
# [DEF:_parse_llm_response:Function]
|
||||
# @PURPOSE: Parse the LLM JSON response into a dict of row_id -> translation.
|
||||
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
|
||||
# @POST: Returns dict mapping string row_id to translation text.
|
||||
@staticmethod
|
||||
def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:
|
||||
with belief_scope("TranslationPreview._parse_llm_response"):
|
||||
try:
|
||||
data = json.loads(response_text)
|
||||
except json.JSONDecodeError:
|
||||
# Try to extract JSON from markdown code block
|
||||
import re
|
||||
match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', response_text, re.DOTALL)
|
||||
if match:
|
||||
try:
|
||||
data = json.loads(match.group(1))
|
||||
except json.JSONDecodeError:
|
||||
raise ValueError("LLM response was not valid JSON")
|
||||
else:
|
||||
raise ValueError("LLM response was not valid JSON")
|
||||
|
||||
rows = data.get("rows", [])
|
||||
if not isinstance(rows, list):
|
||||
raise ValueError("LLM response missing 'rows' array")
|
||||
|
||||
translations: Dict[str, str] = {}
|
||||
for item in rows:
|
||||
row_id = str(item.get("row_id", ""))
|
||||
translation = str(item.get("translation", ""))
|
||||
if row_id:
|
||||
translations[row_id] = translation
|
||||
|
||||
if len(translations) < expected_count:
|
||||
logger.explore("LLM returned fewer translations than expected", {
|
||||
"expected": expected_count,
|
||||
"received": len(translations),
|
||||
"missing": [str(i) for i in range(expected_count) if str(i) not in translations],
|
||||
})
|
||||
|
||||
return translations
|
||||
# [/DEF:_parse_llm_response:Function]
|
||||
|
||||
# [DEF:_compute_config_hash:Function]
|
||||
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
|
||||
@staticmethod
|
||||
def _compute_config_hash(job: TranslationJob) -> str:
|
||||
config_str = json.dumps({
|
||||
"source_dialect": job.source_dialect,
|
||||
"target_dialect": job.target_dialect,
|
||||
"source_datasource_id": job.source_datasource_id,
|
||||
"translation_column": job.translation_column,
|
||||
"context_columns": job.context_columns,
|
||||
"target_language": job.target_language,
|
||||
"provider_id": job.provider_id,
|
||||
"batch_size": job.batch_size,
|
||||
"upsert_strategy": job.upsert_strategy,
|
||||
}, sort_keys=True)
|
||||
return hashlib.sha256(config_str.encode()).hexdigest()[:16]
|
||||
# [/DEF:_compute_config_hash:Function]
|
||||
|
||||
# [DEF:_compute_dict_snapshot_hash:Function]
|
||||
# @PURPOSE: Compute a hash of the dictionary state for snapshot comparison.
|
||||
def _compute_dict_snapshot_hash(self, job_id: str) -> str:
|
||||
dict_links = (
|
||||
self.db.query(TranslationJobDictionary)
|
||||
.filter(TranslationJobDictionary.job_id == job_id)
|
||||
.all()
|
||||
)
|
||||
dict_ids = sorted([dl.dictionary_id for dl in dict_links])
|
||||
hash_input = ",".join(dict_ids)
|
||||
return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
|
||||
# [/DEF:_compute_dict_snapshot_hash:Function]
|
||||
|
||||
|
||||
# [/DEF:TranslationPreview:Class]
|
||||
|
||||
# [/DEF:TranslationPreview:Module]
|
||||
361
backend/src/plugins/translate/scheduler.py
Normal file
361
backend/src/plugins/translate/scheduler.py
Normal file
@@ -0,0 +1,361 @@
|
||||
# [DEF:TranslationScheduler:Module]
|
||||
# @COMPLEXITY: 4
|
||||
# @SEMANTICS: translate, scheduler, apscheduler, cron
|
||||
# @PURPOSE: Manage TranslationSchedule rows and register them with core SchedulerService.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [TranslationSchedule:Class]
|
||||
# @RELATION: DEPENDS_ON -> [SchedulerService:Class]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationOrchestrator:Class]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationEventLog:Class]
|
||||
# @PRE: Database session and SchedulerService are available.
|
||||
# @POST: TranslationSchedule CRUD persisted; APScheduler jobs registered/updated/removed.
|
||||
# @SIDE_EFFECT: Registers APScheduler jobs; runs translations on trigger; creates events.
|
||||
# @RATIONALE: Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance.
|
||||
# @REJECTED: Separate scheduler instance would create resource contention.
|
||||
# @REJECTED: Polling-based approach — event-driven APScheduler is more precise.
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...core.logger import logger, belief_scope
|
||||
from ...core.config_manager import ConfigManager
|
||||
from ...models.translate import TranslationSchedule, TranslationJob, TranslationRun
|
||||
from .events import TranslationEventLog
|
||||
|
||||
|
||||
# [DEF:TranslationScheduler:Class]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: CRUD for TranslationSchedule rows + APScheduler registration wrappers.
|
||||
class TranslationScheduler:
|
||||
|
||||
def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None):
|
||||
self.db = db
|
||||
self.config_manager = config_manager
|
||||
self.current_user = current_user
|
||||
self.event_log = TranslationEventLog(db)
|
||||
|
||||
# [DEF:create_schedule:Function]
|
||||
# @PURPOSE: Create a new schedule for a job.
|
||||
# @PRE: job_id exists. cron_expression is valid.
|
||||
# @POST: TranslationSchedule row created.
|
||||
def create_schedule(
|
||||
self,
|
||||
job_id: str,
|
||||
cron_expression: str,
|
||||
timezone: str = "UTC",
|
||||
is_active: bool = True,
|
||||
) -> TranslationSchedule:
|
||||
with belief_scope("TranslationScheduler.create_schedule"):
|
||||
# Verify job exists
|
||||
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
|
||||
if not job:
|
||||
raise ValueError(f"Translation job '{job_id}' not found")
|
||||
|
||||
schedule = TranslationSchedule(
|
||||
id=str(uuid.uuid4()),
|
||||
job_id=job_id,
|
||||
cron_expression=cron_expression,
|
||||
timezone=timezone,
|
||||
is_active=is_active,
|
||||
created_by=self.current_user,
|
||||
)
|
||||
self.db.add(schedule)
|
||||
self.db.commit()
|
||||
self.db.refresh(schedule)
|
||||
|
||||
self.event_log.log_event(
|
||||
job_id=job_id,
|
||||
event_type="SCHEDULE_CREATED",
|
||||
payload={
|
||||
"schedule_id": schedule.id,
|
||||
"cron_expression": cron_expression,
|
||||
"timezone": timezone,
|
||||
},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
logger.reflect("Schedule created", {"schedule_id": schedule.id, "job_id": job_id})
|
||||
return schedule
|
||||
# [/DEF:create_schedule:Function]
|
||||
|
||||
# [DEF:update_schedule:Function]
|
||||
# @PURPOSE: Update an existing schedule.
|
||||
# @PRE: job_id has an existing schedule.
|
||||
# @POST: Schedule updated.
|
||||
def update_schedule(
|
||||
self,
|
||||
job_id: str,
|
||||
cron_expression: Optional[str] = None,
|
||||
timezone_str: Optional[str] = None,
|
||||
is_active: Optional[bool] = None,
|
||||
) -> TranslationSchedule:
|
||||
with belief_scope("TranslationScheduler.update_schedule"):
|
||||
schedule = self.db.query(TranslationSchedule).filter(
|
||||
TranslationSchedule.job_id == job_id
|
||||
).first()
|
||||
if not schedule:
|
||||
raise ValueError(f"No schedule found for job '{job_id}'")
|
||||
|
||||
if cron_expression is not None:
|
||||
schedule.cron_expression = cron_expression
|
||||
if timezone_str is not None:
|
||||
schedule.timezone = timezone_str
|
||||
if is_active is not None:
|
||||
schedule.is_active = is_active
|
||||
schedule.updated_at = datetime.now(timezone.utc)
|
||||
self.db.commit()
|
||||
self.db.refresh(schedule)
|
||||
|
||||
self.event_log.log_event(
|
||||
job_id=job_id,
|
||||
event_type="SCHEDULE_UPDATED",
|
||||
payload={
|
||||
"schedule_id": schedule.id,
|
||||
"cron_expression": schedule.cron_expression,
|
||||
"timezone": schedule.timezone,
|
||||
"is_active": schedule.is_active,
|
||||
},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
logger.reflect("Schedule updated", {"schedule_id": schedule.id, "job_id": job_id})
|
||||
return schedule
|
||||
# [/DEF:update_schedule:Function]
|
||||
|
||||
# [DEF:delete_schedule:Function]
|
||||
# @PURPOSE: Delete a schedule for a job.
|
||||
# @PRE: job_id has an existing schedule.
|
||||
# @POST: Schedule deleted.
|
||||
def delete_schedule(self, job_id: str) -> None:
|
||||
with belief_scope("TranslationScheduler.delete_schedule"):
|
||||
schedule = self.db.query(TranslationSchedule).filter(
|
||||
TranslationSchedule.job_id == job_id
|
||||
).first()
|
||||
if not schedule:
|
||||
raise ValueError(f"No schedule found for job '{job_id}'")
|
||||
|
||||
schedule_id = schedule.id
|
||||
self.db.delete(schedule)
|
||||
self.db.commit()
|
||||
|
||||
self.event_log.log_event(
|
||||
job_id=job_id,
|
||||
event_type="SCHEDULE_DELETED",
|
||||
payload={"schedule_id": schedule_id},
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
logger.reflect("Schedule deleted", {"schedule_id": schedule_id, "job_id": job_id})
|
||||
# [/DEF:delete_schedule:Function]
|
||||
|
||||
# [DEF:enable_disable_schedule:Function]
|
||||
# @PURPOSE: Enable or disable a schedule.
|
||||
# @PRE: job_id has an existing schedule.
|
||||
# @POST: Schedule is_active updated.
|
||||
def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule:
|
||||
with belief_scope("TranslationScheduler.set_schedule_active"):
|
||||
schedule = self.db.query(TranslationSchedule).filter(
|
||||
TranslationSchedule.job_id == job_id
|
||||
).first()
|
||||
if not schedule:
|
||||
raise ValueError(f"No schedule found for job '{job_id}'")
|
||||
|
||||
schedule.is_active = is_active
|
||||
schedule.updated_at = datetime.now(timezone.utc)
|
||||
self.db.commit()
|
||||
self.db.refresh(schedule)
|
||||
|
||||
logger.reflect("Schedule active state set", {
|
||||
"schedule_id": schedule.id,
|
||||
"job_id": job_id,
|
||||
"is_active": is_active,
|
||||
})
|
||||
return schedule
|
||||
# [/DEF:enable_disable_schedule:Function]
|
||||
|
||||
# [DEF:get_schedule:Function]
|
||||
# @PURPOSE: Get schedule for a job.
|
||||
# @PRE: job_id exists.
|
||||
# @POST: Returns TranslationSchedule or raises ValueError.
|
||||
def get_schedule(self, job_id: str) -> TranslationSchedule:
|
||||
with belief_scope("TranslationScheduler.get_schedule"):
|
||||
schedule = self.db.query(TranslationSchedule).filter(
|
||||
TranslationSchedule.job_id == job_id
|
||||
).first()
|
||||
if not schedule:
|
||||
raise ValueError(f"No schedule found for job '{job_id}'")
|
||||
return schedule
|
||||
# [/DEF:get_schedule:Function]
|
||||
|
||||
# [DEF:list_active_schedules:Function]
|
||||
# @PURPOSE: List all active schedules.
|
||||
# @POST: Returns list of active TranslationSchedule rows.
|
||||
@staticmethod
|
||||
def list_active_schedules(db: Session) -> List[TranslationSchedule]:
|
||||
return (
|
||||
db.query(TranslationSchedule)
|
||||
.filter(TranslationSchedule.is_active == True)
|
||||
.all()
|
||||
)
|
||||
# [/DEF:list_active_schedules:Function]
|
||||
|
||||
# [DEF:get_next_executions:Function]
|
||||
# @PURPOSE: Compute next N execution times from cron expression.
|
||||
# @PRE: cron_expression is valid.
|
||||
# @POST: Returns list of ISO datetime strings.
|
||||
@staticmethod
|
||||
def get_next_executions(cron_expression: str, timezone_str: str = "UTC", n: int = 3) -> List[str]:
|
||||
from zoneinfo import ZoneInfo
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
|
||||
try:
|
||||
tz = ZoneInfo(timezone_str)
|
||||
trigger = CronTrigger.from_crontab(cron_expression, timezone=tz)
|
||||
except (ValueError, KeyError) as e:
|
||||
logger.warning(f"[get_next_executions] Invalid cron: {e}")
|
||||
return []
|
||||
|
||||
now = datetime.now(tz)
|
||||
results = []
|
||||
next_time = now
|
||||
prev = None
|
||||
for _ in range(n):
|
||||
ft = trigger.get_next_fire_time(prev, next_time)
|
||||
if ft is None:
|
||||
break
|
||||
results.append(ft.isoformat())
|
||||
prev = ft
|
||||
next_time = ft
|
||||
return results
|
||||
# [/DEF:get_next_executions:Function]
|
||||
|
||||
|
||||
# [/DEF:TranslationScheduler:Class]
|
||||
|
||||
|
||||
# [DEF:execute_scheduled_translation:Function]
|
||||
# @PURPOSE: APScheduler job handler — runs scheduled translation with concurrency check and new-key-only fallback.
|
||||
# @PRE: schedule_id is valid.
|
||||
# @POST: Translation run created and executed if no concurrent run exists.
|
||||
# @SIDE_EFFECT: DB writes; LLM calls; Superset API calls.
|
||||
# @RATIONALE: New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.
|
||||
def execute_scheduled_translation(
|
||||
schedule_id: str,
|
||||
job_id: str,
|
||||
db_session_maker,
|
||||
config_manager: ConfigManager,
|
||||
) -> None:
|
||||
"""APScheduler job callback for scheduled translations."""
|
||||
db: Session = db_session_maker()
|
||||
try:
|
||||
with belief_scope("execute_scheduled_translation"):
|
||||
# Load schedule
|
||||
schedule = db.query(TranslationSchedule).filter(
|
||||
TranslationSchedule.id == schedule_id,
|
||||
TranslationSchedule.job_id == job_id,
|
||||
).first()
|
||||
if not schedule or not schedule.is_active:
|
||||
logger.info(f"[scheduled_translation] Schedule inactive/missing: {schedule_id}")
|
||||
return
|
||||
|
||||
logger.reason("Scheduled translation triggered", {
|
||||
"schedule_id": schedule_id,
|
||||
"job_id": job_id,
|
||||
})
|
||||
|
||||
# Concurrency check: max 1 pending/running run per job
|
||||
active_run = (
|
||||
db.query(TranslationRun)
|
||||
.filter(
|
||||
TranslationRun.job_id == job_id,
|
||||
TranslationRun.status.in_(["PENDING", "RUNNING"]),
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if active_run:
|
||||
logger.explore("Skipping scheduled run — concurrent run in progress", {
|
||||
"job_id": job_id,
|
||||
"existing_run_id": active_run.id,
|
||||
})
|
||||
event_log = TranslationEventLog(db)
|
||||
event_log.log_event(
|
||||
job_id=job_id,
|
||||
event_type="RUN_STARTED",
|
||||
payload={"reason": "skipped_concurrent", "existing_run_id": active_run.id},
|
||||
)
|
||||
return
|
||||
|
||||
# Check baseline expiry
|
||||
baseline_expired = False
|
||||
most_recent = (
|
||||
db.query(TranslationRun)
|
||||
.filter(
|
||||
TranslationRun.job_id == job_id,
|
||||
TranslationRun.insert_status == "succeeded",
|
||||
)
|
||||
.order_by(TranslationRun.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
if most_recent:
|
||||
age = datetime.now(timezone.utc) - most_recent.created_at
|
||||
if age > timedelta(days=90):
|
||||
baseline_expired = True
|
||||
logger.reason("Baseline expired — full translation", {
|
||||
"job_id": job_id,
|
||||
"last_run": most_recent.created_at.isoformat(),
|
||||
"age_days": age.days,
|
||||
})
|
||||
|
||||
# Import and execute
|
||||
from .orchestrator import TranslationOrchestrator
|
||||
|
||||
orch = TranslationOrchestrator(db, config_manager, current_user="scheduler")
|
||||
run = orch.start_run(
|
||||
job_id=job_id,
|
||||
is_scheduled=True,
|
||||
)
|
||||
run.trigger_type = "scheduled"
|
||||
db.flush()
|
||||
|
||||
if baseline_expired:
|
||||
event_log = TranslationEventLog(db)
|
||||
event_log.log_event(
|
||||
job_id=job_id,
|
||||
run_id=run.id,
|
||||
event_type="RUN_STARTED",
|
||||
payload={"reason": "baseline_expired", "age_days": age.days},
|
||||
)
|
||||
|
||||
# Execute in same thread (APScheduler runs in background thread pool)
|
||||
try:
|
||||
orch.execute_run(run)
|
||||
run.insert_status = run.insert_status or "succeeded"
|
||||
except Exception as exec_err:
|
||||
logger.explore("Scheduled translation execution failed", {
|
||||
"run_id": run.id,
|
||||
"error": str(exec_err),
|
||||
})
|
||||
# Leave schedule enabled — schedule continues on failure
|
||||
run.status = "FAILED"
|
||||
run.error_message = str(exec_err)
|
||||
run.completed_at = datetime.now(timezone.utc)
|
||||
|
||||
# Update schedule tracking
|
||||
schedule.last_run_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
|
||||
logger.reflect("Scheduled translation complete", {
|
||||
"schedule_id": schedule_id,
|
||||
"run_id": run.id,
|
||||
"status": run.status,
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"[scheduled_translation] Unexpected error: {e}")
|
||||
finally:
|
||||
db.close()
|
||||
# [/DEF:execute_scheduled_translation:Function]
|
||||
# [/DEF:TranslationScheduler:Module]
|
||||
541
backend/src/plugins/translate/service.py
Normal file
541
backend/src/plugins/translate/service.py
Normal file
@@ -0,0 +1,541 @@
|
||||
# [DEF:TranslateJobService:Module]
|
||||
# @COMPLEXITY: 4
|
||||
# @SEMANTICS: translate, service, crud, validation
|
||||
# @PURPOSE: Service layer for translation job CRUD with datasource column validation and database dialect detection.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [TranslationJob]
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION: DEPENDS_ON -> [ConfigManager]
|
||||
# @PRE: Database session and config manager are available.
|
||||
# @POST: Translation jobs are created/updated/deleted with column validation and dialect caching.
|
||||
# @SIDE_EFFECT: Queries Superset for column metadata and database dialect at save time.
|
||||
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
|
||||
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity.
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from sqlalchemy.orm import Session
|
||||
from datetime import datetime, timezone
|
||||
import uuid
|
||||
|
||||
from ...core.logger import logger
|
||||
from ...core.config_manager import ConfigManager
|
||||
from ...core.superset_client import SupersetClient
|
||||
from ...models.translate import TranslationJob, TranslationJobDictionary
|
||||
from ...schemas.translate import (
|
||||
TranslateJobCreate,
|
||||
TranslateJobUpdate,
|
||||
TranslateJobResponse,
|
||||
DatasourceColumnsResponse,
|
||||
DatasourceColumnResponse,
|
||||
)
|
||||
|
||||
# Supported database dialects for translation
|
||||
SUPPORTED_DIALECTS = {
|
||||
"postgresql", "mysql", "clickhouse", "sqlite", "mssql",
|
||||
"oracle", "snowflake", "bigquery", "redshift", "presto",
|
||||
"trino", "druid", "hive", "spark", "databricks",
|
||||
}
|
||||
|
||||
|
||||
# [DEF:get_dialect_from_database:Function]
|
||||
# @PURPOSE: Extract normalized dialect string from a Superset database record.
|
||||
# @PRE: database_record is a dict from Superset API.
|
||||
# @POST: Returns normalized dialect string or raises ValueError if unsupported.
|
||||
def get_dialect_from_database(database_record: Dict[str, Any]) -> str:
|
||||
"""Extract and validate dialect from Superset database record."""
|
||||
backend = (
|
||||
database_record.get("backend")
|
||||
or database_record.get("engine")
|
||||
or ""
|
||||
).lower().strip()
|
||||
|
||||
if not backend:
|
||||
raise ValueError("Could not determine database dialect from connection")
|
||||
|
||||
# Map Superset backend names to normalized dialect
|
||||
dialect_map = {
|
||||
"postgresql": "postgresql",
|
||||
"mysql": "mysql",
|
||||
"clickhouse": "clickhouse",
|
||||
"sqlite": "sqlite",
|
||||
"mssql": "mssql",
|
||||
"oracle": "oracle",
|
||||
"snowflake": "snowflake",
|
||||
"bigquery": "bigquery",
|
||||
"redshift": "redshift",
|
||||
"presto": "presto",
|
||||
"trino": "trino",
|
||||
"druid": "druid",
|
||||
"hive": "hive",
|
||||
"spark": "spark",
|
||||
"databricks": "databricks",
|
||||
}
|
||||
|
||||
normalized = dialect_map.get(backend, backend)
|
||||
if normalized not in SUPPORTED_DIALECTS:
|
||||
raise ValueError(
|
||||
f"Unsupported database dialect: '{backend}'. "
|
||||
f"Supported dialects: {', '.join(sorted(SUPPORTED_DIALECTS))}"
|
||||
)
|
||||
return normalized
|
||||
# [/DEF:get_dialect_from_database:Function]
|
||||
|
||||
|
||||
# [DEF:fetch_datasource_metadata:Function]
|
||||
# @PURPOSE: Fetch datasource columns and database dialect from Superset.
|
||||
# @PRE: dataset_id is a valid Superset dataset ID and environment has valid credentials.
|
||||
# @POST: Returns (columns_list, dialect_string) or raises on failure.
|
||||
def fetch_datasource_metadata(
|
||||
dataset_id: int,
|
||||
env_id: str,
|
||||
config_manager: ConfigManager,
|
||||
) -> Tuple[List[Dict[str, Any]], str]:
|
||||
"""Fetch column metadata and database dialect for a datasource from Superset."""
|
||||
# Find environment config
|
||||
environments = config_manager.get_environments()
|
||||
env_config = next((e for e in environments if e.id == env_id), None)
|
||||
if not env_config:
|
||||
raise ValueError(f"Superset environment '{env_id}' not found in configuration")
|
||||
|
||||
# Create Superset client and fetch dataset detail
|
||||
client = SupersetClient(env_config)
|
||||
dataset_detail = client.get_dataset_detail(dataset_id)
|
||||
|
||||
# Extract columns
|
||||
raw_columns = dataset_detail.get("columns", [])
|
||||
columns = []
|
||||
for col in raw_columns:
|
||||
col_name = col.get("name") or col.get("column_name")
|
||||
if not col_name:
|
||||
continue
|
||||
columns.append({
|
||||
"name": str(col_name),
|
||||
"type": col.get("type"),
|
||||
"is_physical": col.get("is_physical", True),
|
||||
"is_dttm": col.get("is_dttm", False),
|
||||
"description": col.get("description", ""),
|
||||
})
|
||||
|
||||
# Extract database dialect from datasource
|
||||
database_info = dataset_detail.get("database", {})
|
||||
if isinstance(database_info, dict):
|
||||
dialect = get_dialect_from_database(database_info)
|
||||
else:
|
||||
# Fallback: try to fetch database directly
|
||||
try:
|
||||
db_id = dataset_detail.get("database_id")
|
||||
if db_id:
|
||||
db_record = client.get_database(int(db_id))
|
||||
db_result = db_record.get("result", db_record) if isinstance(db_record, dict) else db_record
|
||||
dialect = get_dialect_from_database(db_result)
|
||||
else:
|
||||
raise ValueError("No database information available for this datasource")
|
||||
except Exception as e:
|
||||
logger.warning(f"[translate_service] Could not fetch database dialect: {e}")
|
||||
raise ValueError(f"Could not determine database dialect: {e}")
|
||||
|
||||
return columns, dialect
|
||||
# [/DEF:fetch_datasource_metadata:Function]
|
||||
|
||||
|
||||
# [DEF:detect_virtual_columns:Function]
|
||||
# @PURPOSE: Identify virtual (calculated) columns from column metadata.
|
||||
# @PRE: columns is a list of dicts with 'is_physical' key.
|
||||
# @POST: Returns list of virtual column names.
|
||||
def detect_virtual_columns(columns: List[Dict[str, Any]]) -> List[str]:
|
||||
"""Return names of columns that are virtual (not physical)."""
|
||||
return [col["name"] for col in columns if not col.get("is_physical", True)]
|
||||
# [/DEF:detect_virtual_columns:Function]
|
||||
|
||||
|
||||
# [DEF:TranslateJobService:Class]
|
||||
# @PURPOSE: Service for translation job CRUD with validation and Superset integration.
|
||||
class TranslateJobService:
|
||||
|
||||
def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None):
|
||||
self.db = db
|
||||
self.config_manager = config_manager
|
||||
self.current_user = current_user
|
||||
|
||||
# [DEF:list_jobs:Function]
|
||||
# @PURPOSE: List translation jobs with optional status filter and pagination.
|
||||
# @POST: Returns tuple of (total_count, list_of_jobs).
|
||||
def list_jobs(
|
||||
self,
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
status_filter: Optional[str] = None,
|
||||
) -> Tuple[int, List[TranslationJob]]:
|
||||
query = self.db.query(TranslationJob)
|
||||
|
||||
if status_filter:
|
||||
query = query.filter(TranslationJob.status == status_filter)
|
||||
|
||||
total = query.count()
|
||||
offset = (page - 1) * page_size
|
||||
jobs = query.order_by(TranslationJob.created_at.desc()).offset(offset).limit(page_size).all()
|
||||
|
||||
return total, jobs
|
||||
# [/DEF:list_jobs:Function]
|
||||
|
||||
# [DEF:get_job:Function]
|
||||
# @PURPOSE: Get a single translation job by ID.
|
||||
# @POST: Returns TranslationJob or raises ValueError.
|
||||
def get_job(self, job_id: str) -> TranslationJob:
|
||||
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
|
||||
if not job:
|
||||
raise ValueError(f"Translation job '{job_id}' not found")
|
||||
return job
|
||||
# [/DEF:get_job:Function]
|
||||
|
||||
# [DEF:create_job:Function]
|
||||
# @PURPOSE: Create a new translation job with column validation.
|
||||
# @PRE: payload contains valid job configuration.
|
||||
# @POST: Returns the created TranslationJob with database_dialect cached.
|
||||
# @SIDE_EFFECT: Validates columns via SupersetClient if source_datasource_id is provided.
|
||||
# @SIDE_EFFECT: Caches database_dialect from Superset connection.
|
||||
def create_job(self, payload: TranslateJobCreate) -> TranslationJob:
|
||||
logger.info(f"[TranslateJobService] Creating job '{payload.name}'")
|
||||
|
||||
# Validate: must have a translation column if datasource is configured
|
||||
if payload.source_datasource_id and not payload.translation_column:
|
||||
raise ValueError("A translation column is required when a datasource is selected")
|
||||
|
||||
# Validate upsert strategy
|
||||
valid_strategies = {"MERGE", "INSERT", "UPDATE"}
|
||||
if payload.upsert_strategy not in valid_strategies:
|
||||
raise ValueError(
|
||||
f"Invalid upsert_strategy '{payload.upsert_strategy}'. "
|
||||
f"Must be one of: {', '.join(sorted(valid_strategies))}"
|
||||
)
|
||||
|
||||
# Detect database dialect and validate columns if datasource is specified
|
||||
dialect = payload.database_dialect
|
||||
if payload.source_datasource_id and payload.source_dialect:
|
||||
# If no explicit dialect, try to detect it
|
||||
if not dialect:
|
||||
try:
|
||||
env_id = payload.source_dialect
|
||||
_, detected_dialect = fetch_datasource_metadata(
|
||||
int(payload.source_datasource_id),
|
||||
env_id,
|
||||
self.config_manager,
|
||||
)
|
||||
dialect = detected_dialect
|
||||
except Exception as e:
|
||||
logger.warning(f"[TranslateJobService] Dialect detection failed: {e}")
|
||||
dialect = payload.source_dialect
|
||||
|
||||
# Build job instance
|
||||
job = TranslationJob(
|
||||
id=str(uuid.uuid4()),
|
||||
name=payload.name,
|
||||
description=payload.description,
|
||||
source_dialect=payload.source_dialect,
|
||||
target_dialect=payload.target_dialect,
|
||||
database_dialect=dialect,
|
||||
source_datasource_id=payload.source_datasource_id,
|
||||
source_table=payload.source_table,
|
||||
target_schema=payload.target_schema,
|
||||
target_table=payload.target_table,
|
||||
source_key_cols=payload.source_key_cols or [],
|
||||
target_key_cols=payload.target_key_cols or [],
|
||||
translation_column=payload.translation_column,
|
||||
context_columns=payload.context_columns or [],
|
||||
target_language=payload.target_language,
|
||||
provider_id=payload.provider_id,
|
||||
batch_size=payload.batch_size,
|
||||
upsert_strategy=payload.upsert_strategy,
|
||||
status="DRAFT",
|
||||
created_by=self.current_user,
|
||||
)
|
||||
self.db.add(job)
|
||||
self.db.flush()
|
||||
|
||||
# Attach dictionaries
|
||||
if payload.dictionary_ids:
|
||||
for dict_id in payload.dictionary_ids:
|
||||
assoc = TranslationJobDictionary(
|
||||
id=str(uuid.uuid4()),
|
||||
job_id=job.id,
|
||||
dictionary_id=dict_id,
|
||||
)
|
||||
self.db.add(assoc)
|
||||
|
||||
self.db.commit()
|
||||
self.db.refresh(job)
|
||||
logger.info(f"[TranslateJobService] Created job '{job.id}'")
|
||||
return job
|
||||
# [/DEF:create_job:Function]
|
||||
|
||||
# [DEF:update_job:Function]
|
||||
# @PURPOSE: Update an existing translation job.
|
||||
# @PRE: payload contains fields to update.
|
||||
# @POST: Returns the updated TranslationJob.
|
||||
# @SIDE_EFFECT: Re-detects database_dialect if source_datasource_id changed.
|
||||
def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob:
|
||||
logger.info(f"[TranslateJobService] Updating job '{job_id}'")
|
||||
job = self.get_job(job_id)
|
||||
|
||||
update_data = payload.model_dump(exclude_unset=True)
|
||||
dict_ids = update_data.pop("dictionary_ids", None)
|
||||
|
||||
for field, value in update_data.items():
|
||||
if hasattr(job, field):
|
||||
setattr(job, field, value)
|
||||
|
||||
# Re-detect dialect if datasource changed
|
||||
if payload.source_datasource_id and not payload.database_dialect:
|
||||
try:
|
||||
env_id = (payload.source_dialect or job.source_dialect)
|
||||
_, detected_dialect = fetch_datasource_metadata(
|
||||
int(payload.source_datasource_id),
|
||||
env_id,
|
||||
self.config_manager,
|
||||
)
|
||||
job.database_dialect = detected_dialect
|
||||
except Exception as e:
|
||||
logger.warning(f"[TranslateJobService] Dialect re-detection failed: {e}")
|
||||
|
||||
job.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
# Update dictionary associations if provided
|
||||
if dict_ids is not None:
|
||||
self.db.query(TranslationJobDictionary).filter(
|
||||
TranslationJobDictionary.job_id == job_id
|
||||
).delete()
|
||||
for dict_id in dict_ids:
|
||||
assoc = TranslationJobDictionary(
|
||||
id=str(uuid.uuid4()),
|
||||
job_id=job_id,
|
||||
dictionary_id=dict_id,
|
||||
)
|
||||
self.db.add(assoc)
|
||||
|
||||
self.db.commit()
|
||||
self.db.refresh(job)
|
||||
logger.info(f"[TranslateJobService] Updated job '{job_id}'")
|
||||
return job
|
||||
# [/DEF:update_job:Function]
|
||||
|
||||
# [DEF:delete_job:Function]
|
||||
# @PURPOSE: Delete a translation job and its associations.
|
||||
# @PRE: job_id must exist.
|
||||
# @POST: Job and all related records are deleted.
|
||||
def delete_job(self, job_id: str) -> None:
|
||||
logger.info(f"[TranslateJobService] Deleting job '{job_id}'")
|
||||
job = self.get_job(job_id)
|
||||
|
||||
# Delete dictionary associations
|
||||
self.db.query(TranslationJobDictionary).filter(
|
||||
TranslationJobDictionary.job_id == job_id
|
||||
).delete()
|
||||
|
||||
self.db.delete(job)
|
||||
self.db.commit()
|
||||
logger.info(f"[TranslateJobService] Deleted job '{job_id}'")
|
||||
# [/DEF:delete_job:Function]
|
||||
|
||||
# [DEF:duplicate_job:Function]
|
||||
# @PURPOSE: Duplicate a translation job with a new name.
|
||||
# @PRE: job_id must exist.
|
||||
# @POST: Returns the duplicated TranslationJob with status DRAFT.
|
||||
def duplicate_job(self, job_id: str, new_name: Optional[str] = None) -> TranslationJob:
|
||||
logger.info(f"[TranslateJobService] Duplicating job '{job_id}'")
|
||||
source = self.get_job(job_id)
|
||||
|
||||
# Copy all fields except id, created_at, updated_at, status
|
||||
new_job = TranslationJob(
|
||||
id=str(uuid.uuid4()),
|
||||
name=new_name or f"{source.name} (Copy)",
|
||||
description=source.description,
|
||||
source_dialect=source.source_dialect,
|
||||
target_dialect=source.target_dialect,
|
||||
database_dialect=source.database_dialect,
|
||||
source_datasource_id=source.source_datasource_id,
|
||||
source_table=source.source_table,
|
||||
target_schema=source.target_schema,
|
||||
target_table=source.target_table,
|
||||
source_key_cols=source.source_key_cols,
|
||||
target_key_cols=source.target_key_cols,
|
||||
translation_column=source.translation_column,
|
||||
context_columns=source.context_columns,
|
||||
target_language=source.target_language,
|
||||
provider_id=source.provider_id,
|
||||
batch_size=source.batch_size,
|
||||
upsert_strategy=source.upsert_strategy,
|
||||
status="DRAFT",
|
||||
created_by=self.current_user,
|
||||
)
|
||||
self.db.add(new_job)
|
||||
self.db.flush()
|
||||
|
||||
# Copy dictionary associations
|
||||
old_dicts = self.db.query(TranslationJobDictionary).filter(
|
||||
TranslationJobDictionary.job_id == job_id
|
||||
).all()
|
||||
for assoc in old_dicts:
|
||||
new_assoc = TranslationJobDictionary(
|
||||
id=str(uuid.uuid4()),
|
||||
job_id=new_job.id,
|
||||
dictionary_id=assoc.dictionary_id,
|
||||
)
|
||||
self.db.add(new_assoc)
|
||||
|
||||
self.db.commit()
|
||||
self.db.refresh(new_job)
|
||||
logger.info(f"[TranslateJobService] Duplicated job '{job_id}' -> '{new_job.id}'")
|
||||
return new_job
|
||||
# [/DEF:duplicate_job:Function]
|
||||
|
||||
# [DEF:get_job_dictionary_ids:Function]
|
||||
# @PURPOSE: Get dictionary IDs attached to a job.
|
||||
# @POST: Returns list of dictionary IDs.
|
||||
def get_job_dictionary_ids(self, job_id: str) -> List[str]:
|
||||
assocs = self.db.query(TranslationJobDictionary).filter(
|
||||
TranslationJobDictionary.job_id == job_id
|
||||
).all()
|
||||
return [a.dictionary_id for a in assocs]
|
||||
# [/DEF:get_job_dictionary_ids:Function]
|
||||
|
||||
# [DEF:fetch_available_datasources:Function]
|
||||
# @PURPOSE: List available Superset datasets for translation job creation.
|
||||
def fetch_available_datasources(self, env_id: str, search: Optional[str] = None) -> list:
|
||||
"""List Superset datasets available for translation."""
|
||||
from ...core.superset_client import SupersetClient
|
||||
env = self.config_manager.get_environment(env_id)
|
||||
if not env:
|
||||
raise ValueError(f"Environment '{env_id}' not found")
|
||||
client = SupersetClient(env)
|
||||
_, datasets = client.get_datasets()
|
||||
result = []
|
||||
for ds in datasets:
|
||||
name = ds.get("table_name", "")
|
||||
if search and search.lower() not in name.lower():
|
||||
continue
|
||||
db_info = ds.get("database", {})
|
||||
backend = db_info.get("backend", "")
|
||||
dialect = _extract_dialect(backend)
|
||||
result.append({
|
||||
"id": ds.get("id"),
|
||||
"table_name": name,
|
||||
"schema": ds.get("schema"),
|
||||
"database_name": db_info.get("database_name", "Unknown"),
|
||||
"database_dialect": dialect,
|
||||
"description": ds.get("description", ""),
|
||||
})
|
||||
return result
|
||||
# [/DEF:fetch_available_datasources:Function]
|
||||
# [/DEF:TranslateJobService:Class]
|
||||
|
||||
|
||||
# [DEF:_extract_dialect:Function]
|
||||
# @PURPOSE: Extract database dialect from Superset backend URI.
|
||||
def _extract_dialect(backend: str) -> str:
|
||||
"""Extract dialect name from a Superset database backend URI (e.g. 'postgresql+psycopg2://...')."""
|
||||
if not backend:
|
||||
return "unknown"
|
||||
try:
|
||||
scheme = backend.split("://")[0]
|
||||
dialect = scheme.split("+")[0]
|
||||
return dialect.lower()
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
|
||||
# [DEF:job_to_response:Function]
|
||||
# @PURPOSE: Convert a TranslationJob ORM model to a TranslateJobResponse schema with dictionary_ids.
|
||||
def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -> TranslateJobResponse:
|
||||
return TranslateJobResponse(
|
||||
id=job.id,
|
||||
name=job.name,
|
||||
description=job.description,
|
||||
source_dialect=job.source_dialect,
|
||||
target_dialect=job.target_dialect,
|
||||
database_dialect=job.database_dialect,
|
||||
source_datasource_id=job.source_datasource_id,
|
||||
source_table=job.source_table,
|
||||
target_schema=job.target_schema,
|
||||
target_table=job.target_table,
|
||||
source_key_cols=job.source_key_cols or [],
|
||||
target_key_cols=job.target_key_cols or [],
|
||||
translation_column=job.translation_column,
|
||||
context_columns=job.context_columns or [],
|
||||
target_language=job.target_language,
|
||||
provider_id=job.provider_id,
|
||||
batch_size=job.batch_size or 50,
|
||||
upsert_strategy=job.upsert_strategy or "MERGE",
|
||||
status=job.status,
|
||||
created_by=job.created_by,
|
||||
created_at=job.created_at,
|
||||
updated_at=job.updated_at,
|
||||
dictionary_ids=dict_ids or [],
|
||||
)
|
||||
# [/DEF:job_to_response:Function]
|
||||
|
||||
|
||||
# [DEF:DatasourceColumnsService:Function]
|
||||
# @PURPOSE: Fetch datasource column metadata from Superset and return structured response.
|
||||
# @PRE: datasource_id is a valid Superset dataset ID.
|
||||
# @POST: Returns DatasourceColumnsResponse with column metadata and database dialect.
|
||||
# @SIDE_EFFECT: Queries Superset API for dataset detail and database info.
|
||||
def get_datasource_columns(
|
||||
datasource_id: int,
|
||||
env_id: str,
|
||||
config_manager: ConfigManager,
|
||||
) -> DatasourceColumnsResponse:
|
||||
"""Fetch and return column metadata for a given Superset datasource."""
|
||||
logger.info(f"[get_datasource_columns] Fetching columns for datasource {datasource_id}")
|
||||
|
||||
# Find environment config
|
||||
environments = config_manager.get_environments()
|
||||
env_config = next((e for e in environments if e.id == env_id), None)
|
||||
if not env_config:
|
||||
raise ValueError(f"Superset environment '{env_id}' not found")
|
||||
|
||||
# Create Superset client
|
||||
client = SupersetClient(env_config)
|
||||
dataset_detail = client.get_dataset_detail(datasource_id)
|
||||
|
||||
# Extract database dialect
|
||||
database_info = dataset_detail.get("database", {})
|
||||
dialect = None
|
||||
if isinstance(database_info, dict):
|
||||
try:
|
||||
dialect = get_dialect_from_database(database_info)
|
||||
except (ValueError, KeyError):
|
||||
dialect = None
|
||||
if dialect is None:
|
||||
database_id = dataset_detail.get("database_id")
|
||||
if database_id:
|
||||
db_record = client.get_database(int(database_id))
|
||||
db_result = db_record.get("result", db_record) if isinstance(db_record, dict) else db_record
|
||||
dialect = get_dialect_from_database(db_result)
|
||||
else:
|
||||
raise ValueError("Could not determine database dialect for this datasource")
|
||||
|
||||
# Extract columns
|
||||
raw_columns = dataset_detail.get("columns", [])
|
||||
columns = []
|
||||
for col in raw_columns:
|
||||
col_name = col.get("name") or col.get("column_name")
|
||||
if not col_name:
|
||||
continue
|
||||
columns.append(DatasourceColumnResponse(
|
||||
name=str(col_name),
|
||||
type=col.get("type"),
|
||||
is_physical=col.get("is_physical", True),
|
||||
is_dttm=col.get("is_dttm", False),
|
||||
description=col.get("description", ""),
|
||||
))
|
||||
|
||||
return DatasourceColumnsResponse(
|
||||
datasource_id=datasource_id,
|
||||
datasource_name=dataset_detail.get("table_name"),
|
||||
schema_name=dataset_detail.get("schema"),
|
||||
database_dialect=dialect,
|
||||
columns=columns,
|
||||
)
|
||||
# [/DEF:DatasourceColumnsService:Function]
|
||||
|
||||
# [/DEF:TranslateJobService:Module]
|
||||
319
backend/src/plugins/translate/sql_generator.py
Normal file
319
backend/src/plugins/translate/sql_generator.py
Normal file
@@ -0,0 +1,319 @@
|
||||
# [DEF:SQLGenerator:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @SEMANTICS: translate, sql, generator, dialect
|
||||
# @PURPOSE: Dialect-aware safe SQL generation for INSERT/UPSERT operations.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [TranslationJob]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationRun]
|
||||
# @PRE: Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS.
|
||||
# @POST: Returns safe SQL strings for the target dialect.
|
||||
# @SIDE_EFFECT: None — pure code generation.
|
||||
# @RATIONALE: Dialect-aware SQL uses ON CONFLICT for PostgreSQL; plain INSERT for ClickHouse with documented limitations.
|
||||
# @REJECTED: UPDATE statements — source is append-only; UPSERT covers overwrite case.
|
||||
# @REJECTED: ORM-based insert bypasses Superset's SQL Lab audit trail.
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from ...core.logger import logger, belief_scope
|
||||
|
||||
# PostgreSQL/Greenplum dialects that support ON CONFLICT
|
||||
POSTGRESQL_DIALECTS = {"postgresql", "redshift", "greenplum"}
|
||||
# Dialects that use backtick or no quoting
|
||||
CLICKHOUSE_DIALECTS = {"clickhouse"}
|
||||
|
||||
|
||||
# [DEF:_quote_identifier:Function]
|
||||
# @PURPOSE: Quote an identifier per dialect rules. PostgreSQL uses double quotes; ClickHouse uses backticks.
|
||||
# @PRE: identifier is a non-empty string.
|
||||
# @POST: Returns safely quoted identifier.
|
||||
def _quote_identifier(identifier: str, dialect: str) -> str:
|
||||
"""Quote a SQL identifier per dialect rules."""
|
||||
if not identifier:
|
||||
return identifier
|
||||
# Remove any existing quotes to avoid double-quoting
|
||||
cleaned = identifier.strip().strip('"').strip('`').strip('[]')
|
||||
if dialect in POSTGRESQL_DIALECTS:
|
||||
return f'"{cleaned}"'
|
||||
elif dialect in CLICKHOUSE_DIALECTS:
|
||||
return f"`{cleaned}`"
|
||||
else:
|
||||
# Generic ANSI double-quote
|
||||
return f'"{cleaned}"'
|
||||
# [/DEF:_quote_identifier:Function]
|
||||
|
||||
|
||||
# [DEF:_encode_sql_value:Function]
|
||||
# @PURPOSE: Encode a Python value into a SQL-safe literal for INSERT VALUES.
|
||||
# @PRE: value is a Python primitive or None.
|
||||
# @POST: Returns SQL-safe string literal representation.
|
||||
def _encode_sql_value(value: Any) -> str:
|
||||
"""Encode a Python value into a SQL-safe literal."""
|
||||
if value is None:
|
||||
return "NULL"
|
||||
if isinstance(value, bool):
|
||||
return "TRUE" if value else "FALSE"
|
||||
if isinstance(value, (int, float)):
|
||||
return str(value)
|
||||
# String — escape single quotes by doubling them
|
||||
escaped = str(value).replace("'", "''")
|
||||
return f"'{escaped}'"
|
||||
# [/DEF:_encode_sql_value:Function]
|
||||
|
||||
|
||||
# [DEF:_build_values_clause:Function]
|
||||
# @PURPOSE: Build a VALUES clause for multiple rows.
|
||||
# @PRE: columns list is non-empty; rows is a list of dicts.
|
||||
# @POST: Returns SQL VALUES clause string.
|
||||
def _build_values_clause(columns: List[str], rows: List[Dict[str, Any]]) -> str:
|
||||
"""Build VALUES (...) clause for multiple rows."""
|
||||
value_groups = []
|
||||
for row in rows:
|
||||
values = [_encode_sql_value(row.get(col)) for col in columns]
|
||||
value_groups.append(f"({', '.join(values)})")
|
||||
return ",\n".join(value_groups)
|
||||
# [/DEF:_build_values_clause:Function]
|
||||
|
||||
|
||||
# [DEF:generate_insert_sql:Function]
|
||||
# @PURPOSE: Generate a dialect-aware INSERT SQL statement.
|
||||
# @PRE: target_table, columns are non-empty.
|
||||
# @POST: Returns SQL string or raises ValueError on invalid input.
|
||||
def generate_insert_sql(
|
||||
target_schema: Optional[str],
|
||||
target_table: str,
|
||||
columns: List[str],
|
||||
rows: List[Dict[str, Any]],
|
||||
) -> str:
|
||||
"""Generate a plain INSERT SQL."""
|
||||
with belief_scope("generate_insert_sql"):
|
||||
if not target_table:
|
||||
raise ValueError("target_table is required for INSERT SQL generation")
|
||||
if not columns:
|
||||
raise ValueError("At least one column is required for INSERT SQL generation")
|
||||
if not rows:
|
||||
raise ValueError("At least one row is required for INSERT SQL generation")
|
||||
|
||||
col_list = ", ".join(columns)
|
||||
values = _build_values_clause(columns, rows)
|
||||
|
||||
table_ref = target_table
|
||||
if target_schema:
|
||||
table_ref = f"{target_schema}.{target_table}"
|
||||
|
||||
sql = f"INSERT INTO {table_ref} ({col_list})\nVALUES\n{values};"
|
||||
return sql
|
||||
# [/DEF:generate_insert_sql:Function]
|
||||
|
||||
|
||||
# [DEF:generate_upsert_sql:Function]
|
||||
# @PURPOSE: Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE.
|
||||
# @PRE: dialect is postgresql-compatible. target_table, columns, key_columns are non-empty.
|
||||
# @POST: Returns UPSERT SQL string or raises ValueError.
|
||||
def generate_upsert_sql(
|
||||
target_schema: Optional[str],
|
||||
target_table: str,
|
||||
columns: List[str],
|
||||
key_columns: List[str],
|
||||
rows: List[Dict[str, Any]],
|
||||
) -> str:
|
||||
"""Generate INSERT ... ON CONFLICT (key_cols) DO UPDATE SET ... SQL."""
|
||||
with belief_scope("generate_upsert_sql"):
|
||||
if not target_table:
|
||||
raise ValueError("target_table is required for UPSERT SQL generation")
|
||||
if not columns:
|
||||
raise ValueError("At least one column is required for UPSERT SQL generation")
|
||||
if not key_columns:
|
||||
raise ValueError("key_columns are required for UPSERT SQL generation")
|
||||
if not rows:
|
||||
raise ValueError("At least one row is required for UPSERT SQL generation")
|
||||
|
||||
col_list = ", ".join(columns)
|
||||
key_list = ", ".join(key_columns)
|
||||
values = _build_values_clause(columns, rows)
|
||||
|
||||
table_ref = target_table
|
||||
if target_schema:
|
||||
table_ref = f"{target_schema}.{target_table}"
|
||||
|
||||
# Build SET clause: exclude key columns from update
|
||||
update_cols = [c for c in columns if c not in key_columns]
|
||||
if not update_cols:
|
||||
# If only key columns, use DO NOTHING
|
||||
conflict_action = "DO NOTHING"
|
||||
else:
|
||||
set_parts = [f"{col} = EXCLUDED.{col}" for col in update_cols]
|
||||
conflict_action = "DO UPDATE SET\n" + ",\n".join(set_parts)
|
||||
|
||||
sql = (
|
||||
f"INSERT INTO {table_ref} ({col_list})\n"
|
||||
f"VALUES\n"
|
||||
f"{values}\n"
|
||||
f"ON CONFLICT ({key_list}) {conflict_action};"
|
||||
)
|
||||
return sql
|
||||
# [/DEF:generate_upsert_sql:Function]
|
||||
|
||||
|
||||
# [DEF:SQLGenerator:Class]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Generate safe, dialect-appropriate SQL INSERT/UPSERT statements.
|
||||
# @PRE: Job has target_schema, target_table, key columns configured.
|
||||
# @POST: Returns generated SQL string for the target dialect.
|
||||
class SQLGenerator:
|
||||
|
||||
# [DEF:SQLGenerator.generate:Function]
|
||||
# @PURPOSE: Generate SQL for a set of rows, detecting dialect from the job configuration.
|
||||
# @PRE: dialect is a supported database dialect. columns list is non-empty. rows is non-empty.
|
||||
# @POST: Returns tuple of (sql_string, statement_count).
|
||||
# @SIDE_EFFECT: None — pure SQL generation.
|
||||
@staticmethod
|
||||
def generate(
|
||||
dialect: str,
|
||||
target_schema: Optional[str],
|
||||
target_table: str,
|
||||
columns: List[str],
|
||||
rows: List[Dict[str, Any]],
|
||||
key_columns: Optional[List[str]] = None,
|
||||
upsert_strategy: str = "MERGE",
|
||||
) -> Tuple[str, int]:
|
||||
"""Generate dialect-appropriate INSERT/UPSERT SQL.
|
||||
|
||||
Args:
|
||||
dialect: Target database dialect (e.g. 'postgresql', 'clickhouse').
|
||||
target_schema: Optional schema name.
|
||||
target_table: Target table name.
|
||||
columns: List of column names to insert.
|
||||
rows: List of row dicts with column values.
|
||||
key_columns: Key columns for conflict resolution (UPSERT).
|
||||
upsert_strategy: 'MERGE' (UPSERT), 'INSERT' (plain INSERT).
|
||||
|
||||
Returns:
|
||||
Tuple of (sql_string, row_count).
|
||||
"""
|
||||
with belief_scope("SQLGenerator.generate"):
|
||||
logger.reason("Generating SQL", {
|
||||
"dialect": dialect,
|
||||
"schema": target_schema,
|
||||
"table": target_table,
|
||||
"columns": len(columns),
|
||||
"rows": len(rows),
|
||||
"strategy": upsert_strategy,
|
||||
})
|
||||
|
||||
# Validate inputs
|
||||
if not target_table:
|
||||
raise ValueError("target_table is required")
|
||||
if not columns:
|
||||
raise ValueError("At least one column is required")
|
||||
if not rows:
|
||||
raise ValueError("At least one row is required")
|
||||
|
||||
# Build fully qualified table reference
|
||||
table_ref = target_table
|
||||
if target_schema:
|
||||
quoted_schema = _quote_identifier(target_schema, dialect)
|
||||
quoted_table = _quote_identifier(target_table, dialect)
|
||||
table_ref = f"{quoted_schema}.{quoted_table}"
|
||||
else:
|
||||
table_ref = _quote_identifier(target_table, dialect)
|
||||
|
||||
# Quote columns per dialect
|
||||
quoted_columns = [_quote_identifier(c, dialect) for c in columns]
|
||||
quoted_key_columns = (
|
||||
[_quote_identifier(k, dialect) for k in key_columns]
|
||||
if key_columns
|
||||
else []
|
||||
)
|
||||
|
||||
# Generate SQL per dialect and strategy
|
||||
use_upsert = upsert_strategy.upper() == "MERGE" and key_columns
|
||||
|
||||
if dialect in POSTGRESQL_DIALECTS or dialect not in CLICKHOUSE_DIALECTS:
|
||||
# PostgreSQL and other ANSI dialects: support UPSERT via ON CONFLICT
|
||||
if use_upsert:
|
||||
sql = generate_upsert_sql(
|
||||
target_schema=None,
|
||||
target_table=table_ref,
|
||||
columns=quoted_columns,
|
||||
key_columns=quoted_key_columns,
|
||||
rows=rows,
|
||||
)
|
||||
else:
|
||||
sql = generate_insert_sql(
|
||||
target_schema=None,
|
||||
target_table=table_ref,
|
||||
columns=quoted_columns,
|
||||
rows=rows,
|
||||
)
|
||||
elif dialect in CLICKHOUSE_DIALECTS:
|
||||
# ClickHouse: plain INSERT, no ON CONFLICT support
|
||||
sql = generate_insert_sql(
|
||||
target_schema=None,
|
||||
target_table=table_ref,
|
||||
columns=quoted_columns,
|
||||
rows=rows,
|
||||
)
|
||||
if use_upsert:
|
||||
logger.reason("ClickHouse UPSERT not supported; using plain INSERT", {
|
||||
"note": "ClickHouse does not support ON CONFLICT. Use ReplacingMergeTree for dedup.",
|
||||
})
|
||||
else:
|
||||
# Fallback: plain INSERT
|
||||
sql = generate_insert_sql(
|
||||
target_schema=None,
|
||||
target_table=table_ref,
|
||||
columns=quoted_columns,
|
||||
rows=rows,
|
||||
)
|
||||
|
||||
logger.reflect("SQL generated", {
|
||||
"dialect": dialect,
|
||||
"row_count": len(rows),
|
||||
"sql_length": len(sql),
|
||||
})
|
||||
return sql, len(rows)
|
||||
# [/DEF:SQLGenerator.generate:Function]
|
||||
|
||||
# [DEF:SQLGenerator.generate_batch:Function]
|
||||
# @PURPOSE: Generate separate INSERT statements for each row (batch-safe version).
|
||||
# @PRE: Same as generate().
|
||||
# @POST: Returns list of (sql_string, row_index) tuples.
|
||||
@staticmethod
|
||||
def generate_batch(
|
||||
dialect: str,
|
||||
target_schema: Optional[str],
|
||||
target_table: str,
|
||||
columns: List[str],
|
||||
rows: List[Dict[str, Any]],
|
||||
key_columns: Optional[List[str]] = None,
|
||||
upsert_strategy: str = "MERGE",
|
||||
max_rows_per_statement: int = 500,
|
||||
) -> List[Tuple[str, int]]:
|
||||
"""Generate SQL in batches, splitting large row sets into multiple statements.
|
||||
|
||||
Returns:
|
||||
List of (sql_string, row_count) tuples.
|
||||
"""
|
||||
with belief_scope("SQLGenerator.generate_batch"):
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
statements = []
|
||||
for i in range(0, len(rows), max_rows_per_statement):
|
||||
chunk = rows[i:i + max_rows_per_statement]
|
||||
sql, count = SQLGenerator.generate(
|
||||
dialect=dialect,
|
||||
target_schema=target_schema,
|
||||
target_table=target_table,
|
||||
columns=columns,
|
||||
rows=chunk,
|
||||
key_columns=key_columns,
|
||||
upsert_strategy=upsert_strategy,
|
||||
)
|
||||
statements.append((sql, count))
|
||||
|
||||
return statements
|
||||
# [/DEF:SQLGenerator.generate_batch:Function]
|
||||
|
||||
|
||||
# [/DEF:SQLGenerator:Class]
|
||||
# [/DEF:SQLGenerator:Module]
|
||||
316
backend/src/plugins/translate/superset_executor.py
Normal file
316
backend/src/plugins/translate/superset_executor.py
Normal file
@@ -0,0 +1,316 @@
|
||||
# [DEF:SupersetSqlLabExecutor:Module]
|
||||
# @COMPLEXITY: 4
|
||||
# @SEMANTICS: translate, superset, sqllab, execute
|
||||
# @PURPOSE: Submit SQL to Superset SQL Lab API and poll execution status.
|
||||
# @LAYER: Infra
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION: DEPENDS_ON -> [ConfigManager]
|
||||
# @PRE: Valid Superset environment configuration and authenticated client.
|
||||
# @POST: SQL is submitted to Superset SQL Lab; execution reference is returned.
|
||||
# @SIDE_EFFECT: Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status.
|
||||
# @RATIONALE: Direct SQL Lab API submission provides audit trail and execution monitoring within Superset.
|
||||
# @REJECTED: Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC.
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from ...core.logger import logger, belief_scope
|
||||
from ...core.config_manager import ConfigManager
|
||||
from ...core.superset_client import SupersetClient
|
||||
|
||||
|
||||
# [DEF:SupersetSqlLabExecutor:Class]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Submit SQL to Superset SQL Lab API with polling and status tracking.
|
||||
# @PRE: Valid environment ID and ConfigManager.
|
||||
# @POST: SQL submitted to Superset; execution reference recorded.
|
||||
# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status.
|
||||
class SupersetSqlLabExecutor:
|
||||
|
||||
def __init__(self, config_manager: ConfigManager, env_id: str):
|
||||
self.config_manager = config_manager
|
||||
self.env_id = env_id
|
||||
self._client: Optional[SupersetClient] = None
|
||||
self._database_id: Optional[int] = None
|
||||
|
||||
# [DEF:_get_client:Function]
|
||||
# @PURPOSE: Lazy-initialize SupersetClient for the configured environment.
|
||||
# @PRE: env_id must correspond to a valid environment config.
|
||||
# @POST: Returns authenticated SupersetClient.
|
||||
# @SIDE_EFFECT: Authenticates against Superset API.
|
||||
def _get_client(self) -> SupersetClient:
|
||||
with belief_scope("SupersetSqlLabExecutor._get_client"):
|
||||
if self._client is not None:
|
||||
return self._client
|
||||
|
||||
environments = self.config_manager.get_environments()
|
||||
env_config = next((e for e in environments if e.id == self.env_id), None)
|
||||
if not env_config:
|
||||
raise ValueError(f"Superset environment '{self.env_id}' not found")
|
||||
|
||||
self._client = SupersetClient(env_config)
|
||||
return self._client
|
||||
# [/DEF:_get_client:Function]
|
||||
|
||||
# [DEF:resolve_database_id:Function]
|
||||
# @PURPOSE: Resolve the target database ID from the environment.
|
||||
# @PRE: database_name or database_id should be known.
|
||||
# @POST: Returns database_id integer or raises ValueError.
|
||||
# @SIDE_EFFECT: Fetches databases list from Superset.
|
||||
def resolve_database_id(self, database_name: Optional[str] = None) -> int:
|
||||
with belief_scope("SupersetSqlLabExecutor.resolve_database_id"):
|
||||
client = self._get_client()
|
||||
_, databases = client.get_databases(
|
||||
query={"columns": ["id", "database_name"]}
|
||||
)
|
||||
if not databases:
|
||||
raise ValueError("No databases found in Superset environment")
|
||||
|
||||
if database_name:
|
||||
for db in databases:
|
||||
if db.get("database_name", "").lower() == database_name.lower():
|
||||
self._database_id = db["id"]
|
||||
logger.reason("Resolved database ID by name", {
|
||||
"database_name": database_name,
|
||||
"database_id": self._database_id,
|
||||
})
|
||||
return self._database_id
|
||||
raise ValueError(
|
||||
f"Database '{database_name}' not found in Superset environment"
|
||||
)
|
||||
|
||||
# Default: use first database
|
||||
self._database_id = databases[0]["id"]
|
||||
logger.reason("Using default database", {
|
||||
"database_name": databases[0].get("database_name"),
|
||||
"database_id": self._database_id,
|
||||
})
|
||||
return self._database_id
|
||||
# [/DEF:resolve_database_id:Function]
|
||||
|
||||
# [DEF:execute_sql:Function]
|
||||
# @PURPOSE: Submit SQL to Superset SQL Lab and return execution tracking info.
|
||||
# @PRE: sql is valid SQL string. database_id is a valid Superset DB ID.
|
||||
# @POST: Returns execution result dict with query_id and status.
|
||||
# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/.
|
||||
def execute_sql(
|
||||
self,
|
||||
sql: str,
|
||||
database_id: Optional[int] = None,
|
||||
run_async: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
with belief_scope("SupersetSqlLabExecutor.execute_sql"):
|
||||
client = self._get_client()
|
||||
db_id = database_id or self._database_id
|
||||
if not db_id:
|
||||
db_id = self.resolve_database_id()
|
||||
|
||||
logger.reason("Submitting SQL to Superset SQL Lab", {
|
||||
"database_id": db_id,
|
||||
"sql_length": len(sql),
|
||||
"run_async": run_async,
|
||||
})
|
||||
|
||||
payload = {
|
||||
"database_id": db_id,
|
||||
"sql": sql,
|
||||
"runAsync": run_async,
|
||||
"schema": None,
|
||||
"tab": "translation-insert",
|
||||
}
|
||||
|
||||
try:
|
||||
response = client.network.request(
|
||||
method="POST",
|
||||
endpoint="/api/v1/sqllab/execute/",
|
||||
data=json.dumps(payload),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.explore("SQL Lab execute failed", {"error": str(e)})
|
||||
raise ValueError(f"Superset SQL Lab execute failed: {e}")
|
||||
|
||||
# Parse response
|
||||
result = response if isinstance(response, dict) else {}
|
||||
query_id = result.get("query_id") or result.get("id")
|
||||
status = result.get("status", "unknown")
|
||||
|
||||
logger.reason("SQL Lab execute response", {
|
||||
"query_id": query_id,
|
||||
"status": status,
|
||||
})
|
||||
|
||||
return {
|
||||
"query_id": query_id,
|
||||
"status": status,
|
||||
"raw_response": result,
|
||||
"database_id": db_id,
|
||||
}
|
||||
# [/DEF:execute_sql:Function]
|
||||
|
||||
# [DEF:poll_execution_status:Function]
|
||||
# @PURPOSE: Poll Superset for SQL execution status until completion or timeout.
|
||||
# @PRE: query_id is a valid Superset query ID.
|
||||
# @POST: Returns final execution status dict.
|
||||
# @SIDE_EFFECT: Makes HTTP GET requests to Superset.
|
||||
def poll_execution_status(
|
||||
self,
|
||||
query_id: str,
|
||||
max_polls: int = 60,
|
||||
poll_interval_seconds: float = 2.0,
|
||||
) -> Dict[str, Any]:
|
||||
with belief_scope("SupersetSqlLabExecutor.poll_execution_status"):
|
||||
client = self._get_client()
|
||||
|
||||
logger.reason("Polling SQL execution status", {
|
||||
"query_id": query_id,
|
||||
"max_polls": max_polls,
|
||||
"interval": poll_interval_seconds,
|
||||
})
|
||||
|
||||
for attempt in range(max_polls):
|
||||
try:
|
||||
response = client.network.request(
|
||||
method="GET",
|
||||
endpoint=f"/api/v1/query/{query_id}",
|
||||
)
|
||||
result = response if isinstance(response, dict) else {}
|
||||
status = result.get("status", "unknown")
|
||||
state = result.get("state", result.get("status", ""))
|
||||
|
||||
# Terminal states
|
||||
if state in ("success", "finished", "completed"):
|
||||
logger.reason("SQL execution completed", {
|
||||
"query_id": query_id,
|
||||
"attempt": attempt + 1,
|
||||
"rows_affected": result.get("rows"),
|
||||
})
|
||||
return {
|
||||
"query_id": query_id,
|
||||
"status": "success",
|
||||
"state": state,
|
||||
"rows_affected": result.get("rows"),
|
||||
"error_message": result.get("error_message"),
|
||||
"results": result.get("results"),
|
||||
"completed_on": result.get("completed_on"),
|
||||
}
|
||||
|
||||
if state in ("failed", "error", "stopped"):
|
||||
error_msg = result.get("error_message", "Unknown error")
|
||||
logger.explore("SQL execution failed", {
|
||||
"query_id": query_id,
|
||||
"state": state,
|
||||
"error": error_msg,
|
||||
})
|
||||
return {
|
||||
"query_id": query_id,
|
||||
"status": "failed",
|
||||
"state": state,
|
||||
"error_message": error_msg,
|
||||
"results": None,
|
||||
}
|
||||
|
||||
if state in ("pending", "running", "started"):
|
||||
time.sleep(poll_interval_seconds)
|
||||
continue
|
||||
|
||||
# Unknown state — treat as still running
|
||||
time.sleep(poll_interval_seconds)
|
||||
|
||||
except Exception as e:
|
||||
logger.explore("Polling error, retrying", {
|
||||
"query_id": query_id,
|
||||
"attempt": attempt + 1,
|
||||
"error": str(e),
|
||||
})
|
||||
time.sleep(poll_interval_seconds)
|
||||
continue
|
||||
|
||||
# Timeout
|
||||
logger.explore("SQL execution polling timed out", {
|
||||
"query_id": query_id,
|
||||
"max_polls": max_polls,
|
||||
})
|
||||
return {
|
||||
"query_id": query_id,
|
||||
"status": "timeout",
|
||||
"error_message": f"Polling timed out after {max_polls} attempts",
|
||||
"results": None,
|
||||
}
|
||||
# [/DEF:poll_execution_status:Function]
|
||||
|
||||
# [DEF:execute_and_poll:Function]
|
||||
# @PURPOSE: Execute SQL and wait for completion. One-shot convenience method.
|
||||
# @PRE: sql is valid SQL.
|
||||
# @POST: Returns final execution result.
|
||||
# @SIDE_EFFECT: Makes HTTP calls to Superset API.
|
||||
def execute_and_poll(
|
||||
self,
|
||||
sql: str,
|
||||
database_id: Optional[int] = None,
|
||||
max_polls: int = 60,
|
||||
poll_interval_seconds: float = 2.0,
|
||||
) -> Dict[str, Any]:
|
||||
with belief_scope("SupersetSqlLabExecutor.execute_and_poll"):
|
||||
exec_result = self.execute_sql(
|
||||
sql=sql,
|
||||
database_id=database_id,
|
||||
run_async=True,
|
||||
)
|
||||
|
||||
query_id = exec_result.get("query_id")
|
||||
if not query_id:
|
||||
logger.explore("No query_id from SQL Lab execute", {
|
||||
"raw_response": exec_result.get("raw_response"),
|
||||
})
|
||||
return {
|
||||
"status": "failed",
|
||||
"error_message": "No query_id returned from SQL Lab",
|
||||
"query_id": None,
|
||||
}
|
||||
|
||||
return self.poll_execution_status(
|
||||
query_id=str(query_id),
|
||||
max_polls=max_polls,
|
||||
poll_interval_seconds=poll_interval_seconds,
|
||||
)
|
||||
# [/DEF:execute_and_poll:Function]
|
||||
|
||||
# [DEF:get_query_results:Function]
|
||||
# @PURPOSE: Fetch the results of a completed query.
|
||||
# @PRE: query_id is a valid Superset query ID.
|
||||
# @POST: Returns query results if available.
|
||||
# @SIDE_EFFECT: Makes HTTP GET to Superset.
|
||||
def get_query_results(self, query_id: str) -> Dict[str, Any]:
|
||||
with belief_scope("SupersetSqlLabExecutor.get_query_results"):
|
||||
client = self._get_client()
|
||||
try:
|
||||
response = client.network.request(
|
||||
method="GET",
|
||||
endpoint=f"/api/v1/query/{query_id}/results",
|
||||
)
|
||||
result = response if isinstance(response, dict) else {}
|
||||
return {
|
||||
"query_id": query_id,
|
||||
"status": "success" if result.get("results") else "no_results",
|
||||
"results": result.get("results"),
|
||||
}
|
||||
except Exception as e:
|
||||
logger.explore("Failed to fetch query results", {
|
||||
"query_id": query_id,
|
||||
"error": str(e),
|
||||
})
|
||||
return {
|
||||
"query_id": query_id,
|
||||
"status": "failed",
|
||||
"error_message": str(e),
|
||||
"results": None,
|
||||
}
|
||||
# [/DEF:get_query_results:Function]
|
||||
|
||||
|
||||
# [/DEF:SupersetSqlLabExecutor:Class]
|
||||
# [/DEF:SupersetSqlLabExecutor:Module]
|
||||
491
backend/src/schemas/translate.py
Normal file
491
backend/src/schemas/translate.py
Normal file
@@ -0,0 +1,491 @@
|
||||
# [DEF:TranslateSchemas:Module]
|
||||
# @COMPLEXITY: 2
|
||||
# @SEMANTICS: translate, schemas, pydantic
|
||||
# @PURPOSE: Pydantic v2 schemas for translation API request/response serialization.
|
||||
# @LAYER: API
|
||||
# @RELATION: DEPENDS_ON -> pydantic
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
from pydantic import BaseModel, Field
|
||||
import json
|
||||
|
||||
|
||||
# [DEF:TranslateJobCreate:Class]
|
||||
# @PURPOSE: Schema for creating a new translation job.
|
||||
class TranslateJobCreate(BaseModel):
|
||||
name: str
|
||||
description: Optional[str] = None
|
||||
source_dialect: str = Field(..., description="Source database dialect (e.g. postgresql, clickhouse)")
|
||||
target_dialect: str = Field(..., description="Target database dialect (e.g. postgresql, clickhouse)")
|
||||
database_dialect: Optional[str] = Field(None, description="Detected dialect from Superset connection at save time")
|
||||
source_datasource_id: Optional[str] = Field(None, description="Superset datasource ID")
|
||||
source_table: Optional[str] = Field(None, description="Source table name")
|
||||
target_schema: Optional[str] = Field(None, description="Target table schema")
|
||||
target_table: Optional[str] = Field(None, description="Target table name")
|
||||
source_key_cols: Optional[List[str]] = Field(default_factory=list, description="Source key column names")
|
||||
target_key_cols: Optional[List[str]] = Field(default_factory=list, description="Target key column names")
|
||||
translation_column: Optional[str] = Field(None, description="Column to translate")
|
||||
context_columns: Optional[List[str]] = Field(default_factory=list, description="Context column names")
|
||||
target_language: Optional[str] = Field(None, description="Target language code")
|
||||
provider_id: Optional[str] = Field(None, description="LLM provider ID")
|
||||
batch_size: int = Field(50, description="Records per batch")
|
||||
upsert_strategy: str = Field("MERGE", description="UPSERT strategy: MERGE, INSERT, UPDATE")
|
||||
dictionary_ids: Optional[List[str]] = Field(default_factory=list, description="Associated terminology dictionary IDs")
|
||||
# [/DEF:TranslateJobCreate:Class]
|
||||
|
||||
|
||||
# [DEF:TranslateJobUpdate:Class]
|
||||
# @PURPOSE: Schema for updating an existing translation job.
|
||||
class TranslateJobUpdate(BaseModel):
|
||||
name: Optional[str] = None
|
||||
description: Optional[str] = None
|
||||
source_dialect: Optional[str] = None
|
||||
target_dialect: Optional[str] = None
|
||||
database_dialect: Optional[str] = None
|
||||
source_datasource_id: Optional[str] = None
|
||||
source_table: Optional[str] = None
|
||||
target_schema: Optional[str] = None
|
||||
target_table: Optional[str] = None
|
||||
source_key_cols: Optional[List[str]] = None
|
||||
target_key_cols: Optional[List[str]] = None
|
||||
translation_column: Optional[str] = None
|
||||
context_columns: Optional[List[str]] = None
|
||||
target_language: Optional[str] = None
|
||||
provider_id: Optional[str] = None
|
||||
batch_size: Optional[int] = None
|
||||
upsert_strategy: Optional[str] = None
|
||||
status: Optional[str] = None
|
||||
dictionary_ids: Optional[List[str]] = None
|
||||
# [/DEF:TranslateJobUpdate:Class]
|
||||
|
||||
|
||||
# [DEF:TranslateJobResponse:Class]
|
||||
# @PURPOSE: Schema for translation job API responses.
|
||||
class TranslateJobResponse(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
description: Optional[str] = None
|
||||
source_dialect: str
|
||||
target_dialect: str
|
||||
database_dialect: Optional[str] = None
|
||||
source_datasource_id: Optional[str] = None
|
||||
source_table: Optional[str] = None
|
||||
target_schema: Optional[str] = None
|
||||
target_table: Optional[str] = None
|
||||
source_key_cols: Optional[List[str]] = None
|
||||
target_key_cols: Optional[List[str]] = None
|
||||
translation_column: Optional[str] = None
|
||||
context_columns: Optional[List[str]] = None
|
||||
target_language: Optional[str] = None
|
||||
provider_id: Optional[str] = None
|
||||
batch_size: int = 50
|
||||
upsert_strategy: str = "MERGE"
|
||||
status: str
|
||||
created_by: Optional[str] = None
|
||||
created_at: datetime
|
||||
updated_at: Optional[datetime] = None
|
||||
dictionary_ids: Optional[List[str]] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# [/DEF:TranslateJobResponse:Class]
|
||||
|
||||
|
||||
# [DEF:DatasourceColumnResponse:Class]
|
||||
# @PURPOSE: Schema for datasource column metadata response.
|
||||
class DatasourceColumnResponse(BaseModel):
|
||||
name: str
|
||||
type: Optional[str] = None
|
||||
is_physical: bool = True
|
||||
is_dttm: bool = False
|
||||
description: Optional[str] = None
|
||||
|
||||
|
||||
# [DEF:DatasourceColumnsResponse:Class]
|
||||
# @PURPOSE: Schema for datasource columns endpoint response.
|
||||
class DatasourceColumnsResponse(BaseModel):
|
||||
datasource_id: int
|
||||
datasource_name: Optional[str] = None
|
||||
schema_name: Optional[str] = None
|
||||
database_dialect: str
|
||||
columns: List[DatasourceColumnResponse] = []
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# [/DEF:DatasourceColumnsResponse:Class]
|
||||
|
||||
|
||||
# [DEF:DuplicateJobResponse:Class]
|
||||
# @PURPOSE: Schema for duplicate job response.
|
||||
class DuplicateJobResponse(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
message: str = "Job duplicated successfully"
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# [/DEF:DuplicateJobResponse:Class]
|
||||
|
||||
|
||||
# [DEF:DictionaryCreate:Class]
|
||||
# @PURPOSE: Schema for creating a new terminology dictionary.
|
||||
class DictionaryCreate(BaseModel):
|
||||
name: str
|
||||
description: Optional[str] = None
|
||||
source_dialect: str
|
||||
target_dialect: str
|
||||
is_active: bool = True
|
||||
# [/DEF:DictionaryCreate:Class]
|
||||
|
||||
|
||||
# [DEF:DictionaryImport:Class]
|
||||
# @PURPOSE: Schema for importing entries into a terminology dictionary.
|
||||
class DictionaryImport(BaseModel):
|
||||
content: str = Field(..., description="CSV or TSV content as raw string")
|
||||
delimiter: Optional[str] = Field(None, description="Detected or forced delimiter: ',' or '\\t'. Auto-detect if omitted.")
|
||||
on_conflict: str = Field("overwrite", description="'overwrite' or 'keep_existing' or 'cancel'")
|
||||
preview_only: bool = Field(False, description="If true, return preview without applying")
|
||||
# [/DEF:DictionaryImport:Class]
|
||||
|
||||
|
||||
# [DEF:DictionaryResponse:Class]
|
||||
# @PURPOSE: Schema for terminology dictionary API responses.
|
||||
class DictionaryResponse(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
description: Optional[str] = None
|
||||
source_dialect: str
|
||||
target_dialect: str
|
||||
is_active: bool
|
||||
created_by: Optional[str] = None
|
||||
created_at: datetime
|
||||
updated_at: Optional[datetime] = None
|
||||
entry_count: Optional[int] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# [/DEF:DictionaryResponse:Class]
|
||||
|
||||
|
||||
# [DEF:DictionaryEntryCreate:Class]
|
||||
# @PURPOSE: Schema for adding/editing a dictionary entry.
|
||||
class DictionaryEntryCreate(BaseModel):
|
||||
source_term: str = Field(..., description="Source term to translate")
|
||||
target_term: str = Field(..., description="Target/translated term")
|
||||
context_notes: Optional[str] = Field(None, description="Optional context notes")
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# [/DEF:DictionaryEntryCreate:Class]
|
||||
|
||||
|
||||
# [DEF:DictionaryEntryResponse:Class]
|
||||
# @PURPOSE: Schema for dictionary entry API responses.
|
||||
class DictionaryEntryResponse(BaseModel):
|
||||
id: str
|
||||
dictionary_id: str
|
||||
source_term: str
|
||||
source_term_normalized: str
|
||||
target_term: str
|
||||
context_notes: Optional[str] = None
|
||||
created_at: datetime
|
||||
updated_at: Optional[datetime] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# [/DEF:DictionaryEntryResponse:Class]
|
||||
|
||||
|
||||
# [DEF:DictionaryImportResult:Class]
|
||||
# @PURPOSE: Schema for dictionary import result.
|
||||
class DictionaryImportResult(BaseModel):
|
||||
total: int = 0
|
||||
created: int = 0
|
||||
updated: int = 0
|
||||
skipped: int = 0
|
||||
errors: List[Dict[str, Any]] = Field(default_factory=list)
|
||||
preview: List[Dict[str, Any]] = Field(default_factory=list, description="Preview rows with conflict flags")
|
||||
# [/DEF:DictionaryImportResult:Class]
|
||||
|
||||
|
||||
# [DEF:PreviewRequest:Class]
|
||||
# @PURPOSE: Schema for triggering a translation preview.
|
||||
class PreviewRequest(BaseModel):
|
||||
sample_size: int = Field(10, ge=1, le=100, description="Number of sample rows to preview")
|
||||
prompt_template: Optional[str] = Field(None, description="Optional custom prompt template")
|
||||
|
||||
|
||||
# [DEF:PreviewRowUpdate:Class]
|
||||
# @PURPOSE: Schema for approving/editing/rejecting a preview row.
|
||||
class PreviewRowUpdate(BaseModel):
|
||||
action: str = Field(..., description="'approve', 'reject', or 'edit'")
|
||||
translation: Optional[str] = Field(None, description="Edited translation (required for 'edit' action)")
|
||||
feedback: Optional[str] = Field(None, description="Optional feedback/comment")
|
||||
# [/DEF:PreviewRowUpdate:Class]
|
||||
|
||||
|
||||
# [DEF:PreviewAcceptResponse:Class]
|
||||
# @PURPOSE: Schema for preview accept response.
|
||||
class PreviewAcceptResponse(BaseModel):
|
||||
id: str
|
||||
job_id: str
|
||||
status: str
|
||||
created_by: Optional[str] = None
|
||||
created_at: datetime
|
||||
expires_at: Optional[datetime] = None
|
||||
records: List['PreviewRow'] = []
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# [/DEF:PreviewAcceptResponse:Class]
|
||||
|
||||
|
||||
# [DEF:CostEstimate:Class]
|
||||
# @PURPOSE: Schema for cost estimation in preview response.
|
||||
class CostEstimate(BaseModel):
|
||||
sample_size: int = 0
|
||||
sample_prompt_tokens: int = 0
|
||||
sample_output_tokens: int = 0
|
||||
sample_total_tokens: int = 0
|
||||
sample_cost: float = 0.0
|
||||
estimated_total_rows: int = 0
|
||||
estimated_tokens: int = 0
|
||||
estimated_cost: float = 0.0
|
||||
# [/DEF:CostEstimate:Class]
|
||||
|
||||
|
||||
# [DEF:TermCorrectionSubmit:Class]
|
||||
# @PURPOSE: Schema for submitting a term correction in a translation preview.
|
||||
class TermCorrectionSubmit(BaseModel):
|
||||
source_term: str
|
||||
incorrect_target_term: str
|
||||
corrected_target_term: str
|
||||
dictionary_id: Optional[str] = Field(None, description="Target dictionary ID (language-filtered)")
|
||||
origin_run_id: Optional[str] = Field(None, description="Run ID from which this correction originated")
|
||||
origin_row_key: Optional[str] = Field(None, description="Row key within the run")
|
||||
# [/DEF:TermCorrectionSubmit:Class]
|
||||
|
||||
|
||||
# [DEF:TermCorrectionBulkSubmit:Class]
|
||||
# @PURPOSE: Schema for submitting multiple term corrections atomically.
|
||||
class TermCorrectionBulkSubmit(BaseModel):
|
||||
corrections: List[TermCorrectionSubmit]
|
||||
dictionary_id: str = Field(..., description="Target dictionary ID for all corrections")
|
||||
# [/DEF:TermCorrectionBulkSubmit:Class]
|
||||
|
||||
|
||||
# [DEF:CorrectionConflictResult:Class]
|
||||
# @PURPOSE: Schema for reporting conflicts in correction submission.
|
||||
class CorrectionConflictResult(BaseModel):
|
||||
source_term: str
|
||||
existing_target_term: str
|
||||
submitted_target_term: str
|
||||
action: str = "keep_existing"
|
||||
# [/DEF:CorrectionConflictResult:Class]
|
||||
|
||||
|
||||
# [DEF:CorrectionSubmitResponse:Class]
|
||||
# @PURPOSE: Schema for correction submission response.
|
||||
class CorrectionSubmitResponse(BaseModel):
|
||||
entry_id: Optional[str] = None
|
||||
action: str # "created", "updated", "conflict_detected", "skipped"
|
||||
source_term: str
|
||||
target_term: str
|
||||
conflict: Optional[CorrectionConflictResult] = None
|
||||
message: Optional[str] = None
|
||||
# [/DEF:CorrectionSubmitResponse:Class]
|
||||
|
||||
|
||||
# [DEF:ScheduleConfig:Class]
|
||||
# @PURPOSE: Schema for configuring a recurring translation schedule.
|
||||
class ScheduleConfig(BaseModel):
|
||||
cron_expression: str = Field(..., description="Cron expression for scheduling (e.g. '0 2 * * *')")
|
||||
timezone: str = Field("UTC", description="Timezone for the cron schedule (e.g. 'UTC', 'Europe/Moscow')")
|
||||
is_active: bool = True
|
||||
# [/DEF:ScheduleConfig:Class]
|
||||
|
||||
|
||||
# [DEF:ScheduleResponse:Class]
|
||||
# @PURPOSE: Schema for schedule API responses.
|
||||
class ScheduleResponse(BaseModel):
|
||||
id: str
|
||||
job_id: str
|
||||
cron_expression: str
|
||||
timezone: str
|
||||
is_active: bool
|
||||
last_run_at: Optional[datetime] = None
|
||||
next_run_at: Optional[datetime] = None
|
||||
created_by: Optional[str] = None
|
||||
created_at: datetime
|
||||
updated_at: Optional[datetime] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# [/DEF:ScheduleResponse:Class]
|
||||
|
||||
|
||||
# [DEF:NextExecutionResponse:Class]
|
||||
# @PURPOSE: Schema for next execution preview.
|
||||
class NextExecutionResponse(BaseModel):
|
||||
job_id: str
|
||||
cron_expression: str
|
||||
timezone: str
|
||||
next_executions: List[str] = []
|
||||
# [/DEF:NextExecutionResponse:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationRunResponse:Class]
|
||||
# @PURPOSE: Schema for translation run API responses.
|
||||
class TranslationRunResponse(BaseModel):
|
||||
id: str
|
||||
job_id: str
|
||||
status: str
|
||||
trigger_type: Optional[str] = None
|
||||
started_at: Optional[datetime] = None
|
||||
completed_at: Optional[datetime] = None
|
||||
error_message: Optional[str] = None
|
||||
total_records: int = 0
|
||||
successful_records: int = 0
|
||||
failed_records: int = 0
|
||||
skipped_records: int = 0
|
||||
insert_status: Optional[str] = None
|
||||
superset_execution_id: Optional[str] = None
|
||||
config_snapshot: Optional[Dict[str, Any]] = None
|
||||
key_hash: Optional[str] = None
|
||||
config_hash: Optional[str] = None
|
||||
dict_snapshot_hash: Optional[str] = None
|
||||
created_by: Optional[str] = None
|
||||
created_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# [/DEF:TranslationRunResponse:Class]
|
||||
|
||||
|
||||
# [DEF:RunDetailResponse:Class]
|
||||
# @PURPOSE: Schema for detailed run response including records, events, and config snapshot.
|
||||
class RunDetailResponse(BaseModel):
|
||||
run: TranslationRunResponse
|
||||
records: List[Dict[str, Any]] = Field(default_factory=list, description="Paginated translation records")
|
||||
events: List[Dict[str, Any]] = Field(default_factory=list, description="Run events")
|
||||
event_invariants: Optional[Dict[str, Any]] = None
|
||||
batch_count: int = 0
|
||||
# [/DEF:RunDetailResponse:Class>
|
||||
|
||||
|
||||
# [DEF:RunHistoryFilter:Class]
|
||||
# @PURPOSE: Schema for filtering run history.
|
||||
class RunHistoryFilter(BaseModel):
|
||||
job_id: Optional[str] = None
|
||||
status: Optional[str] = None
|
||||
trigger_type: Optional[str] = None
|
||||
created_by: Optional[str] = None
|
||||
date_from: Optional[datetime] = None
|
||||
date_to: Optional[datetime] = None
|
||||
page: int = 1
|
||||
page_size: int = 20
|
||||
# [/DEF:RunHistoryFilter:Class>
|
||||
|
||||
|
||||
# [DEF:RunListResponse:Class]
|
||||
# @PURPOSE: Schema for paginated run list response.
|
||||
class RunListResponse(BaseModel):
|
||||
items: List[TranslationRunResponse] = []
|
||||
total: int = 0
|
||||
page: int = 1
|
||||
page_size: int = 20
|
||||
# [/DEF:RunListResponse:Class>
|
||||
|
||||
|
||||
# [DEF:AggregatedMetricsResponse:Class]
|
||||
# @PURPOSE: Schema for aggregated per-job metrics.
|
||||
class AggregatedMetricsResponse(BaseModel):
|
||||
job_id: str
|
||||
total_runs: int = 0
|
||||
successful_runs: int = 0
|
||||
failed_runs: int = 0
|
||||
cancelled_runs: int = 0
|
||||
total_records: int = 0
|
||||
successful_records: int = 0
|
||||
failed_records: int = 0
|
||||
skipped_records: int = 0
|
||||
cumulative_tokens: Optional[int] = None
|
||||
cumulative_cost: Optional[float] = None
|
||||
avg_duration_ms: Optional[float] = None
|
||||
last_run_at: Optional[datetime] = None
|
||||
next_scheduled_run: Optional[datetime] = None
|
||||
# [/DEF:AggregatedMetricsResponse:Class]
|
||||
|
||||
|
||||
# [DEF:PreviewRow:Class]
|
||||
# @PURPOSE: A single row in a translation preview showing original and translated content side by side.
|
||||
class PreviewRow(BaseModel):
|
||||
id: str
|
||||
source_sql: Optional[str] = None
|
||||
target_sql: Optional[str] = None
|
||||
source_object_type: Optional[str] = None
|
||||
source_object_id: Optional[str] = None
|
||||
source_object_name: Optional[str] = None
|
||||
status: str = "PENDING"
|
||||
feedback: Optional[str] = None
|
||||
# [/DEF:PreviewRow:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationPreviewResponse:Class]
|
||||
# @PURPOSE: Schema for translation preview session API responses.
|
||||
class TranslationPreviewResponse(BaseModel):
|
||||
id: str
|
||||
job_id: str
|
||||
run_id: Optional[str] = None
|
||||
status: str
|
||||
created_by: Optional[str] = None
|
||||
created_at: datetime
|
||||
expires_at: Optional[datetime] = None
|
||||
records: List[PreviewRow] = []
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# [/DEF:TranslationPreviewResponse:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationBatchResponse:Class]
|
||||
# @PURPOSE: Schema for translation batch API responses.
|
||||
class TranslationBatchResponse(BaseModel):
|
||||
id: str
|
||||
run_id: str
|
||||
batch_index: int
|
||||
status: str
|
||||
total_records: int = 0
|
||||
successful_records: int = 0
|
||||
failed_records: int = 0
|
||||
started_at: Optional[datetime] = None
|
||||
completed_at: Optional[datetime] = None
|
||||
created_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# [/DEF:TranslationBatchResponse:Class]
|
||||
|
||||
|
||||
# [DEF:MetricsResponse:Class]
|
||||
# @PURPOSE: Schema for translation metrics API responses.
|
||||
class MetricsResponse(BaseModel):
|
||||
job_id: str
|
||||
snapshot_date: datetime
|
||||
total_jobs: int = 0
|
||||
total_runs: int = 0
|
||||
total_records: int = 0
|
||||
successful_records: int = 0
|
||||
failed_records: int = 0
|
||||
skipped_records: int = 0
|
||||
avg_duration_ms: Optional[int] = None
|
||||
p50_duration_ms: Optional[int] = None
|
||||
p95_duration_ms: Optional[int] = None
|
||||
p99_duration_ms: Optional[int] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# [/DEF:MetricsResponse:Class]
|
||||
|
||||
# [/DEF:TranslateSchemas:Module]
|
||||
252
backend/tests/test_translate_corrections.py
Normal file
252
backend/tests/test_translate_corrections.py
Normal file
@@ -0,0 +1,252 @@
|
||||
# [DEF:TranslateCorrectionTests:Module]
|
||||
# @COMPLEXITY: 4
|
||||
# @SEMANTICS: tests, translate, corrections, dictionary
|
||||
# @PURPOSE: Tests for term correction API endpoints and DictionaryManager correction methods.
|
||||
# @LAYER: Test
|
||||
# @RELATION: BINDS_TO -> [DictionaryManagerModule:Module]
|
||||
# @RELATION: BINDS_TO -> [TranslateRoutes:Module]
|
||||
#
|
||||
# @TEST_CONTRACT: CorrectionFlow ->
|
||||
# {
|
||||
# required_fields: {source_term, incorrect_target_term, corrected_target_term, dictionary_id},
|
||||
# invariants: [
|
||||
# "POST /api/translate/corrections creates or updates entry",
|
||||
# "Conflict detection works for existing entries",
|
||||
# "POST /api/translate/corrections/bulk atomic all-or-nothing",
|
||||
# "Origin tracking fields are populated"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_FIXTURE: valid_dictionary -> created via DictionaryManager
|
||||
# @TEST_EDGE: missing_dictionary_id -> 422
|
||||
# @TEST_EDGE: conflict_detected -> conflict response returned
|
||||
# @TEST_EDGE: bulk_correction_all_succeed -> atomic commit
|
||||
# @TEST_EDGE: bulk_correction_with_conflicts -> conflicts listed
|
||||
# @TEST_INVARIANT: correction_flow -> VERIFIED_BY: [valid_dictionary, conflict_detected]
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from datetime import datetime, timezone
|
||||
import uuid
|
||||
|
||||
from src.core.database import Base
|
||||
from src.app import app
|
||||
from src.dependencies import get_current_user, get_config_manager, has_permission, get_db
|
||||
from src.plugins.translate.dictionary import DictionaryManager
|
||||
from src.models.translate import TerminologyDictionary, DictionaryEntry
|
||||
|
||||
# In-memory SQLite
|
||||
SQLALCHEMY_DATABASE_URL = "sqlite:///:memory:"
|
||||
engine = create_engine(SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False})
|
||||
TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
mock_user = MagicMock()
|
||||
mock_user.username = "testuser"
|
||||
mock_user.roles = []
|
||||
admin_role = MagicMock()
|
||||
admin_role.name = "Admin"
|
||||
mock_user.roles.append(admin_role)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db_session():
|
||||
connection = engine.connect()
|
||||
transaction = connection.begin()
|
||||
session = TestingSessionLocal(bind=connection)
|
||||
yield session
|
||||
session.close()
|
||||
transaction.rollback()
|
||||
connection.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_api_deps():
|
||||
config_manager = MagicMock()
|
||||
config_manager.get_environments.return_value = []
|
||||
|
||||
connection = engine.connect()
|
||||
transaction = connection.begin()
|
||||
session = TestingSessionLocal(bind=connection)
|
||||
|
||||
def override_get_db():
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
pass
|
||||
|
||||
overrides = {
|
||||
get_current_user: lambda: mock_user,
|
||||
get_config_manager: lambda: config_manager,
|
||||
get_db: override_get_db,
|
||||
}
|
||||
for dep, override in overrides.items():
|
||||
app.dependency_overrides[dep] = override
|
||||
|
||||
yield {"config_manager": config_manager, "session": session}
|
||||
|
||||
app.dependency_overrides.clear()
|
||||
session.close()
|
||||
transaction.rollback()
|
||||
connection.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(mock_api_deps):
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
# [DEF:test_submit_correction_creates_entry:Function]
|
||||
# @PURPOSE: Verify that a correction creates a new dictionary entry.
|
||||
def test_submit_correction_creates_entry(db_session):
|
||||
"""Test that submitting a correction creates a new entry."""
|
||||
dict_obj = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test Dict", source_dialect="en", target_dialect="ru",
|
||||
created_by="testuser",
|
||||
)
|
||||
|
||||
result = DictionaryManager.submit_correction(
|
||||
db_session,
|
||||
dict_id=dict_obj.id,
|
||||
source_term="hello",
|
||||
incorrect_target_term="privet",
|
||||
corrected_target_term="zdravstvuyte",
|
||||
origin_run_id="run-123",
|
||||
origin_row_key="row-1",
|
||||
origin_user_id="testuser",
|
||||
)
|
||||
|
||||
assert result["action"] == "created"
|
||||
assert result["entry_id"] is not None
|
||||
|
||||
entries, total = DictionaryManager.list_entries(db_session, dict_obj.id)
|
||||
assert total == 1
|
||||
assert entries[0].source_term == "hello"
|
||||
assert entries[0].target_term == "zdravstvuyte"
|
||||
assert entries[0].origin_run_id == "run-123"
|
||||
assert entries[0].origin_row_key == "row-1"
|
||||
assert entries[0].origin_user_id == "testuser"
|
||||
# [/DEF:test_submit_correction_creates_entry:Function]
|
||||
|
||||
|
||||
# [DEF:test_submit_correction_conflict_detected:Function]
|
||||
# @PURPOSE: Verify conflict detection when entry already exists.
|
||||
def test_submit_correction_conflict_detected(db_session):
|
||||
"""Test that conflict is detected when entry already exists."""
|
||||
dict_obj = DictionaryManager.create_dictionary(
|
||||
db_session, name="Dict", source_dialect="en", target_dialect="ru",
|
||||
)
|
||||
|
||||
# Create initial entry
|
||||
DictionaryManager.add_entry(db_session, dict_obj.id, "hello", "privet")
|
||||
|
||||
# Submit correction with conflict
|
||||
result = DictionaryManager.submit_correction(
|
||||
db_session,
|
||||
dict_id=dict_obj.id,
|
||||
source_term="hello",
|
||||
incorrect_target_term="privet",
|
||||
corrected_target_term="zdravstvuyte",
|
||||
on_conflict="keep_existing",
|
||||
)
|
||||
|
||||
assert result["action"] == "conflict_detected"
|
||||
assert result["conflict"] is not None
|
||||
assert result["conflict"]["existing_target_term"] == "privet"
|
||||
# [/DEF:test_submit_correction_conflict_detected:Function]
|
||||
|
||||
|
||||
# [DEF:test_submit_correction_overwrite:Function]
|
||||
# @PURPOSE: Verify that correction overwrites existing entry.
|
||||
def test_submit_correction_overwrite(db_session):
|
||||
"""Test that correction overwrites existing entry."""
|
||||
dict_obj = DictionaryManager.create_dictionary(
|
||||
db_session, name="Dict", source_dialect="en", target_dialect="ru",
|
||||
)
|
||||
|
||||
DictionaryManager.add_entry(db_session, dict_obj.id, "hello", "privet")
|
||||
|
||||
result = DictionaryManager.submit_correction(
|
||||
db_session,
|
||||
dict_id=dict_obj.id,
|
||||
source_term="hello",
|
||||
incorrect_target_term="privet",
|
||||
corrected_target_term="hi_there",
|
||||
on_conflict="overwrite",
|
||||
)
|
||||
|
||||
assert result["action"] == "updated"
|
||||
entries, _ = DictionaryManager.list_entries(db_session, dict_obj.id)
|
||||
assert entries[0].target_term == "hi_there"
|
||||
# [/DEF:test_submit_correction_overwrite:Function]
|
||||
|
||||
|
||||
# [DEF:test_bulk_corrections_atomic:Function]
|
||||
# @PURPOSE: Verify bulk corrections are applied atomically.
|
||||
def test_bulk_corrections_atomic(db_session):
|
||||
"""Test that bulk corrections are applied atomically."""
|
||||
dict_obj = DictionaryManager.create_dictionary(
|
||||
db_session, name="Bulk Dict", source_dialect="en", target_dialect="ru",
|
||||
)
|
||||
|
||||
corrections = [
|
||||
{"source_term": "cat", "incorrect_target_term": "kot", "corrected_target_term": "koshka"},
|
||||
{"source_term": "dog", "incorrect_target_term": "sobaka", "corrected_target_term": "pyos"},
|
||||
]
|
||||
|
||||
result = DictionaryManager.submit_bulk_corrections(
|
||||
db_session,
|
||||
dict_id=dict_obj.id,
|
||||
corrections=corrections,
|
||||
origin_user_id="testuser",
|
||||
)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
assert result["applied"] == 2
|
||||
assert len(result["conflicts"]) == 0
|
||||
|
||||
entries, total = DictionaryManager.list_entries(db_session, dict_obj.id)
|
||||
assert total == 2
|
||||
# [/DEF:test_bulk_corrections_atomic:Function]
|
||||
|
||||
|
||||
# [DEF:test_api_correction_missing_dict:Function]
|
||||
# @PURPOSE: Verify POST corrections without dictionary_id returns 422.
|
||||
def test_api_correction_missing_dict(client):
|
||||
"""Test correction without dict_id returns 422."""
|
||||
response = client.post("/api/translate/corrections", json={
|
||||
"source_term": "hello",
|
||||
"incorrect_target_term": "privet",
|
||||
"corrected_target_term": "zdravstvuyte",
|
||||
})
|
||||
assert response.status_code == 422
|
||||
# [/DEF:test_api_correction_missing_dict:Function]
|
||||
|
||||
|
||||
# [DEF:test_api_bulk_corrections:Function]
|
||||
# @PURPOSE: Verify POST /corrections/bulk works.
|
||||
def test_api_bulk_corrections(client, mock_api_deps):
|
||||
"""Test bulk corrections endpoint."""
|
||||
session = mock_api_deps["session"]
|
||||
dict_obj = DictionaryManager.create_dictionary(
|
||||
session, name="API Dict", source_dialect="en", target_dialect="ru",
|
||||
)
|
||||
|
||||
response = client.post("/api/translate/corrections/bulk", json={
|
||||
"dictionary_id": dict_obj.id,
|
||||
"corrections": [
|
||||
{"source_term": "hello", "incorrect_target_term": "privet", "corrected_target_term": "hi"},
|
||||
],
|
||||
})
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "completed"
|
||||
# [/DEF:test_api_bulk_corrections:Function]
|
||||
# [/DEF:TranslateCorrectionTests:Module]
|
||||
300
backend/tests/test_translate_history.py
Normal file
300
backend/tests/test_translate_history.py
Normal file
@@ -0,0 +1,300 @@
|
||||
# [DEF:TranslateHistoryTests:Module]
|
||||
# @COMPLEXITY: 4
|
||||
# @SEMANTICS: tests, translate, history, metrics
|
||||
# @PURPOSE: Tests for run history list/detail endpoints and metrics aggregation.
|
||||
# @LAYER: Test
|
||||
# @RELATION: BINDS_TO -> [TranslateRoutes:Module]
|
||||
# @RELATION: BINDS_TO -> [TranslationMetrics:Module]
|
||||
#
|
||||
# @TEST_CONTRACT: HistoryFlow ->
|
||||
# {
|
||||
# invariants: [
|
||||
# "GET /api/translate/runs returns paginated list",
|
||||
# "GET /api/translate/runs supports filters (job_id, status, trigger_type)",
|
||||
# "GET /api/translate/runs/{id}/detail returns config_snapshot, records, events",
|
||||
# "GET /api/translate/jobs/{id}/metrics returns aggregated data",
|
||||
# "Metrics include run counts, record counts, duration"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_FIXTURE: completed_run -> run with COMPLETED status
|
||||
# @TEST_EDGE: filter_by_job_id -> returns only that job's runs
|
||||
# @TEST_EDGE: filter_by_status -> returns only matching status
|
||||
# @TEST_EDGE: metrics_empty_job -> returns zero counts
|
||||
# @TEST_INVARIANT: history_list_and_detail -> VERIFIED_BY: [completed_run, filter_by_status]
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from datetime import datetime, timezone
|
||||
import uuid
|
||||
import json
|
||||
|
||||
from src.core.database import Base
|
||||
from src.app import app
|
||||
from src.dependencies import get_current_user, get_config_manager, has_permission, get_db
|
||||
from src.plugins.translate.metrics import TranslationMetrics
|
||||
from src.models.translate import TranslationJob, TranslationRun, TranslationEvent, MetricSnapshot
|
||||
|
||||
# In-memory SQLite
|
||||
SQLALCHEMY_DATABASE_URL = "sqlite:///:memory:"
|
||||
engine = create_engine(SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False})
|
||||
TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
mock_user = MagicMock()
|
||||
mock_user.username = "testuser"
|
||||
mock_user.roles = []
|
||||
admin_role = MagicMock()
|
||||
admin_role.name = "Admin"
|
||||
mock_user.roles.append(admin_role)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db_session():
|
||||
connection = engine.connect()
|
||||
transaction = connection.begin()
|
||||
session = TestingSessionLocal(bind=connection)
|
||||
yield session
|
||||
session.close()
|
||||
transaction.rollback()
|
||||
connection.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_api_deps():
|
||||
config_manager = MagicMock()
|
||||
config_manager.get_environments.return_value = []
|
||||
|
||||
connection = engine.connect()
|
||||
transaction = connection.begin()
|
||||
session = TestingSessionLocal(bind=connection)
|
||||
|
||||
def override_get_db():
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
pass
|
||||
|
||||
overrides = {
|
||||
get_current_user: lambda: mock_user,
|
||||
get_config_manager: lambda: config_manager,
|
||||
get_db: override_get_db,
|
||||
}
|
||||
for dep, override in overrides.items():
|
||||
app.dependency_overrides[dep] = override
|
||||
|
||||
yield {"config_manager": config_manager, "session": session}
|
||||
|
||||
app.dependency_overrides.clear()
|
||||
session.close()
|
||||
transaction.rollback()
|
||||
connection.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(mock_api_deps):
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def _create_test_job(db_session) -> TranslationJob:
|
||||
job = TranslationJob(
|
||||
id=str(uuid.uuid4()),
|
||||
name="Test Job",
|
||||
source_dialect="postgresql",
|
||||
target_dialect="clickhouse",
|
||||
status="ACTIVE",
|
||||
)
|
||||
db_session.add(job)
|
||||
db_session.flush()
|
||||
return job
|
||||
|
||||
|
||||
def _create_test_run(db_session, job_id: str, status: str = "COMPLETED", trigger: str = "manual") -> TranslationRun:
|
||||
run = TranslationRun(
|
||||
id=str(uuid.uuid4()),
|
||||
job_id=job_id,
|
||||
status=status,
|
||||
trigger_type=trigger,
|
||||
total_records=100,
|
||||
successful_records=80,
|
||||
failed_records=10,
|
||||
skipped_records=10,
|
||||
config_snapshot={"test": True},
|
||||
config_hash="abc123",
|
||||
dict_snapshot_hash="def456",
|
||||
key_hash="ghi789",
|
||||
created_by="testuser",
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
db_session.add(run)
|
||||
db_session.flush()
|
||||
return run
|
||||
|
||||
|
||||
# [DEF:test_list_runs_empty:Function]
|
||||
# @PURPOSE: Verify runs list returns empty result initially.
|
||||
def test_list_runs_empty(client):
|
||||
"""Test runs list initially returns empty."""
|
||||
response = client.get("/api/translate/runs")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total"] == 0
|
||||
assert data["items"] == []
|
||||
# [/DEF:test_list_runs_empty:Function]
|
||||
|
||||
|
||||
# [DEF:test_list_runs_with_data:Function]
|
||||
# @PURPOSE: Verify runs list returns data when runs exist.
|
||||
def test_list_runs_with_data(client, mock_api_deps):
|
||||
"""Test runs list with existing runs."""
|
||||
session = mock_api_deps["session"]
|
||||
job = _create_test_job(session)
|
||||
_create_test_run(session, job.id)
|
||||
session.commit()
|
||||
|
||||
response = client.get("/api/translate/runs")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total"] >= 1
|
||||
assert len(data["items"]) >= 1
|
||||
assert data["items"][0]["job_id"] == job.id
|
||||
# [/DEF:test_list_runs_with_data:Function]
|
||||
|
||||
|
||||
# [DEF:test_list_runs_filter_job_id:Function]
|
||||
# @PURPOSE: Verify filtering runs by job_id works.
|
||||
def test_list_runs_filter_job_id(client, mock_api_deps):
|
||||
"""Test filtering runs by job_id."""
|
||||
session = mock_api_deps["session"]
|
||||
job1 = _create_test_job(session)
|
||||
job2 = _create_test_job(session)
|
||||
_create_test_run(session, job1.id)
|
||||
_create_test_run(session, job2.id)
|
||||
session.commit()
|
||||
|
||||
response = client.get(f"/api/translate/runs?job_id={job1.id}")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total"] >= 1
|
||||
for item in data["items"]:
|
||||
assert item["job_id"] == job1.id
|
||||
# [/DEF:test_list_runs_filter_job_id:Function]
|
||||
|
||||
|
||||
# [DEF:test_list_runs_filter_status:Function]
|
||||
# @PURPOSE: Verify filtering runs by status works.
|
||||
def test_list_runs_filter_status(client, mock_api_deps):
|
||||
"""Test filtering runs by status."""
|
||||
session = mock_api_deps["session"]
|
||||
job = _create_test_job(session)
|
||||
_create_test_run(session, job.id, status="COMPLETED")
|
||||
_create_test_run(session, job.id, status="FAILED")
|
||||
session.commit()
|
||||
|
||||
response = client.get(f"/api/translate/runs?status=FAILED")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
for item in data["items"]:
|
||||
assert item["status"] == "FAILED"
|
||||
# [/DEF:test_list_runs_filter_status:Function]
|
||||
|
||||
|
||||
# [DEF:test_get_run_detail:Function]
|
||||
# @PURPOSE: Verify run detail returns config_snapshot, records, events.
|
||||
def test_get_run_detail(client, mock_api_deps):
|
||||
"""Test run detail endpoint."""
|
||||
session = mock_api_deps["session"]
|
||||
job = _create_test_job(session)
|
||||
run = _create_test_run(session, job.id)
|
||||
|
||||
# Add an event
|
||||
event = TranslationEvent(
|
||||
id=str(uuid.uuid4()),
|
||||
job_id=job.id,
|
||||
run_id=run.id,
|
||||
event_type="RUN_STARTED",
|
||||
event_data={},
|
||||
created_by="testuser",
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
session.add(event)
|
||||
session.commit()
|
||||
|
||||
response = client.get(f"/api/translate/runs/{run.id}/detail")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["id"] == run.id
|
||||
assert data["config_snapshot"] == {"test": True}
|
||||
assert data["config_hash"] == "abc123"
|
||||
assert len(data["events"]) >= 1
|
||||
assert data["events"][0]["event_type"] == "RUN_STARTED"
|
||||
# [/DEF:test_get_run_detail:Function]
|
||||
|
||||
|
||||
# [DEF:test_get_job_metrics:Function]
|
||||
# @PURPOSE: Verify metrics endpoint returns aggregated data.
|
||||
def test_get_job_metrics(client, mock_api_deps):
|
||||
"""Test job metrics endpoint."""
|
||||
session = mock_api_deps["session"]
|
||||
job = _create_test_job(session)
|
||||
_create_test_run(session, job.id, status="COMPLETED")
|
||||
_create_test_run(session, job.id, status="FAILED")
|
||||
session.commit()
|
||||
|
||||
response = client.get(f"/api/translate/jobs/{job.id}/metrics")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["job_id"] == job.id
|
||||
assert data["total_runs"] >= 2
|
||||
assert data["successful_runs"] >= 1
|
||||
assert data["failed_runs"] >= 1
|
||||
# [/DEF:test_get_job_metrics:Function]
|
||||
|
||||
|
||||
# [DEF:test_get_all_metrics:Function]
|
||||
# @PURPOSE: Verify global metrics endpoint returns data.
|
||||
def test_get_all_metrics(client, mock_api_deps):
|
||||
"""Test global metrics endpoint."""
|
||||
session = mock_api_deps["session"]
|
||||
job = _create_test_job(session)
|
||||
_create_test_run(session, job.id)
|
||||
session.commit()
|
||||
|
||||
response = client.get("/api/translate/metrics")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert isinstance(data, list)
|
||||
assert len(data) >= 1
|
||||
# [/DEF:test_get_all_metrics:Function]
|
||||
|
||||
|
||||
# [DEF:test_metrics_empty_job:Function]
|
||||
# @PURPOSE: Verify metrics for job with no runs returns zeros.
|
||||
def test_metrics_empty_job(client, mock_api_deps):
|
||||
"""Test metrics for job with no runs."""
|
||||
session = mock_api_deps["session"]
|
||||
job = _create_test_job(session)
|
||||
session.commit()
|
||||
|
||||
response = client.get(f"/api/translate/jobs/{job.id}/metrics")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total_runs"] == 0
|
||||
# [/DEF:test_metrics_empty_job:Function]
|
||||
|
||||
|
||||
# [DEF:test_run_detail_not_found:Function]
|
||||
# @PURPOSE: Verify 404 on non-existent run detail.
|
||||
def test_run_detail_not_found(client):
|
||||
"""Test run detail returns 404 for non-existent run."""
|
||||
response = client.get("/api/translate/runs/non-existent/detail")
|
||||
assert response.status_code == 404
|
||||
# [/DEF:test_run_detail_not_found:Function]
|
||||
# [/DEF:TranslateHistoryTests:Module]
|
||||
569
backend/tests/test_translate_jobs.py
Normal file
569
backend/tests/test_translate_jobs.py
Normal file
@@ -0,0 +1,569 @@
|
||||
# [DEF:TranslateJobTests:Module]
|
||||
# @COMPLEXITY: 4
|
||||
# @SEMANTICS: tests, translate, jobs, crud, validation
|
||||
# @PURPOSE: Tests for translation job CRUD endpoints and service layer with column validation.
|
||||
# @LAYER: Test
|
||||
# @RELATION: BINDS_TO -> [TranslateRoutes:Module]
|
||||
# @RELATION: BINDS_TO -> [TranslateJobService:Module]
|
||||
#
|
||||
# @TEST_CONTRACT: TranslateJobCRUD ->
|
||||
# {
|
||||
# required_fields: {name: str, source_dialect: str, target_dialect: str},
|
||||
# optional_fields: {description, source_datasource_id, translation_column, ...},
|
||||
# invariants: [
|
||||
# "POST /api/translate/jobs returns 201 with valid config",
|
||||
# "GET /api/translate/jobs returns job list",
|
||||
# "GET /api/translate/jobs/{id} returns single job",
|
||||
# "PUT /api/translate/jobs/{id} updates and returns job",
|
||||
# "DELETE /api/translate/jobs/{id} returns 204",
|
||||
# "POST /api/translate/jobs/{id}/duplicate returns new job copy"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_FIXTURE: valid_job_payload -> {name: "Test Job", source_dialect: "postgresql", target_dialect: "clickhouse"}
|
||||
# @TEST_EDGE: missing_translation_column -> 422 when datasource set but no translation column
|
||||
# @TEST_EDGE: virtual_key_column_warning -> warning emitted when virtual column used as key
|
||||
# @TEST_EDGE: invalid_dialect_rejected -> error on unsupported dialect
|
||||
# @TEST_INVARIANT: valid_CRUD_flow -> VERIFIED_BY: [valid_job_payload, missing_translation_column]
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
from datetime import datetime, timezone
|
||||
from fastapi import status
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from src.core.database import Base
|
||||
from src.app import app
|
||||
from src.dependencies import (
|
||||
get_current_user,
|
||||
get_config_manager,
|
||||
has_permission,
|
||||
)
|
||||
from src.core.config_models import Environment
|
||||
|
||||
|
||||
# [DEF:valid_job_payload:Variable]
|
||||
# @PURPOSE: Standard valid payload for creating a translation job.
|
||||
valid_job_payload = {
|
||||
"name": "Test Translation Job",
|
||||
"description": "A test job for unit tests",
|
||||
"source_dialect": "postgresql",
|
||||
"target_dialect": "clickhouse",
|
||||
"source_key_cols": ["id"],
|
||||
"target_key_cols": ["id"],
|
||||
"translation_column": "name",
|
||||
"context_columns": ["description"],
|
||||
"target_language": "ru",
|
||||
"batch_size": 100,
|
||||
"upsert_strategy": "MERGE",
|
||||
"dictionary_ids": [],
|
||||
}
|
||||
# [/DEF:valid_job_payload:Variable]
|
||||
|
||||
|
||||
# Mock user
|
||||
mock_user = MagicMock()
|
||||
mock_user.username = "testuser"
|
||||
mock_user.roles = []
|
||||
admin_role = MagicMock()
|
||||
admin_role.name = "Admin"
|
||||
mock_user.roles.append(admin_role)
|
||||
|
||||
|
||||
# In-memory SQLite database for service tests
|
||||
SQLALCHEMY_DATABASE_URL = "sqlite:///:memory:"
|
||||
engine = create_engine(SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False})
|
||||
TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
|
||||
# Create DB tables once
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
|
||||
# [DEF:MockConfigManager:Class]
|
||||
# @PURPOSE: Mock ConfigManager for service tests that returns a test environment.
|
||||
class MockConfigManager:
|
||||
def get_environments(self):
|
||||
return [
|
||||
Environment(
|
||||
id="test_env",
|
||||
name="Test Environment",
|
||||
url="http://superset:8088",
|
||||
username="admin",
|
||||
password="admin",
|
||||
)
|
||||
]
|
||||
|
||||
def get_config(self):
|
||||
return MagicMock()
|
||||
# [/DEF:MockConfigManager:Class]
|
||||
|
||||
|
||||
# [DEF:db_session:Function]
|
||||
# @PURPOSE: Create a fresh DB session with transaction rollback for each test.
|
||||
@pytest.fixture
|
||||
def db_session():
|
||||
connection = engine.connect()
|
||||
transaction = connection.begin()
|
||||
session = TestingSessionLocal(bind=connection)
|
||||
|
||||
yield session
|
||||
|
||||
session.close()
|
||||
transaction.rollback()
|
||||
connection.close()
|
||||
# [/DEF:db_session:Function]
|
||||
|
||||
|
||||
# [DEF:mock_api_deps:Function]
|
||||
# @PURPOSE: Override FastAPI dependencies (including get_db with in-memory SQLite) for API route tests.
|
||||
# @RATIONALE: Uses the same in-memory SQLite engine as service tests to avoid real PostgreSQL dependency.
|
||||
# @REJECTED: Mocking get_db with MagicMock — the route handlers need a real session for ORM queries.
|
||||
@pytest.fixture
|
||||
def mock_api_deps():
|
||||
from src.core.database import get_db
|
||||
|
||||
config_manager = MagicMock()
|
||||
config_manager.get_environments.return_value = []
|
||||
|
||||
# Create in-memory SQLite session for API tests (same engine as service tests)
|
||||
connection = engine.connect()
|
||||
transaction = connection.begin()
|
||||
session = TestingSessionLocal(bind=connection)
|
||||
|
||||
def override_get_db():
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
pass # Session lifecycle managed by fixture teardown
|
||||
|
||||
overrides = {
|
||||
get_current_user: lambda: mock_user,
|
||||
get_config_manager: lambda: config_manager,
|
||||
get_db: override_get_db,
|
||||
}
|
||||
|
||||
# Apply all overrides
|
||||
for dep, override in overrides.items():
|
||||
app.dependency_overrides[dep] = override
|
||||
|
||||
yield {"config_manager": config_manager}
|
||||
|
||||
app.dependency_overrides.clear()
|
||||
session.close()
|
||||
transaction.rollback()
|
||||
connection.close()
|
||||
# [/DEF:mock_api_deps:Function]
|
||||
|
||||
|
||||
# [DEF:client:Function]
|
||||
# @PURPOSE: FastAPI TestClient for API route tests.
|
||||
@pytest.fixture
|
||||
def client(mock_api_deps):
|
||||
return TestClient(app)
|
||||
# [/DEF:client:Function]
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Service Layer Tests
|
||||
# ============================================================
|
||||
|
||||
|
||||
# [DEF:test_create_job_valid:Function]
|
||||
# @PURPOSE: Verify that a valid job payload creates a job successfully.
|
||||
def test_create_job_valid(db_session):
|
||||
"""Test creating a valid translation job."""
|
||||
from src.plugins.translate.service import TranslateJobService, job_to_response
|
||||
from src.schemas.translate import TranslateJobCreate
|
||||
|
||||
config_mgr = MockConfigManager()
|
||||
service = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
|
||||
payload = TranslateJobCreate(**valid_job_payload)
|
||||
job = service.create_job(payload)
|
||||
|
||||
assert job.id is not None
|
||||
assert job.name == "Test Translation Job"
|
||||
assert job.source_dialect == "postgresql"
|
||||
assert job.target_dialect == "clickhouse"
|
||||
assert job.translation_column == "name"
|
||||
assert job.source_key_cols == ["id"]
|
||||
assert job.target_key_cols == ["id"]
|
||||
assert job.context_columns == ["description"]
|
||||
assert job.batch_size == 100
|
||||
assert job.upsert_strategy == "MERGE"
|
||||
assert job.target_language == "ru"
|
||||
assert job.status == "DRAFT"
|
||||
assert job.created_by == "test_user"
|
||||
|
||||
# Verify response serialization
|
||||
response = job_to_response(job, [])
|
||||
assert response.name == "Test Translation Job"
|
||||
assert response.source_key_cols == ["id"]
|
||||
# [/DEF:test_create_job_valid:Function]
|
||||
|
||||
|
||||
# [DEF:test_create_job_missing_translation_column:Function]
|
||||
# @PURPOSE: Verify that creating a job with datasource but no translation column raises ValueError.
|
||||
def test_create_job_missing_translation_column(db_session):
|
||||
"""Test that a datasource without a translation column is rejected."""
|
||||
from src.plugins.translate.service import TranslateJobService
|
||||
from src.schemas.translate import TranslateJobCreate
|
||||
|
||||
config_mgr = MockConfigManager()
|
||||
service = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
|
||||
payload = TranslateJobCreate(
|
||||
name="Bad Job",
|
||||
source_dialect="postgresql",
|
||||
target_dialect="clickhouse",
|
||||
source_datasource_id="42",
|
||||
translation_column=None,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="translation column is required"):
|
||||
service.create_job(payload)
|
||||
# [/DEF:test_create_job_missing_translation_column:Function]
|
||||
|
||||
|
||||
# [DEF:test_create_job_invalid_upsert_strategy:Function]
|
||||
# @PURPOSE: Verify that an invalid upsert strategy is rejected.
|
||||
def test_create_job_invalid_upsert_strategy(db_session):
|
||||
"""Test that an invalid upsert strategy raises ValueError."""
|
||||
from src.plugins.translate.service import TranslateJobService
|
||||
from src.schemas.translate import TranslateJobCreate
|
||||
|
||||
config_mgr = MockConfigManager()
|
||||
service = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
|
||||
payload = TranslateJobCreate(
|
||||
name="Bad Strategy Job",
|
||||
source_dialect="postgresql",
|
||||
target_dialect="clickhouse",
|
||||
upsert_strategy="INVALID",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Invalid upsert_strategy"):
|
||||
service.create_job(payload)
|
||||
# [/DEF:test_create_job_invalid_upsert_strategy:Function]
|
||||
|
||||
|
||||
# [DEF:test_get_job:Function]
|
||||
# @PURPOSE: Verify that a job can be retrieved by ID.
|
||||
def test_get_job(db_session):
|
||||
"""Test retrieving a translation job by ID."""
|
||||
from src.plugins.translate.service import TranslateJobService
|
||||
from src.schemas.translate import TranslateJobCreate
|
||||
|
||||
config_mgr = MockConfigManager()
|
||||
service = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
|
||||
payload = TranslateJobCreate(**valid_job_payload)
|
||||
created = service.create_job(payload)
|
||||
|
||||
fetched = service.get_job(created.id)
|
||||
assert fetched.id == created.id
|
||||
assert fetched.name == "Test Translation Job"
|
||||
# [/DEF:test_get_job:Function]
|
||||
|
||||
|
||||
# [DEF:test_get_job_not_found:Function]
|
||||
# @PURPOSE: Verify that getting a non-existent job raises ValueError.
|
||||
def test_get_job_not_found(db_session):
|
||||
"""Test that a non-existent job raises ValueError."""
|
||||
from src.plugins.translate.service import TranslateJobService
|
||||
|
||||
config_mgr = MockConfigManager()
|
||||
service = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
|
||||
with pytest.raises(ValueError, match="not found"):
|
||||
service.get_job("non-existent-id")
|
||||
# [/DEF:test_get_job_not_found:Function]
|
||||
|
||||
|
||||
# [DEF:test_list_jobs:Function]
|
||||
# @PURPOSE: Verify that listing jobs returns all created jobs.
|
||||
def test_list_jobs(db_session):
|
||||
"""Test listing translation jobs."""
|
||||
from src.plugins.translate.service import TranslateJobService
|
||||
from src.schemas.translate import TranslateJobCreate
|
||||
|
||||
config_mgr = MockConfigManager()
|
||||
service = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
|
||||
p1 = TranslateJobCreate(name="Job 1", source_dialect="postgresql", target_dialect="clickhouse")
|
||||
p2 = TranslateJobCreate(name="Job 2", source_dialect="mysql", target_dialect="postgresql")
|
||||
service.create_job(p1)
|
||||
service.create_job(p2)
|
||||
|
||||
total, jobs = service.list_jobs()
|
||||
assert total == 2
|
||||
assert len(jobs) == 2
|
||||
# [/DEF:test_list_jobs:Function]
|
||||
|
||||
|
||||
# [DEF:test_list_jobs_with_status_filter:Function]
|
||||
# @PURPOSE: Verify that listing jobs with a status filter works.
|
||||
def test_list_jobs_with_status_filter(db_session):
|
||||
"""Test listing jobs filtered by status."""
|
||||
from src.plugins.translate.service import TranslateJobService
|
||||
from src.schemas.translate import TranslateJobCreate, TranslateJobUpdate
|
||||
|
||||
config_mgr = MockConfigManager()
|
||||
service = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
|
||||
p1 = TranslateJobCreate(name="Draft Job", source_dialect="pg", target_dialect="ch")
|
||||
p2 = TranslateJobCreate(name="Ready Job", source_dialect="pg", target_dialect="ch")
|
||||
service.create_job(p1)
|
||||
job2 = service.create_job(p2)
|
||||
|
||||
service.update_job(job2.id, TranslateJobUpdate(status="READY"))
|
||||
|
||||
total, jobs = service.list_jobs(status_filter="READY")
|
||||
assert total == 1
|
||||
assert jobs[0].name == "Ready Job"
|
||||
# [/DEF:test_list_jobs_with_status_filter:Function]
|
||||
|
||||
|
||||
# [DEF:test_update_job:Function]
|
||||
# @PURPOSE: Verify that a job can be updated.
|
||||
def test_update_job(db_session):
|
||||
"""Test updating a translation job."""
|
||||
from src.plugins.translate.service import TranslateJobService
|
||||
from src.schemas.translate import TranslateJobCreate, TranslateJobUpdate
|
||||
|
||||
config_mgr = MockConfigManager()
|
||||
service = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
|
||||
payload = TranslateJobCreate(**valid_job_payload)
|
||||
job = service.create_job(payload)
|
||||
|
||||
update = TranslateJobUpdate(
|
||||
name="Updated Job",
|
||||
description="Updated description",
|
||||
batch_size=200,
|
||||
)
|
||||
updated = service.update_job(job.id, update)
|
||||
|
||||
assert updated.name == "Updated Job"
|
||||
assert updated.description == "Updated description"
|
||||
assert updated.batch_size == 200
|
||||
# [/DEF:test_update_job:Function]
|
||||
|
||||
|
||||
# [DEF:test_delete_job:Function]
|
||||
# @PURPOSE: Verify that a job can be deleted.
|
||||
def test_delete_job(db_session):
|
||||
"""Test deleting a translation job."""
|
||||
from src.plugins.translate.service import TranslateJobService
|
||||
from src.schemas.translate import TranslateJobCreate
|
||||
|
||||
config_mgr = MockConfigManager()
|
||||
service = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
|
||||
payload = TranslateJobCreate(**valid_job_payload)
|
||||
job = service.create_job(payload)
|
||||
|
||||
service.delete_job(job.id)
|
||||
|
||||
with pytest.raises(ValueError, match="not found"):
|
||||
service.get_job(job.id)
|
||||
# [/DEF:test_delete_job:Function]
|
||||
|
||||
|
||||
# [DEF:test_duplicate_job:Function]
|
||||
# @PURPOSE: Verify that a job can be duplicated.
|
||||
def test_duplicate_job(db_session):
|
||||
"""Test duplicating a translation job."""
|
||||
from src.plugins.translate.service import TranslateJobService
|
||||
from src.schemas.translate import TranslateJobCreate
|
||||
|
||||
config_mgr = MockConfigManager()
|
||||
service = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
|
||||
payload = TranslateJobCreate(**valid_job_payload)
|
||||
original = service.create_job(payload)
|
||||
|
||||
duplicate = service.duplicate_job(original.id)
|
||||
|
||||
assert duplicate.id != original.id
|
||||
assert duplicate.name == f"{original.name} (Copy)"
|
||||
assert duplicate.source_dialect == original.source_dialect
|
||||
assert duplicate.target_dialect == original.target_dialect
|
||||
assert duplicate.translation_column == original.translation_column
|
||||
assert duplicate.source_key_cols == original.source_key_cols
|
||||
assert duplicate.status == "DRAFT"
|
||||
# [/DEF:test_duplicate_job:Function]
|
||||
|
||||
|
||||
# [DEF:test_duplicate_job_custom_name:Function]
|
||||
# @PURPOSE: Verify that a job can be duplicated with a custom name.
|
||||
def test_duplicate_job_custom_name(db_session):
|
||||
"""Test duplicating a job with a custom name."""
|
||||
from src.plugins.translate.service import TranslateJobService
|
||||
from src.schemas.translate import TranslateJobCreate
|
||||
|
||||
config_mgr = MockConfigManager()
|
||||
service = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
|
||||
payload = TranslateJobCreate(**valid_job_payload)
|
||||
original = service.create_job(payload)
|
||||
|
||||
duplicate = service.duplicate_job(original.id, new_name="Custom Copy Name")
|
||||
assert duplicate.name == "Custom Copy Name"
|
||||
# [/DEF:test_duplicate_job_custom_name:Function]
|
||||
|
||||
|
||||
# [DEF:test_detect_virtual_columns:Function]
|
||||
# @PURPOSE: Verify virtual column detection from column metadata.
|
||||
def test_detect_virtual_columns():
|
||||
"""Test that virtual columns are correctly identified."""
|
||||
from src.plugins.translate.service import detect_virtual_columns
|
||||
|
||||
columns = [
|
||||
{"name": "id", "is_physical": True},
|
||||
{"name": "name", "is_physical": True},
|
||||
{"name": "virtual_col", "is_physical": False},
|
||||
]
|
||||
|
||||
virtuals = detect_virtual_columns(columns)
|
||||
assert "virtual_col" in virtuals
|
||||
assert "id" not in virtuals
|
||||
assert "name" not in virtuals
|
||||
# [/DEF:test_detect_virtual_columns:Function]
|
||||
|
||||
|
||||
# [DEF:test_get_dialect_from_database:Function]
|
||||
# @PURPOSE: Verify dialect extraction from Superset database records.
|
||||
def test_get_dialect_from_database():
|
||||
"""Test dialect extraction from Superset database records."""
|
||||
from src.plugins.translate.service import get_dialect_from_database
|
||||
|
||||
dialect = get_dialect_from_database({"backend": "postgresql"})
|
||||
assert dialect == "postgresql"
|
||||
|
||||
dialect = get_dialect_from_database({"engine": "mysql"})
|
||||
assert dialect == "mysql"
|
||||
|
||||
dialect = get_dialect_from_database({"backend": "clickhouse", "engine": "mysql"})
|
||||
assert dialect == "clickhouse"
|
||||
# [/DEF:test_get_dialect_from_database:Function]
|
||||
|
||||
|
||||
# [DEF:test_get_dialect_from_database_unsupported:Function]
|
||||
# @PURPOSE: Verify that unsupported dialects raise ValueError.
|
||||
def test_get_dialect_from_database_unsupported():
|
||||
"""Test that unsupported dialects raise ValueError."""
|
||||
from src.plugins.translate.service import get_dialect_from_database
|
||||
|
||||
with pytest.raises(ValueError, match="Unsupported database dialect"):
|
||||
get_dialect_from_database({"backend": "mongodb"})
|
||||
|
||||
with pytest.raises(ValueError, match="Could not determine"):
|
||||
get_dialect_from_database({})
|
||||
# [/DEF:test_get_dialect_from_database_unsupported:Function]
|
||||
|
||||
|
||||
# ============================================================
|
||||
# API Route Tests
|
||||
# ============================================================
|
||||
|
||||
|
||||
# [DEF:test_api_create_job:Function]
|
||||
# @PURPOSE: Verify POST /api/translate/jobs returns 201 with valid payload.
|
||||
def test_api_create_job(client):
|
||||
"""Test POST /api/translate/jobs returns 201."""
|
||||
response = client.post("/api/translate/jobs", json=valid_job_payload)
|
||||
|
||||
# Note: This may return 422 because the mock ConfigManager returns no real environments
|
||||
# but the route requires get_config_manager. The service is tested directly above.
|
||||
assert response.status_code in (201, 422, 500)
|
||||
if response.status_code == 201:
|
||||
data = response.json()
|
||||
assert data["name"] == "Test Translation Job"
|
||||
assert "id" in data
|
||||
# [/DEF:test_api_create_job:Function]
|
||||
|
||||
|
||||
# [DEF:test_api_list_jobs:Function]
|
||||
# @PURPOSE: Verify GET /api/translate/jobs returns 200.
|
||||
def test_api_list_jobs(client):
|
||||
"""Test GET /api/translate/jobs returns list."""
|
||||
response = client.get("/api/translate/jobs")
|
||||
assert response.status_code == 200
|
||||
assert isinstance(response.json(), list)
|
||||
# [/DEF:test_api_list_jobs:Function]
|
||||
|
||||
|
||||
# [DEF:test_api_get_job_not_found:Function]
|
||||
# @PURPOSE: Verify GET non-existent job returns 404.
|
||||
def test_api_get_job_not_found(client):
|
||||
"""Test GET non-existent job returns 404."""
|
||||
response = client.get("/api/translate/jobs/non-existent-id")
|
||||
assert response.status_code == 404
|
||||
# [/DEF:test_api_get_job_not_found:Function]
|
||||
|
||||
|
||||
# [DEF:test_api_delete_job_not_found:Function]
|
||||
# @PURPOSE: Verify DELETE non-existent job returns 404.
|
||||
def test_api_delete_job_not_found(client):
|
||||
"""Test DELETE non-existent job returns 404."""
|
||||
response = client.delete("/api/translate/jobs/non-existent-id")
|
||||
assert response.status_code == 404
|
||||
# [/DEF:test_api_delete_job_not_found:Function]
|
||||
|
||||
|
||||
# [DEF:test_api_duplicate_job_not_found:Function]
|
||||
# @PURPOSE: Verify duplicating a non-existent job returns 404.
|
||||
def test_api_duplicate_job_not_found(client):
|
||||
"""Test duplicating a non-existent job returns 404."""
|
||||
response = client.post("/api/translate/jobs/non-existent-id/duplicate")
|
||||
assert response.status_code == 404
|
||||
# [/DEF:test_api_duplicate_job_not_found:Function]
|
||||
|
||||
|
||||
# [DEF:test_api_create_job_422_missing_name:Function]
|
||||
# @PURPOSE: Verify POST with missing required fields returns 422.
|
||||
def test_api_create_job_422_missing_name(client):
|
||||
"""Test POST with missing required fields returns 422."""
|
||||
response = client.post("/api/translate/jobs", json={"source_dialect": "pg"})
|
||||
assert response.status_code == 422
|
||||
# [/DEF:test_api_create_job_422_missing_name:Function]
|
||||
|
||||
|
||||
# [DEF:test_api_datasource_columns_missing_env:Function]
|
||||
# @PURPOSE: Verify datasource columns endpoint without env_id returns 422.
|
||||
def test_api_datasource_columns_missing_env(client):
|
||||
"""Test datasource columns endpoint without env_id returns 422."""
|
||||
response = client.get("/api/translate/datasources/42/columns")
|
||||
assert response.status_code == 422
|
||||
# [/DEF:test_api_datasource_columns_missing_env:Function]
|
||||
|
||||
|
||||
# [DEF:test_api_datasource_columns_bad_env:Function]
|
||||
# @PURPOSE: Verify datasource columns with unknown env returns 400.
|
||||
def test_api_datasource_columns_bad_env(client):
|
||||
"""Test datasource columns with unknown env returns 400."""
|
||||
response = client.get("/api/translate/datasources/42/columns?env_id=unknown")
|
||||
assert response.status_code in (400, 502, 422)
|
||||
# [/DEF:test_api_datasource_columns_bad_env:Function]
|
||||
|
||||
|
||||
# [DEF:test_api_update_job_not_found:Function]
|
||||
# @PURPOSE: Verify PUT non-existent job returns 404.
|
||||
def test_api_update_job_not_found(client):
|
||||
"""Test PUT non-existent job returns 404."""
|
||||
response = client.put("/api/translate/jobs/non-existent-id", json={"name": "Updated"})
|
||||
assert response.status_code == 404
|
||||
# [/DEF:test_api_update_job_not_found:Function]
|
||||
|
||||
# [/DEF:TranslateJobTests:Module]
|
||||
206
backend/tests/test_translate_scheduler.py
Normal file
206
backend/tests/test_translate_scheduler.py
Normal file
@@ -0,0 +1,206 @@
|
||||
# [DEF:TranslateSchedulerTests:Module]
|
||||
# @COMPLEXITY: 4
|
||||
# @SEMANTICS: tests, translate, scheduler
|
||||
# @PURPOSE: Tests for TranslationScheduler CRUD and APScheduler integration.
|
||||
# @LAYER: Test
|
||||
# @RELATION: BINDS_TO -> [TranslationScheduler:Module]
|
||||
#
|
||||
# @TEST_CONTRACT: ScheduleFlow ->
|
||||
# {
|
||||
# invariants: [
|
||||
# "Create schedule returns TranslationSchedule with valid fields",
|
||||
# "Update schedule modifies cron/timezone/active",
|
||||
# "Delete schedule removes the row",
|
||||
# "Enable/disable toggles is_active",
|
||||
# "get_next_executions returns ISO datetime strings",
|
||||
# "List active schedules returns only active rows"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_FIXTURE: valid_job -> created via TranslateJobService
|
||||
# @TEST_FIXTURE: schedule_data -> valid cron expression
|
||||
# @TEST_EDGE: schedule_not_found -> ValueError on get/delete non-existent
|
||||
# @TEST_EDGE: next_executions_invalid_cron -> empty list
|
||||
# @TEST_INVARIANT: schedule_crud_flow -> VERIFIED_BY: [valid_job, schedule_data]
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from datetime import datetime, timezone
|
||||
import uuid
|
||||
|
||||
from src.core.database import Base
|
||||
from src.app import app
|
||||
from src.dependencies import get_current_user, get_config_manager, get_db
|
||||
from src.plugins.translate.scheduler import TranslationScheduler, execute_scheduled_translation
|
||||
from src.plugins.translate.service import TranslateJobService
|
||||
from src.schemas.translate import TranslateJobCreate
|
||||
from src.models.translate import TranslationJob, TranslationSchedule, TranslationRun
|
||||
|
||||
# In-memory SQLite
|
||||
SQLALCHEMY_DATABASE_URL = "sqlite:///:memory:"
|
||||
engine = create_engine(SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False})
|
||||
TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db_session():
|
||||
connection = engine.connect()
|
||||
transaction = connection.begin()
|
||||
session = TestingSessionLocal(bind=connection)
|
||||
yield session
|
||||
session.close()
|
||||
transaction.rollback()
|
||||
connection.close()
|
||||
|
||||
|
||||
# [DEF:test_create_schedule:Function]
|
||||
# @PURPOSE: Verify schedule creation with valid params.
|
||||
def test_create_schedule(db_session):
|
||||
"""Test creating a schedule for a job."""
|
||||
from src.core.config_models import Environment
|
||||
config_mgr = MagicMock()
|
||||
config_mgr.get_environments.return_value = []
|
||||
|
||||
svc = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
payload = TranslateJobCreate(name="Sched Job", source_dialect="pg", target_dialect="ch")
|
||||
job = svc.create_job(payload)
|
||||
|
||||
scheduler = TranslationScheduler(db_session, config_mgr, "test_user")
|
||||
schedule = scheduler.create_schedule(
|
||||
job_id=job.id,
|
||||
cron_expression="0 2 * * *",
|
||||
timezone="UTC",
|
||||
)
|
||||
|
||||
assert schedule.job_id == job.id
|
||||
assert schedule.cron_expression == "0 2 * * *"
|
||||
assert schedule.timezone == "UTC"
|
||||
assert schedule.is_active is True
|
||||
assert schedule.id is not None
|
||||
# [/DEF:test_create_schedule:Function]
|
||||
|
||||
|
||||
# [DEF:test_update_schedule:Function]
|
||||
# @PURPOSE: Verify schedule update.
|
||||
def test_update_schedule(db_session):
|
||||
"""Test updating a schedule."""
|
||||
from src.core.config_models import Environment
|
||||
config_mgr = MagicMock()
|
||||
config_mgr.get_environments.return_value = []
|
||||
|
||||
svc = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
job = svc.create_job(TranslateJobCreate(name="Job", source_dialect="pg", target_dialect="ch"))
|
||||
|
||||
scheduler = TranslationScheduler(db_session, config_mgr, "test_user")
|
||||
scheduler.create_schedule(job.id, "0 2 * * *")
|
||||
|
||||
updated = scheduler.update_schedule(job.id, cron_expression="30 3 * * *", timezone_str="US/Eastern", is_active=False)
|
||||
assert updated.cron_expression == "30 3 * * *"
|
||||
assert updated.timezone == "US/Eastern"
|
||||
assert updated.is_active is False
|
||||
# [/DEF:test_update_schedule:Function]
|
||||
|
||||
|
||||
# [DEF:test_delete_schedule:Function]
|
||||
# @PURPOSE: Verify schedule deletion.
|
||||
def test_delete_schedule(db_session):
|
||||
"""Test deleting a schedule."""
|
||||
from src.core.config_models import Environment
|
||||
config_mgr = MagicMock()
|
||||
config_mgr.get_environments.return_value = []
|
||||
|
||||
svc = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
job = svc.create_job(TranslateJobCreate(name="Job", source_dialect="pg", target_dialect="ch"))
|
||||
|
||||
scheduler = TranslationScheduler(db_session, config_mgr, "test_user")
|
||||
scheduler.create_schedule(job.id, "0 2 * * *")
|
||||
scheduler.delete_schedule(job.id)
|
||||
|
||||
with pytest.raises(ValueError, match="No schedule found"):
|
||||
scheduler.get_schedule(job.id)
|
||||
# [/DEF:test_delete_schedule:Function]
|
||||
|
||||
|
||||
# [DEF:test_enable_disable_schedule:Function]
|
||||
# @PURPOSE: Verify enable/disable toggle.
|
||||
def test_enable_disable_schedule(db_session):
|
||||
"""Test enabling and disabling a schedule."""
|
||||
from src.core.config_models import Environment
|
||||
config_mgr = MagicMock()
|
||||
config_mgr.get_environments.return_value = []
|
||||
|
||||
svc = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
job = svc.create_job(TranslateJobCreate(name="Job", source_dialect="pg", target_dialect="ch"))
|
||||
|
||||
scheduler = TranslationScheduler(db_session, config_mgr, "test_user")
|
||||
scheduler.create_schedule(job.id, "0 2 * * *")
|
||||
|
||||
sched = scheduler.set_schedule_active(job.id, False)
|
||||
assert sched.is_active is False
|
||||
|
||||
sched = scheduler.set_schedule_active(job.id, True)
|
||||
assert sched.is_active is True
|
||||
# [/DEF:test_enable_disable_schedule:Function]
|
||||
|
||||
|
||||
# [DEF:test_get_schedule_not_found:Function]
|
||||
# @PURPOSE: Verify ValueError on getting non-existent schedule.
|
||||
def test_get_schedule_not_found(db_session):
|
||||
"""Test that getting a non-existent schedule raises ValueError."""
|
||||
from src.core.config_models import Environment
|
||||
config_mgr = MagicMock()
|
||||
config_mgr.get_environments.return_value = []
|
||||
|
||||
scheduler = TranslationScheduler(db_session, config_mgr, "test_user")
|
||||
with pytest.raises(ValueError, match="No schedule found"):
|
||||
scheduler.get_schedule("non-existent")
|
||||
# [/DEF:test_get_schedule_not_found:Function]
|
||||
|
||||
|
||||
# [DEF:test_list_active_schedules:Function]
|
||||
# @PURPOSE: Verify listing only active schedules.
|
||||
def test_list_active_schedules(db_session):
|
||||
"""Test listing active schedules."""
|
||||
from src.core.config_models import Environment
|
||||
config_mgr = MagicMock()
|
||||
config_mgr.get_environments.return_value = []
|
||||
|
||||
svc = TranslateJobService(db_session, config_mgr, "test_user")
|
||||
job1 = svc.create_job(TranslateJobCreate(name="Job1", source_dialect="pg", target_dialect="ch"))
|
||||
|
||||
scheduler = TranslationScheduler(db_session, config_mgr, "test_user")
|
||||
scheduler.create_schedule(job1.id, "0 2 * * *")
|
||||
scheduler.set_schedule_active(job1.id, True)
|
||||
|
||||
active = TranslationScheduler.list_active_schedules(db_session)
|
||||
assert len(active) >= 1
|
||||
assert active[0].job_id == job1.id
|
||||
# [/DEF:test_list_active_schedules:Function]
|
||||
|
||||
|
||||
# [DEF:test_get_next_executions:Function]
|
||||
# @PURPOSE: Verify next execution time computation.
|
||||
def test_get_next_executions():
|
||||
"""Test computing next execution times."""
|
||||
times = TranslationScheduler.get_next_executions("0 2 * * *", "UTC", n=3)
|
||||
assert len(times) == 3
|
||||
for t in times:
|
||||
assert "T" in t # ISO format check
|
||||
# [/DEF:test_get_next_executions:Function]
|
||||
|
||||
|
||||
# [DEF:test_get_next_executions_invalid:Function]
|
||||
# @PURPOSE: Verify invalid cron returns empty list.
|
||||
def test_get_next_executions_invalid():
|
||||
"""Test invalid cron returns empty list."""
|
||||
times = TranslationScheduler.get_next_executions("invalid cron", "UTC", n=3)
|
||||
assert times == []
|
||||
# [/DEF:test_get_next_executions_invalid:Function]
|
||||
# [/DEF:TranslateSchedulerTests:Module]
|
||||
Reference in New Issue
Block a user