feat: implement LLM table translation service with searchable datasource dropdown, i18n, sidebar, and RBAC
- Add backend plugin modules: preview, executor, orchestrator, events, sql_generator, superset_executor, dictionary, scheduler, metrics - Add API routes for translate jobs, runs, dictionaries, preview, schedule, metrics, corrections - Add ORM models (12 tables) and Pydantic schemas (15 DTOs) - Register translate router in app.py with RBAC permission guards - Add frontend pages: job list, job config, dictionary list/editor, history - Add 7 reusable Svelte 5 components: Preview, RunProgress, RunResult, TermCorrection, BulkCorrection, ScheduleConfig, MetricsDashboard - Add searchable datasource dropdown with Superset API integration - Add i18n (en/ru) with ~300 keys across common, config, jobs, dictionaries, preview namespaces - Add Translation sidebar category with Jobs/Dictionaries/History sub-items - Hide health monitor error toast via suppressToast API option - Add 69 backend tests and 44 frontend test files - Fix: SupersetClient env resolution (string -> Environment object) - Fix: Dataset detail API returns proper database dict - Fix: Database dialect extraction fallback when metadata incomplete
This commit is contained in:
292
backend/src/models/translate.py
Normal file
292
backend/src/models/translate.py
Normal file
@@ -0,0 +1,292 @@
|
||||
# [DEF:TranslateModels:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @SEMANTICS: translate, models, sqlalchemy, persistence
|
||||
# @PURPOSE: SQLAlchemy ORM models for LLM-based SQL/dashboard translation across dialects.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: INHERITS_FROM -> MappingModels:Base
|
||||
|
||||
from sqlalchemy import (
|
||||
Column, String, Boolean, DateTime, JSON, Text,
|
||||
ForeignKey, Integer, UniqueConstraint, Index
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSON as PG_JSON
|
||||
from datetime import datetime, timezone
|
||||
import uuid
|
||||
import hashlib
|
||||
from .mapping import Base
|
||||
|
||||
|
||||
def generate_uuid():
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
# [DEF:TranslationJob:Class]
|
||||
# @PURPOSE: A translation job representing a multi-dialect conversion task with column mappings, LLM config, and dictionary attachments.
|
||||
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
|
||||
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity.
|
||||
class TranslationJob(Base):
|
||||
__tablename__ = "translation_jobs"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
name = Column(String, nullable=False)
|
||||
description = Column(Text, nullable=True)
|
||||
source_dialect = Column(String, nullable=False)
|
||||
target_dialect = Column(String, nullable=False)
|
||||
database_dialect = Column(String, nullable=True, comment="Detected dialect from Superset connection at save time")
|
||||
status = Column(String, nullable=False, default="DRAFT") # DRAFT, READY, RUNNING, COMPLETED, FAILED, CANCELLED
|
||||
|
||||
# Datasource & target table configuration
|
||||
source_datasource_id = Column(String, nullable=True, comment="Superset datasource ID")
|
||||
source_table = Column(String, nullable=True, comment="Source table name resolved from datasource")
|
||||
target_schema = Column(String, nullable=True, comment="Target table schema")
|
||||
target_table = Column(String, nullable=True, comment="Target table name")
|
||||
|
||||
# Column mapping
|
||||
source_key_cols = Column(JSON, nullable=True, comment="Source key column names for composite key")
|
||||
target_key_cols = Column(JSON, nullable=True, comment="Target key column names for composite key")
|
||||
translation_column = Column(String, nullable=True, comment="The column whose values will be translated")
|
||||
context_columns = Column(JSON, nullable=True, comment="Context column names included in LLM prompt")
|
||||
|
||||
# LLM & processing settings
|
||||
target_language = Column(String, nullable=True, comment="Target language code (e.g. en, ru)")
|
||||
provider_id = Column(String, nullable=True, comment="LLM provider ID")
|
||||
batch_size = Column(Integer, nullable=False, default=50, comment="Records per batch")
|
||||
upsert_strategy = Column(String, nullable=False, default="MERGE", comment="MERGE, INSERT, UPDATE")
|
||||
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
|
||||
# [/DEF:TranslationJob:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationRun:Class]
|
||||
# @PURPOSE: Represents a single execution of a translation job, capturing status, timing, and Superset execution metadata.
|
||||
class TranslationRun(Base):
|
||||
__tablename__ = "translation_runs"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
||||
status = Column(String, nullable=False, default="PENDING") # PENDING, RUNNING, COMPLETED, FAILED, CANCELLED
|
||||
trigger_type = Column(String, nullable=True, comment="manual, scheduled, retry, baseline_expired")
|
||||
started_at = Column(DateTime, nullable=True)
|
||||
completed_at = Column(DateTime, nullable=True)
|
||||
error_message = Column(Text, nullable=True)
|
||||
total_records = Column(Integer, default=0)
|
||||
successful_records = Column(Integer, default=0)
|
||||
failed_records = Column(Integer, default=0)
|
||||
skipped_records = Column(Integer, default=0)
|
||||
insert_status = Column(String, nullable=True, comment="Status of the Superset insert/update operation")
|
||||
superset_execution_id = Column(String, nullable=True, comment="Superset execution/task ID")
|
||||
superset_execution_log = Column(JSON, nullable=True, comment="Superset execution log output")
|
||||
config_snapshot = Column(JSON, nullable=True, comment="Snapshot of job config at run creation time")
|
||||
key_hash = Column(String, nullable=True, comment="Hash of source key fields for dedup")
|
||||
config_hash = Column(String, nullable=True, comment="Hash of translation configuration state")
|
||||
dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at run time")
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
# [/DEF:TranslationRun:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationBatch:Class]
|
||||
# @PURPOSE: Groups translation records within a run into manageable batches with timing and record counts.
|
||||
class TranslationBatch(Base):
|
||||
__tablename__ = "translation_batches"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=False, index=True)
|
||||
batch_index = Column(Integer, nullable=False)
|
||||
status = Column(String, nullable=False, default="PENDING")
|
||||
total_records = Column(Integer, default=0)
|
||||
successful_records = Column(Integer, default=0)
|
||||
failed_records = Column(Integer, default=0)
|
||||
started_at = Column(DateTime, nullable=True)
|
||||
completed_at = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
# [/DEF:TranslationBatch:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationRecord:Class]
|
||||
# @PURPOSE: Individual translation result for a single SQL statement or dashboard element, tracking source, target, and error state.
|
||||
class TranslationRecord(Base):
|
||||
__tablename__ = "translation_records"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
batch_id = Column(String, ForeignKey("translation_batches.id"), nullable=False, index=True)
|
||||
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=False, index=True)
|
||||
source_sql = Column(Text, nullable=True)
|
||||
target_sql = Column(Text, nullable=True)
|
||||
source_object_type = Column(String, nullable=True) # query, dashboard, chart, dataset
|
||||
source_object_id = Column(String, nullable=True)
|
||||
source_object_name = Column(String, nullable=True)
|
||||
status = Column(String, nullable=False, default="PENDING") # PENDING, SUCCESS, FAILED, SKIPPED
|
||||
error_message = Column(Text, nullable=True)
|
||||
token_count_input = Column(Integer, nullable=True)
|
||||
token_count_output = Column(Integer, nullable=True)
|
||||
translation_duration_ms = Column(Integer, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_translation_records_run_status", "run_id", "status"),
|
||||
)
|
||||
# [/DEF:TranslationRecord:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationEvent:Class]
|
||||
# @PURPOSE: Audit/event log for translation operations, with optional run_id for context.
|
||||
class TranslationEvent(Base):
|
||||
__tablename__ = "translation_events"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
||||
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=True, index=True)
|
||||
event_type = Column(String, nullable=False) # JOB_CREATED, JOB_STARTED, JOB_COMPLETED, JOB_FAILED, etc.
|
||||
event_data = Column(JSON, nullable=True)
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
# [/DEF:TranslationEvent:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationPreviewSession:Class]
|
||||
# @PURPOSE: A preview session allowing users to review proposed translations before applying them.
|
||||
class TranslationPreviewSession(Base):
|
||||
__tablename__ = "translation_preview_sessions"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
||||
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=True)
|
||||
status = Column(String, nullable=False, default="ACTIVE") # ACTIVE, APPLIED, DISCARDED, EXPIRED
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
expires_at = Column(DateTime, nullable=True)
|
||||
# [/DEF:TranslationPreviewSession:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationPreviewRecord:Class]
|
||||
# @PURPOSE: Individual preview entry within a preview session, showing original and translated content side by side.
|
||||
class TranslationPreviewRecord(Base):
|
||||
__tablename__ = "translation_preview_records"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
session_id = Column(String, ForeignKey("translation_preview_sessions.id"), nullable=False, index=True)
|
||||
source_sql = Column(Text, nullable=True)
|
||||
target_sql = Column(Text, nullable=True)
|
||||
source_object_type = Column(String, nullable=True)
|
||||
source_object_id = Column(String, nullable=True)
|
||||
source_object_name = Column(String, nullable=True)
|
||||
status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED
|
||||
feedback = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
# [/DEF:TranslationPreviewRecord:Class]
|
||||
|
||||
|
||||
# [DEF:TerminologyDictionary:Class]
|
||||
# @PURPOSE: A named collection of terminology mappings used during translation to ensure consistent term translation.
|
||||
class TerminologyDictionary(Base):
|
||||
__tablename__ = "terminology_dictionaries"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
name = Column(String, nullable=False)
|
||||
description = Column(Text, nullable=True)
|
||||
source_dialect = Column(String, nullable=False)
|
||||
target_dialect = Column(String, nullable=False)
|
||||
is_active = Column(Boolean, default=True)
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
|
||||
# [/DEF:TerminologyDictionary:Class]
|
||||
|
||||
|
||||
# [DEF:DictionaryEntry:Class]
|
||||
# @PURPOSE: A single term mapping entry with case-insensitive unique constraint on source_term_normalized and origin tracking.
|
||||
class DictionaryEntry(Base):
|
||||
__tablename__ = "dictionary_entries"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
dictionary_id = Column(String, ForeignKey("terminology_dictionaries.id"), nullable=False, index=True)
|
||||
source_term = Column(String, nullable=False)
|
||||
source_term_normalized = Column(String, nullable=False)
|
||||
target_term = Column(String, nullable=False)
|
||||
context_notes = Column(Text, nullable=True)
|
||||
origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated")
|
||||
origin_row_key = Column(String, nullable=True, comment="Row key within the run that triggered this correction")
|
||||
origin_user_id = Column(String, nullable=True, comment="User who submitted the correction")
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"dictionary_id", "source_term_normalized",
|
||||
name="uq_dictionary_entry_term"
|
||||
),
|
||||
)
|
||||
# [/DEF:DictionaryEntry:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationSchedule:Class]
|
||||
# @PURPOSE: Defines a cron-based schedule for recurring translation jobs.
|
||||
class TranslationSchedule(Base):
|
||||
__tablename__ = "translation_schedules"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
||||
cron_expression = Column(String, nullable=False)
|
||||
timezone = Column(String, nullable=False, default="UTC")
|
||||
is_active = Column(Boolean, default=True)
|
||||
last_run_at = Column(DateTime, nullable=True)
|
||||
next_run_at = Column(DateTime, nullable=True)
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
|
||||
# [/DEF:TranslationSchedule:Class]
|
||||
|
||||
|
||||
# [DEF:TranslationJobDictionary:Class]
|
||||
# @PURPOSE: Many-to-many association between translation jobs and terminology dictionaries.
|
||||
class TranslationJobDictionary(Base):
|
||||
__tablename__ = "translation_job_dictionaries"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
||||
dictionary_id = Column(String, ForeignKey("terminology_dictionaries.id"), nullable=False, index=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"job_id", "dictionary_id",
|
||||
name="uq_job_dictionary"
|
||||
),
|
||||
)
|
||||
# [/DEF:TranslationJobDictionary:Class]
|
||||
|
||||
|
||||
# [DEF:MetricSnapshot:Class]
|
||||
# @PURPOSE: Captures translation performance metrics at a point in time with config/content/dictionary hash keys for aggregation.
|
||||
class MetricSnapshot(Base):
|
||||
__tablename__ = "translation_metric_snapshots"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
||||
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=True, index=True)
|
||||
key_hash = Column(String, nullable=False, comment="Hash of dimension key fields for aggregation")
|
||||
config_hash = Column(String, nullable=True, comment="Hash of translation configuration state")
|
||||
dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at capture time")
|
||||
covers_events_before = Column(DateTime, nullable=True, comment="Indicates snapshot covers events before this timestamp")
|
||||
total_jobs = Column(Integer, default=0)
|
||||
total_runs = Column(Integer, default=0)
|
||||
total_records = Column(Integer, default=0)
|
||||
successful_records = Column(Integer, default=0)
|
||||
failed_records = Column(Integer, default=0)
|
||||
skipped_records = Column(Integer, default=0)
|
||||
avg_duration_ms = Column(Integer, nullable=True)
|
||||
p50_duration_ms = Column(Integer, nullable=True)
|
||||
p95_duration_ms = Column(Integer, nullable=True)
|
||||
p99_duration_ms = Column(Integer, nullable=True)
|
||||
snapshot_date = Column(DateTime, nullable=False, default=lambda: datetime.now(timezone.utc))
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_metric_snapshots_job_date", "job_id", "snapshot_date"),
|
||||
)
|
||||
# [/DEF:MetricSnapshot:Class]
|
||||
|
||||
# [/DEF:TranslateModels:Module]
|
||||
Reference in New Issue
Block a user