busya
9f995f22ae
- Convert all 84 contracts from legacy [DEF:] to #region/#endregion syntax - Fix complexity tiers: 14 modules re-tiered (6 C4 route modules, 7 C4→C5 plugin services) - Remove forbidden tags: @RATIONALE/@REJECTED stripped from C1–C4 contracts - Add required tags: @PRE/@POST/@SIDE_EFFECT on C4, @RELATION on C3, @DATA_CONTRACT/@INVARIANT on C5 - Add belief runtime markers (reason/reflect/explore) to 7 service.py functions - Fix @LAYER: route files → UI, plugins → Domain, superset_executor → Infra - Fix pre-existing test mock_service fixture in test_orchestrator.py - 196/196 translation tests pass, zero regressions
284 lines
14 KiB
Python
284 lines
14 KiB
Python
# #region TranslateModels [C:2] [TYPE Module]
|
|
# @BRIEF SQLAlchemy ORM models for LLM-based SQL/dashboard translation across dialects.
|
|
# @LAYER Domain
|
|
|
|
from sqlalchemy import (
|
|
Column, String, Boolean, DateTime, JSON, Text,
|
|
ForeignKey, Integer, UniqueConstraint, Index
|
|
)
|
|
from sqlalchemy.dialects.postgresql import JSON as PG_JSON
|
|
from datetime import datetime, timezone
|
|
import uuid
|
|
import hashlib
|
|
from .mapping import Base
|
|
|
|
|
|
# #region generate_uuid [C:1] [TYPE Function]
|
|
def generate_uuid():
|
|
return str(uuid.uuid4())
|
|
# #endregion generate_uuid
|
|
|
|
|
|
# #region TranslationJob [C:1] [TYPE Class]
|
|
class TranslationJob(Base):
|
|
__tablename__ = "translation_jobs"
|
|
|
|
id = Column(String, primary_key=True, default=generate_uuid)
|
|
name = Column(String, nullable=False)
|
|
description = Column(Text, nullable=True)
|
|
source_dialect = Column(String, nullable=False)
|
|
target_dialect = Column(String, nullable=False)
|
|
database_dialect = Column(String, nullable=True, comment="Detected dialect from Superset connection at save time")
|
|
status = Column(String, nullable=False, default="DRAFT") # DRAFT, READY, RUNNING, COMPLETED, FAILED, CANCELLED
|
|
|
|
# Datasource & target table configuration
|
|
source_datasource_id = Column(String, nullable=True, comment="Superset datasource ID")
|
|
source_table = Column(String, nullable=True, comment="Source table name resolved from datasource")
|
|
target_schema = Column(String, nullable=True, comment="Target table schema")
|
|
target_table = Column(String, nullable=True, comment="Target table name")
|
|
|
|
# Column mapping
|
|
source_key_cols = Column(JSON, nullable=True, comment="Source key column names for composite key")
|
|
target_key_cols = Column(JSON, nullable=True, comment="Target key column names for composite key")
|
|
translation_column = Column(String, nullable=True, comment="The column whose values will be translated")
|
|
context_columns = Column(JSON, nullable=True, comment="Context column names included in LLM prompt")
|
|
|
|
# LLM & processing settings
|
|
target_language = Column(String, nullable=True, comment="Target language code (e.g. en, ru)")
|
|
provider_id = Column(String, nullable=True, comment="LLM provider ID")
|
|
batch_size = Column(Integer, nullable=False, default=50, comment="Records per batch")
|
|
upsert_strategy = Column(String, nullable=False, default="MERGE", comment="MERGE, INSERT, UPDATE")
|
|
|
|
# Environment association
|
|
environment_id = Column(String, nullable=True, comment="Superset environment ID for datasource access")
|
|
target_database_id = Column(String, nullable=True, comment="Superset database ID for INSERT via SQL Lab")
|
|
|
|
created_by = Column(String, nullable=True)
|
|
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
|
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
|
|
# #endregion TranslationJob
|
|
|
|
|
|
# #region TranslationRun [C:1] [TYPE Class]
|
|
class TranslationRun(Base):
|
|
__tablename__ = "translation_runs"
|
|
|
|
id = Column(String, primary_key=True, default=generate_uuid)
|
|
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
|
status = Column(String, nullable=False, default="PENDING") # PENDING, RUNNING, COMPLETED, FAILED, CANCELLED
|
|
trigger_type = Column(String, nullable=True, comment="manual, scheduled, retry, baseline_expired")
|
|
started_at = Column(DateTime, nullable=True)
|
|
completed_at = Column(DateTime, nullable=True)
|
|
error_message = Column(Text, nullable=True)
|
|
total_records = Column(Integer, default=0)
|
|
successful_records = Column(Integer, default=0)
|
|
failed_records = Column(Integer, default=0)
|
|
skipped_records = Column(Integer, default=0)
|
|
insert_status = Column(String, nullable=True, comment="Status of the Superset insert/update operation")
|
|
superset_execution_id = Column(String, nullable=True, comment="Superset execution/task ID")
|
|
superset_execution_log = Column(JSON, nullable=True, comment="Superset execution log output")
|
|
config_snapshot = Column(JSON, nullable=True, comment="Snapshot of job config at run creation time")
|
|
key_hash = Column(String, nullable=True, comment="Hash of source key fields for dedup")
|
|
config_hash = Column(String, nullable=True, comment="Hash of translation configuration state")
|
|
dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at run time")
|
|
created_by = Column(String, nullable=True)
|
|
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
|
# #endregion TranslationRun
|
|
|
|
|
|
# #region TranslationBatch [C:1] [TYPE Class]
|
|
class TranslationBatch(Base):
|
|
__tablename__ = "translation_batches"
|
|
|
|
id = Column(String, primary_key=True, default=generate_uuid)
|
|
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=False, index=True)
|
|
batch_index = Column(Integer, nullable=False)
|
|
status = Column(String, nullable=False, default="PENDING")
|
|
total_records = Column(Integer, default=0)
|
|
successful_records = Column(Integer, default=0)
|
|
failed_records = Column(Integer, default=0)
|
|
started_at = Column(DateTime, nullable=True)
|
|
completed_at = Column(DateTime, nullable=True)
|
|
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
|
# #endregion TranslationBatch
|
|
|
|
|
|
# #region TranslationRecord [C:1] [TYPE Class]
|
|
class TranslationRecord(Base):
|
|
__tablename__ = "translation_records"
|
|
|
|
id = Column(String, primary_key=True, default=generate_uuid)
|
|
batch_id = Column(String, ForeignKey("translation_batches.id"), nullable=False, index=True)
|
|
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=False, index=True)
|
|
source_sql = Column(Text, nullable=True)
|
|
target_sql = Column(Text, nullable=True)
|
|
source_object_type = Column(String, nullable=True) # query, dashboard, chart, dataset
|
|
source_object_id = Column(String, nullable=True)
|
|
source_object_name = Column(String, nullable=True)
|
|
status = Column(String, nullable=False, default="PENDING") # PENDING, SUCCESS, FAILED, SKIPPED
|
|
error_message = Column(Text, nullable=True)
|
|
token_count_input = Column(Integer, nullable=True)
|
|
token_count_output = Column(Integer, nullable=True)
|
|
translation_duration_ms = Column(Integer, nullable=True)
|
|
source_data = Column(JSON, nullable=True, comment="Snapshot of source row key/context column values for INSERT phase")
|
|
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
|
|
|
__table_args__ = (
|
|
Index("ix_translation_records_run_status", "run_id", "status"),
|
|
)
|
|
# #endregion TranslationRecord
|
|
|
|
|
|
# #region TranslationEvent [C:1] [TYPE Class]
|
|
class TranslationEvent(Base):
|
|
__tablename__ = "translation_events"
|
|
|
|
id = Column(String, primary_key=True, default=generate_uuid)
|
|
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
|
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=True, index=True)
|
|
event_type = Column(String, nullable=False) # JOB_CREATED, JOB_STARTED, JOB_COMPLETED, JOB_FAILED, etc.
|
|
event_data = Column(JSON, nullable=True)
|
|
created_by = Column(String, nullable=True)
|
|
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
|
# #endregion TranslationEvent
|
|
|
|
|
|
# #region TranslationPreviewSession [C:1] [TYPE Class]
|
|
class TranslationPreviewSession(Base):
|
|
__tablename__ = "translation_preview_sessions"
|
|
|
|
id = Column(String, primary_key=True, default=generate_uuid)
|
|
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
|
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=True)
|
|
status = Column(String, nullable=False, default="ACTIVE") # ACTIVE, APPLIED, DISCARDED, EXPIRED
|
|
created_by = Column(String, nullable=True)
|
|
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
|
expires_at = Column(DateTime, nullable=True)
|
|
# #endregion TranslationPreviewSession
|
|
|
|
|
|
# #region TranslationPreviewRecord [C:1] [TYPE Class]
|
|
class TranslationPreviewRecord(Base):
|
|
__tablename__ = "translation_preview_records"
|
|
|
|
id = Column(String, primary_key=True, default=generate_uuid)
|
|
session_id = Column(String, ForeignKey("translation_preview_sessions.id"), nullable=False, index=True)
|
|
source_sql = Column(Text, nullable=True)
|
|
target_sql = Column(Text, nullable=True)
|
|
source_object_type = Column(String, nullable=True)
|
|
source_object_id = Column(String, nullable=True)
|
|
source_object_name = Column(String, nullable=True)
|
|
status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED
|
|
feedback = Column(Text, nullable=True)
|
|
source_data = Column(JSON, nullable=True, comment="Snapshot of source row values for key/context columns")
|
|
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
|
# #endregion TranslationPreviewRecord
|
|
|
|
|
|
# #region TerminologyDictionary [C:1] [TYPE Class]
|
|
class TerminologyDictionary(Base):
|
|
__tablename__ = "terminology_dictionaries"
|
|
|
|
id = Column(String, primary_key=True, default=generate_uuid)
|
|
name = Column(String, nullable=False)
|
|
description = Column(Text, nullable=True)
|
|
source_dialect = Column(String, nullable=False)
|
|
target_dialect = Column(String, nullable=False)
|
|
is_active = Column(Boolean, default=True)
|
|
created_by = Column(String, nullable=True)
|
|
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
|
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
|
|
# #endregion TerminologyDictionary
|
|
|
|
|
|
# #region DictionaryEntry [C:1] [TYPE Class]
|
|
class DictionaryEntry(Base):
|
|
__tablename__ = "dictionary_entries"
|
|
|
|
id = Column(String, primary_key=True, default=generate_uuid)
|
|
dictionary_id = Column(String, ForeignKey("terminology_dictionaries.id"), nullable=False, index=True)
|
|
source_term = Column(String, nullable=False)
|
|
source_term_normalized = Column(String, nullable=False)
|
|
target_term = Column(String, nullable=False)
|
|
context_notes = Column(Text, nullable=True)
|
|
origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated")
|
|
origin_row_key = Column(String, nullable=True, comment="Row key within the run that triggered this correction")
|
|
origin_user_id = Column(String, nullable=True, comment="User who submitted the correction")
|
|
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
|
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
|
|
|
|
__table_args__ = (
|
|
UniqueConstraint(
|
|
"dictionary_id", "source_term_normalized",
|
|
name="uq_dictionary_entry_term"
|
|
),
|
|
)
|
|
# #endregion DictionaryEntry
|
|
|
|
|
|
# #region TranslationSchedule [C:1] [TYPE Class]
|
|
class TranslationSchedule(Base):
|
|
__tablename__ = "translation_schedules"
|
|
|
|
id = Column(String, primary_key=True, default=generate_uuid)
|
|
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
|
cron_expression = Column(String, nullable=False)
|
|
timezone = Column(String, nullable=False, default="UTC")
|
|
is_active = Column(Boolean, default=True)
|
|
last_run_at = Column(DateTime, nullable=True)
|
|
next_run_at = Column(DateTime, nullable=True)
|
|
created_by = Column(String, nullable=True)
|
|
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
|
updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
|
|
# #endregion TranslationSchedule
|
|
|
|
|
|
# #region TranslationJobDictionary [C:1] [TYPE Class]
|
|
class TranslationJobDictionary(Base):
|
|
__tablename__ = "translation_job_dictionaries"
|
|
|
|
id = Column(String, primary_key=True, default=generate_uuid)
|
|
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
|
dictionary_id = Column(String, ForeignKey("terminology_dictionaries.id"), nullable=False, index=True)
|
|
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
|
|
|
__table_args__ = (
|
|
UniqueConstraint(
|
|
"job_id", "dictionary_id",
|
|
name="uq_job_dictionary"
|
|
),
|
|
)
|
|
# #endregion TranslationJobDictionary
|
|
|
|
|
|
# #region MetricSnapshot [C:1] [TYPE Class]
|
|
class MetricSnapshot(Base):
|
|
__tablename__ = "translation_metric_snapshots"
|
|
|
|
id = Column(String, primary_key=True, default=generate_uuid)
|
|
job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True)
|
|
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=True, index=True)
|
|
key_hash = Column(String, nullable=False, comment="Hash of dimension key fields for aggregation")
|
|
config_hash = Column(String, nullable=True, comment="Hash of translation configuration state")
|
|
dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at capture time")
|
|
covers_events_before = Column(DateTime, nullable=True, comment="Indicates snapshot covers events before this timestamp")
|
|
total_jobs = Column(Integer, default=0)
|
|
total_runs = Column(Integer, default=0)
|
|
total_records = Column(Integer, default=0)
|
|
successful_records = Column(Integer, default=0)
|
|
failed_records = Column(Integer, default=0)
|
|
skipped_records = Column(Integer, default=0)
|
|
avg_duration_ms = Column(Integer, nullable=True)
|
|
p50_duration_ms = Column(Integer, nullable=True)
|
|
p95_duration_ms = Column(Integer, nullable=True)
|
|
p99_duration_ms = Column(Integer, nullable=True)
|
|
snapshot_date = Column(DateTime, nullable=False, default=lambda: datetime.now(timezone.utc))
|
|
created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
|
|
|
__table_args__ = (
|
|
Index("ix_metric_snapshots_job_date", "job_id", "snapshot_date"),
|
|
)
|
|
# #endregion MetricSnapshot
|
|
|
|
# #endregion TranslateModels
|