diff --git a/backend/alembic/versions/8dd0a93af539_drop_deprecated_source_language_column_.py b/backend/alembic/versions/8dd0a93af539_drop_deprecated_source_language_column_.py new file mode 100644 index 00000000..dae44ad7 --- /dev/null +++ b/backend/alembic/versions/8dd0a93af539_drop_deprecated_source_language_column_.py @@ -0,0 +1,39 @@ +"""Drop deprecated source_language column from translation_jobs + +Revision ID: 8dd0a93af539 +Revises: 543d43d752b8 +Create Date: 2026-05-14 23:43:32.216941 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import sqlite + +# revision identifiers, used by Alembic. +revision: str = '8dd0a93af539' +down_revision: Union[str, Sequence[str], None] = '543d43d752b8' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # Use DROP COLUMN IF EXISTS for PostgreSQL safety (column may not exist on all environments) + # SQLite does not support IF EXISTS, but op.drop_column works when column exists + bind = op.get_bind() + if bind.engine.name == "sqlite": + # SQLite: drop only if column exists + inspector = sa.inspect(bind) + columns = [c["name"] for c in inspector.get_columns("translation_jobs")] + if "source_language" in columns: + op.drop_column("translation_jobs", "source_language") + else: + # PostgreSQL and others: use IF EXISTS + op.execute("ALTER TABLE translation_jobs DROP COLUMN IF EXISTS source_language") + + +def downgrade() -> None: + """Downgrade schema.""" + op.add_column("translation_jobs", sa.Column("source_language", sa.VARCHAR(), nullable=True)) diff --git a/backend/src/api/routes/mappings.py b/backend/src/api/routes/mappings.py index 2af2a357..ebef6708 100644 --- a/backend/src/api/routes/mappings.py +++ b/backend/src/api/routes/mappings.py @@ -6,7 +6,7 @@ # @RELATION DEPENDS_ON -> [DatabaseModule] # @RELATION DEPENDS_ON -> [mapping_service] # -# @INVARIANT: Mappings are persisted in the SQLite database. +# from fastapi import APIRouter, Depends, HTTPException diff --git a/backend/src/models/translate.py b/backend/src/models/translate.py index f7bf377b..519f4d6a 100644 --- a/backend/src/models/translate.py +++ b/backend/src/models/translate.py @@ -47,8 +47,7 @@ class TranslationJob(Base): # LLM & processing settings # @DEPRECATED — use target_languages target_language = Column(String, nullable=True, comment="Target language code (e.g. en, ru) [DEPRECATED: use target_languages]") - # @DEPRECATED — Auto-detected per row by LLM, kept as fallback hint - source_language = Column(String, nullable=True, comment="Fallback source language hint [DEPRECATED: auto-detected per row]") + # source_language removed — deprecated, auto-detected per row by LLM; use TranslationLanguage.source_language_detected instead target_languages = Column(JSON, nullable=True, comment="List of BCP-47 target language codes (multi-language support)") provider_id = Column(String, nullable=True, comment="LLM provider ID") batch_size = Column(Integer, nullable=False, default=50, comment="Records per batch") diff --git a/backend/src/schemas/translate.py b/backend/src/schemas/translate.py index 7102f2c1..7bac1ae1 100644 --- a/backend/src/schemas/translate.py +++ b/backend/src/schemas/translate.py @@ -106,7 +106,7 @@ class TranslateJobResponse(BaseModel): target_column: str | None = None context_columns: list[str] | None = None target_language: str | None = None - source_language: str | None = None + # source_language removed — deprecated, per-row auto-detected target_languages: list[str] | None = None provider_id: str | None = None batch_size: int = 50 diff --git a/backend/src/scripts/migrate_sqlite_to_postgres.py b/backend/src/scripts/migrate_sqlite_to_postgres.py deleted file mode 100644 index d0cce640..00000000 --- a/backend/src/scripts/migrate_sqlite_to_postgres.py +++ /dev/null @@ -1,401 +0,0 @@ -# #region MigrateSqliteToPostgresScript [C:3] [TYPE Module] [SEMANTICS sqlalchemy, sqlite, storage, task, history, config] -# -# @BRIEF Migrates legacy config and task history from SQLite/file storage to PostgreSQL. -# @LAYER: Scripts -# @RELATION READS_FROM -> backend/tasks.db -# @RELATION READS_FROM -> backend/config.json -# @RELATION WRITES_TO -> postgresql.task_records -# @RELATION WRITES_TO -> postgresql.task_logs -# @RELATION WRITES_TO -> postgresql.app_configurations -# -# @INVARIANT: Script is idempotent for task_records and app_configurations. - -import argparse -import json -import os -import sqlite3 -from collections.abc import Iterable -from pathlib import Path -from typing import Any - -from sqlalchemy import create_engine, text -from sqlalchemy.exc import SQLAlchemyError - -from src.core.cot_logger import seed_trace_id -from src.core.logger import belief_scope, logger - -# #region Constants [TYPE Section] -DEFAULT_TARGET_URL = os.getenv( - "DATABASE_URL", - os.getenv( - "POSTGRES_URL", - "postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools", - ), -) -# #endregion Constants - - -# #region _json_load_if_needed [C:3] [TYPE Function] -# @BRIEF Parses JSON-like values from SQLite TEXT/JSON columns to Python objects. -# @PRE: value is scalar JSON/text/list/dict or None. -# @POST: Returns normalized Python object or original scalar value. -def _json_load_if_needed(value: Any) -> Any: - with belief_scope("_json_load_if_needed"): - if value is None: - return None - if isinstance(value, (dict, list)): - return value - if isinstance(value, str): - raw = value.strip() - if not raw: - return None - if raw[0] in "{[": - try: - return json.loads(raw) - except json.JSONDecodeError: - return value - return value - - -# #endregion _json_load_if_needed - - -# #region _find_legacy_config_path [TYPE Function] -# @BRIEF Resolves the existing legacy config.json path from candidates. -def _find_legacy_config_path(explicit_path: str | None) -> Path | None: - with belief_scope("_find_legacy_config_path"): - if explicit_path: - p = Path(explicit_path) - return p if p.exists() else None - - candidates = [ - Path("backend/config.json"), - Path("config.json"), - ] - for candidate in candidates: - if candidate.exists(): - return candidate - return None - - -# #endregion _find_legacy_config_path - - -# #region _connect_sqlite [TYPE Function] -# @BRIEF Opens a SQLite connection with row factory. -def _connect_sqlite(path: Path) -> sqlite3.Connection: - with belief_scope("_connect_sqlite"): - conn = sqlite3.connect(str(path)) - conn.row_factory = sqlite3.Row - return conn - - -# #endregion _connect_sqlite - - -# #region _ensure_target_schema [TYPE Function] -# @BRIEF Ensures required PostgreSQL tables exist before migration. -def _ensure_target_schema(engine) -> None: - with belief_scope("_ensure_target_schema"): - stmts: Iterable[str] = ( - """ - CREATE TABLE IF NOT EXISTS app_configurations ( - id TEXT PRIMARY KEY, - payload JSONB NOT NULL, - updated_at TIMESTAMPTZ DEFAULT NOW() - ) - """, - """ - CREATE TABLE IF NOT EXISTS task_records ( - id TEXT PRIMARY KEY, - type TEXT NOT NULL, - status TEXT NOT NULL, - environment_id TEXT NULL, - started_at TIMESTAMPTZ NULL, - finished_at TIMESTAMPTZ NULL, - logs JSONB NULL, - error TEXT NULL, - result JSONB NULL, - created_at TIMESTAMPTZ DEFAULT NOW(), - params JSONB NULL - ) - """, - """ - CREATE TABLE IF NOT EXISTS task_logs ( - id INTEGER PRIMARY KEY, - task_id TEXT NOT NULL, - timestamp TIMESTAMPTZ NOT NULL, - level VARCHAR(16) NOT NULL, - source VARCHAR(64) NOT NULL DEFAULT 'system', - message TEXT NOT NULL, - metadata_json TEXT NULL, - CONSTRAINT fk_task_logs_task - FOREIGN KEY(task_id) - REFERENCES task_records(id) - ON DELETE CASCADE - ) - """, - "CREATE INDEX IF NOT EXISTS ix_task_logs_task_timestamp ON task_logs (task_id, timestamp)", - "CREATE INDEX IF NOT EXISTS ix_task_logs_task_level ON task_logs (task_id, level)", - "CREATE INDEX IF NOT EXISTS ix_task_logs_task_source ON task_logs (task_id, source)", - """ - DO $$ - BEGIN - IF EXISTS ( - SELECT 1 FROM pg_class WHERE relkind = 'S' AND relname = 'task_logs_id_seq' - ) THEN - PERFORM 1; - ELSE - CREATE SEQUENCE task_logs_id_seq OWNED BY task_logs.id; - END IF; - END $$; - """, - "ALTER TABLE task_logs ALTER COLUMN id SET DEFAULT nextval('task_logs_id_seq')", - ) - with engine.begin() as conn: - for stmt in stmts: - conn.execute(text(stmt)) - - -# #endregion _ensure_target_schema - - -# #region _migrate_config [TYPE Function] -# @BRIEF Migrates legacy config.json into app_configurations(global). -def _migrate_config(engine, legacy_config_path: Path | None) -> int: - with belief_scope("_migrate_config"): - if legacy_config_path is None: - logger.reason( - "No legacy config.json found, skipping", - extra={"src": "_migrate_config"}, - ) - return 0 - - payload = json.loads(legacy_config_path.read_text(encoding="utf-8")) - with engine.begin() as conn: - conn.execute( - text( - """ - INSERT INTO app_configurations (id, payload, updated_at) - VALUES ('global', CAST(:payload AS JSONB), NOW()) - ON CONFLICT (id) - DO UPDATE SET payload = EXCLUDED.payload, updated_at = NOW() - """ - ), - {"payload": json.dumps(payload, ensure_ascii=True)}, - ) - logger.info( - "[_migrate_config][Coherence:OK] Config migrated from %s", - legacy_config_path, - ) - return 1 - - -# #endregion _migrate_config - - -# #region _migrate_tasks_and_logs [TYPE Function] -# @BRIEF Migrates task_records and task_logs from SQLite into PostgreSQL. -def _migrate_tasks_and_logs(engine, sqlite_conn: sqlite3.Connection) -> dict[str, int]: - with belief_scope("_migrate_tasks_and_logs"): - stats = { - "task_records_total": 0, - "task_records_inserted": 0, - "task_logs_total": 0, - "task_logs_inserted": 0, - } - - rows = sqlite_conn.execute( - """ - SELECT id, type, status, environment_id, started_at, finished_at, logs, error, result, created_at, params - FROM task_records - ORDER BY created_at ASC - """ - ).fetchall() - stats["task_records_total"] = len(rows) - - with engine.begin() as conn: - existing_env_ids = { - row[0] - for row in conn.execute(text("SELECT id FROM environments")).fetchall() - } - for row in rows: - params_obj = _json_load_if_needed(row["params"]) - result_obj = _json_load_if_needed(row["result"]) - logs_obj = _json_load_if_needed(row["logs"]) - environment_id = row["environment_id"] - if environment_id and environment_id not in existing_env_ids: - # Legacy task may reference environments that were not migrated; keep task row and drop FK value. - environment_id = None - - res = conn.execute( - text( - """ - INSERT INTO task_records ( - id, type, status, environment_id, started_at, finished_at, - logs, error, result, created_at, params - ) VALUES ( - :id, :type, :status, :environment_id, :started_at, :finished_at, - CAST(:logs AS JSONB), :error, CAST(:result AS JSONB), :created_at, CAST(:params AS JSONB) - ) - ON CONFLICT (id) DO NOTHING - """ - ), - { - "id": row["id"], - "type": row["type"], - "status": row["status"], - "environment_id": environment_id, - "started_at": row["started_at"], - "finished_at": row["finished_at"], - "logs": json.dumps(logs_obj, ensure_ascii=True) - if logs_obj is not None - else None, - "error": row["error"], - "result": json.dumps(result_obj, ensure_ascii=True) - if result_obj is not None - else None, - "created_at": row["created_at"], - "params": json.dumps(params_obj, ensure_ascii=True) - if params_obj is not None - else None, - }, - ) - if res.rowcount and res.rowcount > 0: - stats["task_records_inserted"] += int(res.rowcount) - - log_rows = sqlite_conn.execute( - """ - SELECT id, task_id, timestamp, level, source, message, metadata_json - FROM task_logs - ORDER BY id ASC - """ - ).fetchall() - stats["task_logs_total"] = len(log_rows) - - with engine.begin() as conn: - for row in log_rows: - # Preserve original IDs to keep migration idempotent. - res = conn.execute( - text( - """ - INSERT INTO task_logs (id, task_id, timestamp, level, source, message, metadata_json) - VALUES (:id, :task_id, :timestamp, :level, :source, :message, :metadata_json) - ON CONFLICT (id) DO NOTHING - """ - ), - { - "id": row["id"], - "task_id": row["task_id"], - "timestamp": row["timestamp"], - "level": row["level"], - "source": row["source"] or "system", - "message": row["message"], - "metadata_json": row["metadata_json"], - }, - ) - if res.rowcount and res.rowcount > 0: - stats["task_logs_inserted"] += int(res.rowcount) - - # Ensure sequence is aligned after explicit id inserts. - conn.execute( - text( - """ - SELECT setval( - 'task_logs_id_seq', - COALESCE((SELECT MAX(id) FROM task_logs), 1), - TRUE - ) - """ - ) - ) - - logger.info( - "[_migrate_tasks_and_logs][Coherence:OK] task_records=%s/%s task_logs=%s/%s", - stats["task_records_inserted"], - stats["task_records_total"], - stats["task_logs_inserted"], - stats["task_logs_total"], - ) - return stats - - -# #endregion _migrate_tasks_and_logs - - -# #region run_migration [TYPE Function] -# @BRIEF Orchestrates migration from SQLite/file to PostgreSQL. -def run_migration( - sqlite_path: Path, target_url: str, legacy_config_path: Path | None -) -> dict[str, int]: - seed_trace_id() - with belief_scope("run_migration"): - logger.reason( - f"sqlite={sqlite_path} target={target_url}", - extra={"src": "run_migration"}, - ) - if not sqlite_path.exists(): - raise FileNotFoundError(f"SQLite source not found: {sqlite_path}") - - sqlite_conn = _connect_sqlite(sqlite_path) - engine = create_engine(target_url, pool_pre_ping=True) - try: - _ensure_target_schema(engine) - config_upserted = _migrate_config(engine, legacy_config_path) - stats = _migrate_tasks_and_logs(engine, sqlite_conn) - stats["config_upserted"] = config_upserted - return stats - finally: - sqlite_conn.close() - - -# #endregion run_migration - - -# #region main [TYPE Function] -# @BRIEF CLI entrypoint. -def main() -> int: - with belief_scope("main"): - parser = argparse.ArgumentParser( - description="Migrate legacy config.json and task logs from SQLite to PostgreSQL.", - ) - parser.add_argument( - "--sqlite-path", - default="backend/tasks.db", - help="Path to source SQLite DB with task_records/task_logs (default: backend/tasks.db).", - ) - parser.add_argument( - "--target-url", - default=DEFAULT_TARGET_URL, - help="Target PostgreSQL SQLAlchemy URL (default: DATABASE_URL/POSTGRES_URL env).", - ) - parser.add_argument( - "--config-path", - default=None, - help="Optional path to legacy config.json (auto-detected when omitted).", - ) - - args = parser.parse_args() - - sqlite_path = Path(args.sqlite_path) - legacy_config_path = _find_legacy_config_path(args.config_path) - try: - stats = run_migration( - sqlite_path=sqlite_path, - target_url=args.target_url, - legacy_config_path=legacy_config_path, - ) - print("Migration completed.") - print(json.dumps(stats, indent=2)) - return 0 - except (SQLAlchemyError, OSError, sqlite3.Error, ValueError) as e: - logger.error("[main][Coherence:Failed] Migration failed: %s", e) - print(f"Migration failed: {e}") - return 1 - - -if __name__ == "__main__": - raise SystemExit(main()) -# #endregion main - -# #endregion MigrateSqliteToPostgresScript