busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

refactor: unify initialization and reset migration baseline

Browse Source
This commit is contained in:
busya
2026-08-25 11:42:31 +03:00
parent cfb13d9a69
commit 5bc1b62bd0
168 changed files with 1242 additions and 7907 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.env.enterprise-clean.example
View File

@@ -39,7 +39,7 @@ ENCRYPTION_KEY=change-me-generate-a-fernet-key
# Сервисный токен для agent→backend вызовов.
# Сгенерировать: python3 -c "import secrets; print('svc-' + secrets.token_urlsafe(24))"
SERVICE_JWT=agent-service-secret
SERVICE_JWT=replace-with-random-service-secret
# ======================================================================
# Сертификаты (корпоративные)

5
.env.example
View File

@@ -14,6 +14,9 @@ POSTGRES_DB=ss_tools
POSTGRES_USER=postgres
POSTGRES_PASSWORD=postgres
# ── Application storage ─────────────────────────────────────────────────
STORAGE_ROOT_PATH=/app/storage
# ── Порты хоста ────────────────────────────────────────────────────────
BACKEND_HOST_PORT=8101
FRONTEND_HOST_PORT=8100
@@ -29,7 +32,7 @@ AUTH_SECRET_KEY=change-me-to-a-random-secret-32-chars-min
ENCRYPTION_KEY=D40dpvWPZxKd41jeaTHtEs2R7nwMVLxbkMRLjAICRls=
# Сервисный токен для agent→backend вызовов.
# Сгенерировать: python3 -c "import secrets; print('svc-' + secrets.token_urlsafe(24))"
SERVICE_JWT=agent-service-secret
SERVICE_JWT=replace-with-random-service-secret
# JWT audience / issuer (опционально)
# JWT_AUDIENCE=superset-tools-api
# JWT_ISSUER=superset-tools

4
AGENTS.md
View File

@@ -30,14 +30,14 @@
1. Проверяет `python3 >= 3.9` и `npm`.
2. Загружает `backend/.env` до database preflight.
3. Берет URL БД в порядке `DATABASE_URL`, `POSTGRES_URL`, затем локальный PostgreSQL:
3. Берет URL БД из `DATABASE_URL` либо использует локальный PostgreSQL:
`postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools`.
4. Для недоступного локального PostgreSQL пытается выполнить `docker compose up -d db` и
ждет доступность порта до 20 секунд.
5. Генерирует и сохраняет отсутствующие или некорректные `ENCRYPTION_KEY` и
`AUTH_SECRET_KEY` в `backend/.env`.
6. Перед запуском Uvicorn выполняет `alembic upgrade head`; существующую схему без
`alembic_version` скрипт пытается `alembic stamp head`.
`alembic_version` схема создаётся единственным `alembic upgrade head`.
7. Загружает `backend/.env` также в backend и agent. `SERVICE_JWT`, если не задан, получает
случайное значение на текущий запуск; при запуске сервисов в отдельных терминалах его
нужно задать одинаковым явно.

2
INSTALL.md
View File

@@ -184,7 +184,7 @@ python src/scripts/create_admin.py --username admin --password '<temporary-secre
| Категория | Переменные | Описание |
|---|---|---|
| **Security** | `AUTH_SECRET_KEY`, `ENCRYPTION_KEY`, `SERVICE_JWT` | JWT-подпись, шифрование данных, сервисный токен agent→backend |
| **Database** | `DATABASE_URL`, `AUTH_DATABASE_URL`, `TASKS_DATABASE_URL` | PostgreSQL подключения |
| **Database** | `DATABASE_URL` | Единственное PostgreSQL подключение |
| **Admin bootstrap** | `INITIAL_ADMIN_CREATE`, `INITIAL_ADMIN_USERNAME`, `INITIAL_ADMIN_PASSWORD`, `INITIAL_ADMIN_EMAIL` | Автосоздание admin при первом запуске |
| **LLM** | `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `LLM_BASE_URL`, `LLM_MODEL` | Провайдеры и модели |
| **Agent** | `AGENT_PORT`, `ENABLE_EMBEDDING_ROUTER` | Порт Gradio, семантический роутинг |

2
agent/src/ss_tools/agent/_jwt_decoder.py
View File

@@ -1,7 +1,7 @@
# agent/src/ss_tools/agent/_jwt_decoder.py
# #region AgentChat.JwtDecoder [C:1] [TYPE Module] [SEMANTICS agent-chat,jwt,decode]
# @BRIEF Lightweight JWT decode for agent — uses AUTH_SECRET_KEY env var, avoids
# pulling backend jwt module which requires AUTH_DATABASE_URL and ORM deps.
# pulling backend jwt module which requires the application DB and ORM deps.
# @RATIONALE The agent only needs stateless JWT validation (exp, sub, signature).
# @INVARIANT AUTH_SECRET_KEY is the ONLY accepted JWT signing key.
# @REJECTED Importing backend jwt was rejected — it drags in SQLAlchemy models.

3
backend/.env.example
View File

@@ -19,8 +19,7 @@ ENCRYPTION_KEY=change-me-generate-a-fernet-key
# ── База данных (ОБЯЗАТЕЛЬНО) ──────────────────────────────────────────
DATABASE_URL=postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools
AUTH_DATABASE_URL=postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools
TASKS_DATABASE_URL=postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools
STORAGE_ROOT_PATH=/home/busya/dev/ss-tools-storage
# ── Admin bootstrap ────────────────────────────────────────────────────
# INITIAL_ADMIN_CREATE=true

22
backend/alembic/env.py
View File

@@ -46,7 +46,8 @@ if config.config_file_name is not None:
# add your model's MetaData object here
# for 'autogenerate' support
# Import ALL model modules so their tables are registered in Base.metadata
# Import every ORM module so the baseline metadata is complete.
import src.models # noqa: F401, E402
from src.models import ( # noqa: F401, E402
agent,
agent_run,
@@ -54,6 +55,7 @@ from src.models import ( # noqa: F401, E402
assistant,
auth,
clean_release,
config as models_config,
dashboard,
dashboard_release,
deployment,
@@ -63,17 +65,22 @@ from src.models import ( # noqa: F401, E402
llm,
load_testing,
maintenance,
mapping,
profile,
report,
scenario_approval,
scenario_artifact,
scenario_automation,
scenario_checkpoint,
scenario_investigation,
scenario_registry,
scenario_run,
scenario_worker,
storage,
task,
translate,
verification_run,
)
# Import config models with explicit alias to avoid name collision with alembic config
import src.models.config as models_config # noqa: F401, E402
from src.models.mapping import Base # noqa: E402
target_metadata = Base.metadata
@@ -115,6 +122,13 @@ def run_migrations_online() -> None:
and associate a connection with the context.
"""
external_connection = config.attributes.get("connection")
if external_connection is not None:
context.configure(connection=external_connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
return
connectable = engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",

65
backend/alembic/versions/0001_baseline.py Normal file
View File

@@ -0,0 +1,65 @@
"""Single baseline schema for the unified application database."""
import sqlalchemy as sa
from alembic import op
from src.models import (
agent, # noqa: F401
agent_run, # noqa: F401
api_key, # noqa: F401
assistant, # noqa: F401
auth, # noqa: F401
clean_release, # noqa: F401
config, # noqa: F401
dashboard, # noqa: F401
dashboard_release, # noqa: F401
deployment, # noqa: F401
filter_state, # noqa: F401
git, # noqa: F401
lineage, # noqa: F401
llm, # noqa: F401
load_testing, # noqa: F401
maintenance, # noqa: F401
mapping, # noqa: F401
profile, # noqa: F401
report, # noqa: F401
scenario_approval, # noqa: F401
scenario_artifact, # noqa: F401
scenario_automation, # noqa: F401
scenario_checkpoint, # noqa: F401
scenario_investigation, # noqa: F401
scenario_registry, # noqa: F401
scenario_run, # noqa: F401
scenario_worker, # noqa: F401
storage, # noqa: F401
task, # noqa: F401
translate, # noqa: F401
verification_run, # noqa: F401
)
from src.models.mapping import Base
revision = "0001_baseline"
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
if bind.dialect.name == "postgresql":
bind.execute(sa.text("SELECT pg_advisory_xact_lock(481920260824)"))
existing_tables = set(sa.inspect(bind).get_table_names())
allowed_metadata_tables = {"alembic_version"}
unexpected_tables = existing_tables - allowed_metadata_tables
if unexpected_tables:
tables = ", ".join(sorted(unexpected_tables))
raise RuntimeError(
"Refusing to stamp an incomplete existing schema. "
"RESET_DATABASE_SCHEMA must be used for the supported legacy revision; "
f"found tables: {tables}"
)
Base.metadata.create_all(bind=bind)
def downgrade() -> None:
raise RuntimeError("0001_baseline is destructive and cannot be downgraded")

37
backend/alembic/versions/015281bd7759_merge_session_activity_and_verification_.py
View File

@@ -1,37 +0,0 @@
# #region Alembic.MergeSessionAndVerificationHeads [C:1] [TYPE Module] [SEMANTICS alembic,migration,merge]
# @ingroup Alembic
# @BRIEF Merge two parallel heads from o1p2q3r4s5t6: session-activity logical sessions
# (a1b2c3d4e5f7) and verification_runs.dashboard_id (p2q3r4s5t6u7). Restores a single
# alembic head so `upgrade head` applies both branches (fixes "multiple head revisions").
# @RELATION DEPENDS_ON -> [Models.VerificationRun]
# @RELATION DEPENDS_ON -> [Models.Auth.SessionActivity]
# @RATIONALE 037 T081 added p2q3r4s5t6u7 while the session-activity change a1b2c3d4e5f7 landed
# concurrently; both branched from o1p2q3r4s5t6. A no-op merge revision collapses
# them into one head so runtime migrations stop failing.
# @REJECTED Editing down_revision of either existing migration was rejected — it would rewrite
# already-applied history on live deployments; a merge revision is the additive fix.
"""merge session activity and verification dashboard_id heads
Revision ID: 015281bd7759
Revises: a1b2c3d4e5f7, p2q3r4s5t6u7
Create Date: 2026-08-07 16:31:55.210245
"""
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = '015281bd7759'
down_revision: str | Sequence[str] | None = ('a1b2c3d4e5f7', 'p2q3r4s5t6u7')
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Upgrade schema (no-op merge)."""
pass
def downgrade() -> None:
"""Downgrade schema (no-op merge)."""
pass
# #endregion Alembic.MergeSessionAndVerificationHeads

65
backend/alembic/versions/20260603_add_token_limits_to_llm_providers.py
View File

@@ -1,65 +0,0 @@
"""Add context_window and max_output_tokens to llm_providers
@RATIONALE llm_providers is created at runtime by init_db() →
Base.metadata.create_all(), not by any Alembic migration. On a fresh
database, the table doesn't exist when migrations run. Guard with
_table_exists() — create_all() will create the table with the model's
columns (which already include context_window and max_output_tokens).
Revision ID: a1b2c3d4e5f6
Revises: f1a2b3c4d5e6
Create Date: 2026-06-03
Add token window configuration to LLM provider records:
- context_window: total context window in tokens (nullable)
- max_output_tokens: max output tokens limit (nullable)
Both NULL = use PROVIDER_DEFAULTS fallback from model name.
"""
from typing import Sequence, Union
from alembic import op
from sqlalchemy import inspect
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "a1b2c3d4e5f6"
down_revision: Union[str, None] = "f1a2b3c4d5e6"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def _table_exists(table_name: str) -> bool:
conn = op.get_bind()
inspector = inspect(conn)
return inspector.has_table(table_name)
def upgrade() -> None:
if not _table_exists("llm_providers"):
return
op.add_column(
"llm_providers",
sa.Column(
"context_window",
sa.Integer(),
nullable=True,
comment="Total context window in tokens. NULL = auto-detect from model name",
),
)
op.add_column(
"llm_providers",
sa.Column(
"max_output_tokens",
sa.Integer(),
nullable=True,
comment="Max output tokens limit. NULL = auto-detect from model name",
),
)
def downgrade() -> None:
if not _table_exists("llm_providers"):
return
op.drop_column("llm_providers", "max_output_tokens")
op.drop_column("llm_providers", "context_window")

38
backend/alembic/versions/20260604_add_is_regex_to_dictionary_entries.py
View File

@@ -1,38 +0,0 @@
"""Add is_regex column to dictionary_entries
Revision ID: b2c3d4e5f6a7
Revises: a1b2c3d4e5f6
Create Date: 2026-06-04
Add regex pattern support to terminology dictionary entries.
When is_regex=True, source_term is treated as a regex pattern
instead of a literal substring for matching and enforcement.
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "b2c3d4e5f6a7"
down_revision: Union[str, None] = "a1b2c3d4e5f6"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"dictionary_entries",
sa.Column(
"is_regex",
sa.Boolean(),
nullable=False,
server_default=sa.text("false"),
comment="Whether source_term is a regex pattern",
),
)
def downgrade() -> None:
op.drop_column("dictionary_entries", "is_regex")

67
backend/alembic/versions/2a7b8c9d0e1f_add_target_languages_to_translation_jobs.py
View File

@@ -1,67 +0,0 @@
"""Add target_languages column to translation_jobs (multi-language support)
Revision ID: 2a7b8c9d0e1f
Revises: 8dd0a93af539
Create Date: 2026-05-14 23:55:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "2a7b8c9d0e1f"
down_revision: str | Sequence[str] | None = "8dd0a93af539"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Add target_languages column to translation_jobs.
Production PostgreSQL may be missing this column if the initial
migration (ed310b33f02c) was applied from a version that predated
the multi-language support feature. This migration safely adds it
with IF NOT EXISTS on PostgreSQL, and handles SQLite explicitly.
"""
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = [c["name"] for c in inspector.get_columns("translation_jobs")]
if "target_languages" in columns:
# Column already exists — nothing to do
return
if bind.engine.name == "sqlite":
op.add_column(
"translation_jobs",
sa.Column(
"target_languages",
sa.JSON(),
nullable=True,
comment="List of BCP-47 target language codes (multi-language support)",
),
)
else:
# PostgreSQL and others: use IF NOT EXISTS for safety
op.execute(
"ALTER TABLE translation_jobs "
"ADD COLUMN IF NOT EXISTS target_languages JSON "
"DEFAULT NULL"
)
def downgrade() -> None:
"""Drop target_languages column from translation_jobs."""
bind = op.get_bind()
if bind.engine.name == "sqlite":
inspector = sa.inspect(bind)
columns = [c["name"] for c in inspector.get_columns("translation_jobs")]
if "target_languages" in columns:
op.drop_column("translation_jobs", "target_languages")
else:
op.execute(
"ALTER TABLE translation_jobs DROP COLUMN IF EXISTS target_languages"
)

81
backend/alembic/versions/2df63b7ce038_add_missing_columns_policy_id_provider_.py
View File

@@ -1,81 +0,0 @@
"""add missing columns policy_id provider_id is_multimodal
@ADR [LOG-003] Catch-up migration for columns added in inserted revisions.
@RATIONALE Migrations 9f8e7d6c5b4a (is_multimodal), a7b1c2d3e4f5 (provider_id),
and b1c2d3e4f5a6 (policy_id) were inserted into the chain before the
existing head 86c7b1d6a710. Databases already at 86c7b1d6a710 skip them
because Alembic sees no gap between current and target revision.
This migration adds the same columns with 86c7b1d6a710 as parent, so
it runs on both fresh and pre-stamped databases.
Revision ID: 2df63b7ce038
Revises: 86c7b1d6a710
Create Date: 2026-05-27 11:17:26.789634
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = '2df63b7ce038'
down_revision: str | Sequence[str] | None = '86c7b1d6a710'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _table_exists(table: str) -> bool:
"""Check if a table exists in the current database connection."""
conn = op.get_bind()
inspector = inspect(conn)
return inspector.has_table(table)
def upgrade() -> None:
"""Upgrade schema."""
conn = op.get_bind()
# Add is_multimodal to llm_providers (from 9f8e7d6c5b4a)
# llm_providers is created by create_all() at runtime — skip if not exist
if _table_exists("llm_providers") and not _column_exists(conn, "llm_providers", "is_multimodal"):
op.add_column("llm_providers",
sa.Column("is_multimodal", sa.Boolean(), nullable=False, server_default="false")
)
op.alter_column("llm_providers", "is_multimodal", server_default=None)
# Add provider_id to validation_policies (from a7b1c2d3e4f5)
# validation_policies is created by create_all() at runtime — skip if not exist
if _table_exists("validation_policies") and not _column_exists(conn, "validation_policies", "provider_id"):
op.add_column("validation_policies",
sa.Column("provider_id", sa.String(), nullable=True)
)
# Add policy_id to llm_validation_results (from b1c2d3e4f5a6)
# llm_validation_results is created by create_all() at runtime — skip if not exist
if _table_exists("llm_validation_results") and not _column_exists(conn, "llm_validation_results", "policy_id"):
op.add_column("llm_validation_results",
sa.Column("policy_id", sa.String(), nullable=True, index=True)
)
def _column_exists(conn, table: str, column: str) -> bool:
"""Check if a column exists in the given table."""
from sqlalchemy import text
result = conn.execute(
text(
"SELECT 1 FROM information_schema.columns "
"WHERE table_name = :table AND column_name = :column"
),
{"table": table, "column": column},
)
return result.scalar() is not None
def downgrade() -> None:
"""Downgrade schema."""
op.drop_column("llm_validation_results", "policy_id")
op.drop_column("validation_policies", "provider_id")
op.drop_column("llm_providers", "is_multimodal")

28
backend/alembic/versions/351afb8f961a_merge_is_regex_and_composite_index_heads.py
View File

@@ -1,28 +0,0 @@
"""merge is_regex and composite index heads
Revision ID: 351afb8f961a
Revises: b2c3d4e5f6a7, c7d8e9f0a1b2
Create Date: 2026-06-04 14:50:00.004262
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '351afb8f961a'
down_revision: Union[str, Sequence[str], None] = ('b2c3d4e5f6a7', 'c7d8e9f0a1b2')
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
pass
def downgrade() -> None:
"""Downgrade schema."""
pass

178
backend/alembic/versions/543d43d752b8_migrate_old_dictionary_entries_and_.py
View File

@@ -1,178 +0,0 @@
"""migrate old dictionary entries and translation records
Revision ID: 543d43d752b8
Revises: c4a3a2f74bfe
Create Date: 2026-05-14 18:00:00.000000
"""
from collections.abc import Sequence
from datetime import UTC
from sqlalchemy.sql import text
from alembic import op
# revision identifiers, used by Alembic.
revision: str = '543d43d752b8'
down_revision: str | Sequence[str] | None = 'c4a3a2f74bfe'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Upgrade schema — migrate old dictionary entries and create TranslationLanguage records."""
connection = op.get_bind()
# === Part 1: Migrate DictionaryEntry rows without language pair ===
# 1a. Set source_language = 'und' for rows where it's NULL or empty
connection.execute(
text("""
UPDATE dictionary_entries
SET source_language = 'und'
WHERE source_language IS NULL OR source_language = '' OR source_language = 'und'
""")
)
# 1b. Set target_language from parent dictionary for rows where it's NULL, empty, or 'und'
connection.execute(
text("""
UPDATE dictionary_entries
SET target_language = COALESCE(
(SELECT td.target_language FROM terminology_dictionaries td
WHERE td.id = dictionary_entries.dictionary_id),
'und'
)
WHERE target_language IS NULL
OR target_language = ''
OR target_language = 'und'
""")
)
# 1c. Set target_language default fallback in case dictionary's own target_language is also NULL/empty
connection.execute(
text("""
UPDATE dictionary_entries
SET target_language = 'und'
WHERE target_language IS NULL OR target_language = ''
""")
)
# === Part 2: Create TranslationLanguage entries for old TranslationRecord rows ===
# Records that have deprecated fields (final_value, llm_translation, user_edit) populated
# but no corresponding TranslationLanguage row
# Determine target_language from the parent run's job
# We use a two-step approach: find records missing TranslationLanguage rows,
# then create one per record using the default language code from the job config
missing = connection.execute(
text("""
SELECT r.id AS record_id,
r.run_id,
COALESCE(r.final_value, r.llm_translation, r.target_sql, '') AS resolved_value,
COALESCE(r.user_edit, r.llm_translation, r.target_sql, '') AS resolved_edit,
r.status
FROM translation_records r
WHERE r.status = 'SUCCESS'
AND (r.final_value IS NOT NULL OR r.llm_translation IS NOT NULL OR r.target_sql IS NOT NULL)
AND NOT EXISTS (
SELECT 1 FROM translation_languages tl
WHERE tl.record_id = r.id
)
""")
).fetchall()
from datetime import datetime
import uuid
for row in missing:
record_id = row[0]
resolved_value = row[2] or ""
resolved_edit = row[3] or ""
status = row[4] or "SUCCESS"
# Map record status to language status
lang_status = "translated"
if status in ("APPROVED", "approved"):
lang_status = "approved"
elif status in ("EDITED", "edited"):
lang_status = "edited"
# Determine language code from job
lang_code_row = connection.execute(
text("""
SELECT j.target_language, j.target_languages
FROM translation_records r
JOIN translation_runs rn ON rn.id = r.run_id
JOIN translation_jobs j ON j.id = rn.job_id
WHERE r.id = :rid
"""),
{"rid": record_id},
).fetchone()
language_code = "und"
if lang_code_row:
lang_code = lang_code_row[0]
lang_codes_json = lang_code_row[1]
if lang_code:
language_code = lang_code
elif lang_codes_json:
try:
import json
codes = json.loads(lang_codes_json) if isinstance(lang_codes_json, str) else lang_codes_json
if isinstance(codes, list) and codes:
language_code = codes[0]
except (json.JSONDecodeError, TypeError):
pass
# Determine final value (prefer user_edit over llm_translation)
final_value = resolved_edit if resolved_edit else resolved_value
now = datetime.now(UTC)
lang_id = str(uuid.uuid4())
connection.execute(
text("""
INSERT INTO translation_languages
(id, record_id, language_code, source_language_detected,
translated_value, user_edit, final_value, status, created_at)
VALUES
(:id, :record_id, :language_code, 'und',
:translated_value, :user_edit, :final_value, :status, :created_at)
"""),
{
"id": lang_id,
"record_id": record_id,
"language_code": language_code,
"translated_value": resolved_value or "",
"user_edit": resolved_edit or "",
"final_value": final_value or "",
"status": lang_status,
"created_at": now,
},
)
def downgrade() -> None:
"""Downgrade — reverse the migration (delete created TranslationLanguage rows, restore old values)."""
connection = op.get_bind()
# Remove TranslationLanguage entries that were created by this migration
# (those linked to records that had no existing TranslationLanguage rows)
connection.execute(
text("""
DELETE FROM translation_languages
WHERE id IN (
SELECT tl.id FROM translation_languages tl
JOIN translation_records r ON r.id = tl.record_id
WHERE r.created_at < '2026-05-15'
AND tl.created_at >= '2026-05-14T18:00:00'
)
""")
)
# Note: dictionary_entries language updates are intentionally NOT reverted
# because 'und' is an acceptable safe default, and reverting could break
# unique constraints if entries were duplicated.

28
backend/alembic/versions/6b8ca3b7405f_merge_heads.py
View File

@@ -1,28 +0,0 @@
"""merge heads
Revision ID: 6b8ca3b7405f
Revises: c0d1e2f3a4b5, f2b3c4d5e6f7
Create Date: 2026-06-10 23:40:49.327783
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '6b8ca3b7405f'
down_revision: Union[str, Sequence[str], None] = ('c0d1e2f3a4b5', 'f2b3c4d5e6f7')
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
pass
def downgrade() -> None:
"""Downgrade schema."""
pass

36
backend/alembic/versions/7703bbc038bd_add_description_to_validation_policy.py
View File

@@ -1,36 +0,0 @@
"""add_description_to_validation_policy
Revision ID: 7703bbc038bd
Revises: c9d8e7f6a5b4
Create Date: 2026-05-31 21:07:34.291012
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = '7703bbc038bd'
down_revision: Union[str, Sequence[str], None] = 'c9d8e7f6a5b4'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def _table_exists(table_name: str) -> bool:
conn = op.get_bind()
inspector = inspect(conn)
return table_name in inspector.get_table_names()
def upgrade() -> None:
"""Add description column to validation_policies."""
if not _table_exists("validation_policies"):
return
op.add_column('validation_policies', sa.Column('description', sa.Text(), nullable=True))
def downgrade() -> None:
"""Remove description column from validation_policies."""
op.drop_column('validation_policies', 'description')

40
backend/alembic/versions/7eaf84b7f6be_merge_6b8ca3b7405f_b4c5d6e7f8a9_.py
View File

@@ -1,40 +0,0 @@
# #region Alembic.MergeThreeHeads [C:2] [TYPE Module] [SEMANTICS alembic,merge,heads]
# @ingroup Alembic
# @BRIEF Merge three migration heads into one: 6b8ca3b7405f, b4c5d6e7f8a9, f4a5b6c7d8e9.
# Branches b4c5d6e7f8a9 and f4a5b6c7d8e9 were created from f2b3c4d5e6f7 after the
# previous merge (6b8ca3b7405f), creating multiple heads. This merge resolves them.
# @LAYER Database
# @RELATION DEPENDS_ON -> [Alembic.AddAgentConversations]
# @RELATION DEPENDS_ON -> [Alembic.AddIncludeSourceReference]
# @RELATION DEPENDS_ON -> [Alembic.AddDeploymentValidation]
"""merge: 6b8ca3b7405f, b4c5d6e7f8a9, f4a5b6c7d8e9
Revision ID: 7eaf84b7f6be
Revises: 6b8ca3b7405f, b4c5d6e7f8a9, f4a5b6c7d8e9
Create Date: 2026-07-14 17:29:58.171927
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '7eaf84b7f6be'
down_revision: Union[str, Sequence[str], None] = ('6b8ca3b7405f', 'b4c5d6e7f8a9', 'f4a5b6c7d8e9')
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
pass
def downgrade() -> None:
"""Downgrade schema."""
pass
# #endregion Alembic.MergeThreeHeads

37
backend/alembic/versions/86c7b1d6a710_add_is_admin_column_to_roles_table.py
View File

@@ -1,37 +0,0 @@
"""add is_admin column to roles table
Revision ID: 86c7b1d6a710
Revises: b0c1d2e3f4a5
Create Date: 2026-05-26 15:27:06.159151
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = '86c7b1d6a710'
down_revision: str | Sequence[str] | None = 'b0c1d2e3f4a5'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _table_exists(table_name: str) -> bool:
conn = op.get_bind()
inspector = inspect(conn)
return table_name in inspector.get_table_names()
def upgrade() -> None:
"""Add is_admin column to roles table, set True for existing Admin roles."""
if not _table_exists("roles"):
return
op.add_column("roles", sa.Column("is_admin", sa.Boolean(), nullable=False, server_default=sa.text("false")))
op.execute("UPDATE roles SET is_admin = true WHERE name = 'Admin'")
def downgrade() -> None:
"""Remove is_admin column from roles table."""
op.drop_column("roles", "is_admin")

39
backend/alembic/versions/8dd0a93af539_drop_deprecated_source_language_column_.py
View File

@@ -1,39 +0,0 @@
"""Drop deprecated source_language column from translation_jobs
Revision ID: 8dd0a93af539
Revises: 543d43d752b8
Create Date: 2026-05-14 23:43:32.216941
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = '8dd0a93af539'
down_revision: str | Sequence[str] | None = '543d43d752b8'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Upgrade schema."""
# Use DROP COLUMN IF EXISTS for PostgreSQL safety (column may not exist on all environments)
# SQLite does not support IF EXISTS, but op.drop_column works when column exists
bind = op.get_bind()
if bind.engine.name == "sqlite":
# SQLite: drop only if column exists
inspector = sa.inspect(bind)
columns = [c["name"] for c in inspector.get_columns("translation_jobs")]
if "source_language" in columns:
op.drop_column("translation_jobs", "source_language")
else:
# PostgreSQL and others: use IF EXISTS
op.execute("ALTER TABLE translation_jobs DROP COLUMN IF EXISTS source_language")
def downgrade() -> None:
"""Downgrade schema."""
op.add_column("translation_jobs", sa.Column("source_language", sa.VARCHAR(), nullable=True))

75
backend/alembic/versions/8e9f0a1b2c3d_add_session_activity_table.py
View File

@@ -1,75 +0,0 @@
# #region Alembic.AddSessionActivityTable [C:2] [TYPE Module] [SEMANTICS alembic,migration,session,activity,auth]
# @defgroup Alembic Create the session_activity table for session timeout enforcement.
# @PRE Previous migration (f7a8b9c0d1e2) has been applied.
# @POST session_activity table is created when users exists; otherwise ORM create_all()
# creates it after the fresh Alembic upgrade.
# @SIDE_EFFECT DDL execution — creates table, index, foreign key constraint.
# @RELATION DEPENDS_ON -> [Models.Auth.SessionActivity]
# @RELATION DEPENDS_ON -> [Models.Auth.User]
# @RATIONALE users is ORM-owned and is created by Base.metadata.create_all() after
# Alembic during a fresh install, so the FK migration must be safely skipped first.
# @REJECTED Unconditionally creating the FK table was rejected — fresh installs fail
# before application startup because users does not yet exist.
"""add session_activity table
Revision ID: 8e9f0a1b2c3d
Revises: f7a8b9c0d1e2
Create Date: 2026-07-23 07:45:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = "8e9f0a1b2c3d"
down_revision: str | Sequence[str] | None = "f7a8b9c0d1e2"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
# #region Alembic.AddSessionActivityTable.TableExists [C:1] [TYPE Function] [SEMANTICS alembic,helper,table]
# @BRIEF Check if a table already exists in the database.
def _table_exists(table_name: str) -> bool:
conn = op.get_bind()
inspector = inspect(conn)
return table_name in inspector.get_table_names()
# #endregion Alembic.AddSessionActivityTable.TableExists
# #region Alembic.AddSessionActivityTable.Upgrade [C:2] [TYPE Function] [SEMANTICS alembic,upgrade]
# @BRIEF Create session_activity table for idle/absolute timeout enforcement.
# @PRE auth.users table exists in the database.
# @POST session_activity table created with jti PK, user_id FK->users.id, indexes.
# @SIDE_EFFECT Executes CREATE TABLE DDL.
def upgrade() -> None:
"""Create session_activity table for idle/absolute timeout enforcement."""
if _table_exists("session_activity") or not _table_exists("users"):
return
op.create_table(
"session_activity",
sa.Column("jti", sa.String(), nullable=False),
sa.Column("user_id", sa.String(), nullable=False, index=True),
sa.Column("issued_at", sa.DateTime(), nullable=False),
sa.Column("expires_at", sa.DateTime(), nullable=False),
sa.Column("last_activity_at", sa.DateTime(), nullable=False, server_default=sa.func.now()),
sa.Column("ip_address", sa.String(), nullable=True),
sa.Column("user_agent", sa.String(), nullable=True),
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("jti"),
)
# #endregion Alembic.AddSessionActivityTable.Upgrade
# #region Alembic.AddSessionActivityTable.Downgrade [C:1] [TYPE Function] [SEMANTICS alembic,downgrade]
# @BRIEF Drop session_activity table, reverting the upgrade.
# @POST session_activity table dropped.
# @SIDE_EFFECT Executes DROP TABLE DDL.
def downgrade() -> None:
"""Drop session_activity table."""
op.drop_table("session_activity")
# #endregion Alembic.AddSessionActivityTable.Downgrade
# #endregion Alembic.AddSessionActivityTable

64
backend/alembic/versions/9a5a3b802c49_add_verification_runs_fanout_plan_id_.py
View File

@@ -1,64 +0,0 @@
"""add verification_runs fanout_plan_id backfill
Revision ID: 9a5a3b802c49
Revises: 015281bd7759
Create Date: 2026-08-07 16:52:10.499918
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '9a5a3b802c49'
down_revision: Union[str, Sequence[str], None] = '015281bd7759'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def _column_names(bind, table: str) -> set[str]:
"""Column names present on a table in the current connection."""
return {col["name"] for col in sa.inspect(bind).get_columns(table)}
def upgrade() -> None:
"""Backfill the missing verification_runs.fanout_plan_id column.
The database was stamped at head before the additive 041 amendment landed on the
lineage migration, so verification_runs never gained this column even though the ORM
model references it. Guarded so it is a no-op on databases that already have it.
"""
bind = op.get_bind()
if not sa.inspect(bind).has_table("verification_runs"):
return
if "fanout_plan_id" in _column_names(bind, "verification_runs"):
return
op.add_column(
"verification_runs",
sa.Column(
"fanout_plan_id",
sa.String(),
sa.ForeignKey("fanout_plans.id", ondelete="SET NULL"),
nullable=True,
),
)
op.create_index(
"ix_verification_runs_fanout_plan",
"verification_runs",
["fanout_plan_id"],
)
def downgrade() -> None:
"""Drop the backfilled column and its index if present."""
bind = op.get_bind()
if not sa.inspect(bind).has_table("verification_runs"):
return
if "fanout_plan_id" not in _column_names(bind, "verification_runs"):
return
op.drop_index("ix_verification_runs_fanout_plan", table_name="verification_runs")
op.drop_column("verification_runs", "fanout_plan_id")

103
backend/alembic/versions/9f8e7d6c5b4a_add_is_multimodal_to_llm_providers.py
View File

@@ -1,103 +0,0 @@
"""Add is_multimodal column to llm_providers with heuristic backfill
Revision ID: 9f8e7d6c5b4a
Revises: b1c2d3e4f5a6
Create Date: 2026-05-20 14:30:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = "9f8e7d6c5b4a"
down_revision: str | Sequence[str] | None = "b1c2d3e4f5a6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _is_multimodal_heuristic(model_name: str) -> bool:
"""Backfill heuristic: mirrors the logic from services/llm_prompt_templates.py
at the time of migration creation. Returns True for known multimodal/vision models.
"""
token = (model_name or "").strip().lower()
if not token:
return False
text_only_markers = (
"text-only",
"embedding",
"rerank",
"whisper",
"tts",
"transcribe",
)
if any(marker in token for marker in text_only_markers):
return False
multimodal_markers = (
"gpt-4o",
"gpt-4.1",
"vision",
"vl",
"gemini",
"claude-3",
"claude-sonnet-4",
"omni",
"multimodal",
"pixtral",
"llava",
"internvl",
"qwen-vl",
"qwen2-vl",
)
return any(marker in token for marker in multimodal_markers)
def _table_exists(table_name: str) -> bool:
"""Check if a table exists in the current database schema."""
conn = op.get_bind()
inspector = inspect(conn)
return table_name in inspector.get_table_names()
def upgrade() -> None:
"""Add is_multimodal column and backfill using heuristic."""
# The llm_providers table is defined in the ORM model (src/models/llm.py)
# and created by Base.metadata.create_all() at app startup, not via migrations.
# If the table doesn't exist yet (fresh DB), skip — the model definition
# already includes is_multimodal, so create_all() will create it with the column.
if not _table_exists("llm_providers"):
return
# Step 1: Add column as nullable first
op.add_column(
"llm_providers",
sa.Column("is_multimodal", sa.Boolean(), nullable=True),
)
# Step 2: Backfill existing rows using heuristic on default_model
connection = op.get_bind()
providers = connection.execute(
sa.text("SELECT id, default_model FROM llm_providers")
).fetchall()
for row in providers:
multimodal = _is_multimodal_heuristic(row.default_model)
connection.execute(
sa.text(
"UPDATE llm_providers SET is_multimodal = :multimodal WHERE id = :id"
),
{"multimodal": multimodal, "id": row.id},
)
# Step 3: Make column non-nullable with default False
# Use text("false") for PostgreSQL compatibility (not "0")
op.alter_column("llm_providers", "is_multimodal", nullable=False, server_default=sa.text("false"))
def downgrade() -> None:
"""Remove is_multimodal column."""
op.drop_column("llm_providers", "is_multimodal")

89
backend/alembic/versions/a1b2c3d4e5f7_session_activity_logical_sessions.py
View File

@@ -1,89 +0,0 @@
# #region Alembic.Migration.SessionActivityLogicalSessions [C:2] [TYPE Migration] [SEMANTICS alembic,migration,session,activity,sid]
# @BRIEF Convert session_activity from per-JWT tracking to logical-session (sid) tracking.
# @PRE Previous migration (o1p2q3r4s5t6) has been applied.
# @POST session_activity rows are keyed by sid; is_revoked column present for whole-session revocation.
# @SIDE_EFFECT DDL execution — renames jti -> sid, adds is_revoked column.
# @RELATION DEPENDS_ON -> [Models.Auth.SessionActivity]
# @RATIONALE Replacement JWTs share one logical session (sid). Existing rows carry their
# historical jti forward as sid; they are non-renewable but still enforceable.
# @REJECTED Dropping and recreating the table was rejected — loses existing audit rows and
# would break live deployments that already enforce idle timeouts.
"""convert session_activity to logical session rows
Revision ID: a1b2c3d4e5f7
Revises: o1p2q3r4s5t6
Create Date: 2026-08-06 12:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = "a1b2c3d4e5f7"
down_revision: str | Sequence[str] | None = "o1p2q3r4s5t6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
# #region Migration.SessionActivityLogicalSessions.TableExists [C:1] [TYPE Function] [SEMANTICS alembic,helper,table]
# @BRIEF Check if a table already exists in the database.
def _table_exists(table_name: str) -> bool:
conn = op.get_bind()
inspector = inspect(conn)
return table_name in inspector.get_table_names()
# #endregion Migration.SessionActivityLogicalSessions.TableExists
# #region Migration.SessionActivityLogicalSessions.ColumnNames [C:1] [TYPE Function] [SEMANTICS alembic,helper,column]
# @BRIEF Return the set of column names of a table.
def _column_names(table_name: str) -> set[str]:
inspector = inspect(op.get_bind())
return {c["name"] for c in inspector.get_columns(table_name)}
# #endregion Migration.SessionActivityLogicalSessions.ColumnNames
# #region Migration.SessionActivityLogicalSessions.Upgrade [C:2] [TYPE Function] [SEMANTICS alembic,upgrade]
# @BRIEF Rename jti -> sid (when needed) and add is_revoked column.
# @PRE session_activity table exists.
# @POST session_activity keyed by sid with is_revoked defaulting to False.
# @SIDE_EFFECT Executes ALTER TABLE DDL.
def upgrade() -> None:
"""Convert session_activity to logical-session keying and add revocation state."""
if not _table_exists("session_activity"):
return
cols = _column_names("session_activity")
with op.batch_alter_table("session_activity") as batch_op:
if "jti" in cols and "sid" not in cols:
batch_op.alter_column("jti", new_column_name="sid")
if "is_revoked" not in cols:
batch_op.add_column(
sa.Column(
"is_revoked",
sa.Boolean(),
nullable=False,
server_default=sa.false(),
)
)
# #endregion Migration.SessionActivityLogicalSessions.Upgrade
# #region Migration.SessionActivityLogicalSessions.Downgrade [C:1] [TYPE Function] [SEMANTICS alembic,downgrade]
# @BRIEF Revert sid -> jti and drop is_revoked.
# @POST session_activity restored to per-JWT keying without revocation state.
# @SIDE_EFFECT Executes ALTER TABLE DDL.
def downgrade() -> None:
"""Revert logical-session keying back to jti and drop is_revoked."""
if not _table_exists("session_activity"):
return
cols = _column_names("session_activity")
with op.batch_alter_table("session_activity") as batch_op:
if "sid" in cols and "jti" not in cols:
batch_op.alter_column("sid", new_column_name="jti")
if "is_revoked" in cols:
batch_op.drop_column("is_revoked")
# #endregion Migration.SessionActivityLogicalSessions.Downgrade
# #endregion Alembic.Migration.SessionActivityLogicalSessions

55
backend/alembic/versions/a5b6c7d8e9f0_set_null_ondelete_for_task_records_env_fk.py
View File

@@ -1,55 +0,0 @@
"""set null ondelete for task_records environment FK
task_records table is now created by prior migration d1e2f3a4b5c6.
This migration alters the FK to add ondelete='SET NULL' for databases
that were migrated before the FK was defined with ondelete.
Revision ID: a5b6c7d8e9f0
Revises: d1e2f3a4b5c6
Create Date: 2026-05-21 18:55:00.000000
"""
from collections.abc import Sequence
from alembic import op
# revision identifiers, used by Alembic.
revision: str = 'a5b6c7d8e9f0'
down_revision: str | Sequence[str] | None = 'd1e2f3a4b5c6'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Upgrade schema."""
op.drop_constraint(
'task_records_environment_id_fkey',
'task_records',
type_='foreignkey',
)
op.create_foreign_key(
'task_records_environment_id_fkey',
'task_records',
'environments',
['environment_id'],
['id'],
ondelete='SET NULL',
)
def downgrade() -> None:
"""Downgrade schema."""
op.drop_constraint(
'task_records_environment_id_fkey',
'task_records',
type_='foreignkey',
)
op.create_foreign_key(
'task_records_environment_id_fkey',
'task_records',
'environments',
['environment_id'],
['id'],
)

40
backend/alembic/versions/a7b1c2d3e4f5_add_provider_id_to_validation_policies.py
View File

@@ -1,40 +0,0 @@
"""Add provider_id column to validation_policies
Revision ID: a7b1c2d3e4f5
Revises: 9f8e7d6c5b4a
Create Date: 2026-05-21 10:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = "a7b1c2d3e4f5"
down_revision: str | Sequence[str] | None = "9f8e7d6c5b4a"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _table_exists(table_name: str) -> bool:
conn = op.get_bind()
inspector = inspect(conn)
return table_name in inspector.get_table_names()
def upgrade() -> None:
"""Add provider_id column to validation_policies as nullable."""
if not _table_exists("validation_policies"):
return
op.add_column(
"validation_policies",
sa.Column("provider_id", sa.String(), nullable=True),
)
def downgrade() -> None:
"""Remove provider_id column from validation_policies."""
op.drop_column("validation_policies", "provider_id")

113
backend/alembic/versions/a8b9c0d1e2f3_add_scenario_automation_tables.py
View File

@@ -1,113 +0,0 @@
# #region Alembic.ScenarioAutomationTables [C:3] [TYPE Module] [SEMANTICS alembic,scenario,automation,schedule,trigger,notification]
# @ingroup Alembic
# @BRIEF Add 046 scenario schedules, trigger rules and notification events tables.
# @LAYER Database
"""add scenario automation tables (046 dashboard-scenario-automation)
Revision ID: a8b9c0d1e2f3
Revises: z7a8b9c0d1e2
Create Date: 2026-08-20
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "a8b9c0d1e2f3"
down_revision: str | Sequence[str] | None = "z7a8b9c0d1e2"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Create scenario_schedules, scenario_trigger_rules, scenario_automation_policies,
scenario_notification_events (046 automation persistence).
revision_id is nullable — required only when revision_policy=pinned; a `current`
schedule/rule resolves the atomically activated revision at dispatch time (042).
"""
# ── scenario_schedules ──
op.create_table(
"scenario_schedules",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("scenario_id", sa.String(length=36), nullable=False),
sa.Column("revision_policy", sa.String(length=16), nullable=False),
sa.Column("revision_id", sa.String(length=36), nullable=True),
sa.Column("environment_id", sa.String(length=128), nullable=False),
sa.Column("cron_expr", sa.String(length=128), nullable=False),
sa.Column("timezone", sa.String(length=64), nullable=False),
sa.Column("missed_execution_policy", sa.String(length=16), nullable=False),
sa.Column("policy_id", sa.String(length=36), nullable=True),
sa.Column("enabled", sa.Boolean(), nullable=False),
sa.Column("max_instances", sa.Integer(), nullable=False),
sa.Column("misfire_grace_time", sa.Integer(), nullable=False),
sa.Column("created_by", sa.String(length=128), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_scenario_schedules_scenario_id", "scenario_schedules", ["scenario_id"])
# ── scenario_trigger_rules ──
op.create_table(
"scenario_trigger_rules",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("trigger", sa.String(length=64), nullable=False),
sa.Column("scenario_id", sa.String(length=36), nullable=False),
sa.Column("revision_policy", sa.String(length=16), nullable=False),
sa.Column("revision_id", sa.String(length=36), nullable=True),
sa.Column("environment_id", sa.String(length=128), nullable=False),
sa.Column("policy_id", sa.String(length=36), nullable=True),
sa.Column("enabled", sa.Boolean(), nullable=False),
sa.Column("conditions", sa.JSON(), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_scenario_trigger_rules_trigger", "scenario_trigger_rules", ["trigger"])
# ── scenario_automation_policies ──
op.create_table(
"scenario_automation_policies",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("name", sa.String(length=255), nullable=False),
sa.Column("enabled", sa.Boolean(), nullable=False),
sa.Column("workload_class", sa.String(length=32), nullable=False),
sa.Column("max_concurrent_per_env", sa.Integer(), nullable=False),
sa.Column("dedup_window_seconds", sa.Integer(), nullable=False),
sa.Column("overlap_rule", sa.String(length=16), nullable=False),
sa.Column("retention_days", sa.Integer(), nullable=False),
sa.Column("prod_gate_required", sa.Boolean(), nullable=False),
sa.Column("on_repeated_failure", sa.String(length=16), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
# ── scenario_notification_events ──
op.create_table(
"scenario_notification_events",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("event_type", sa.String(length=32), nullable=False),
sa.Column("scenario_id", sa.String(length=36), nullable=False),
sa.Column("run_id", sa.String(length=36), nullable=True),
sa.Column("severity", sa.String(length=16), nullable=False),
sa.Column("payload", sa.JSON(), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
"ix_scenario_notification_events_event_type", "scenario_notification_events", ["event_type"]
)
def downgrade() -> None:
"""Drop scenario automation tables in reverse dependency order."""
op.drop_index("ix_scenario_notification_events_event_type", table_name="scenario_notification_events")
op.drop_table("scenario_notification_events")
op.drop_table("scenario_automation_policies")
op.drop_index("ix_scenario_trigger_rules_trigger", table_name="scenario_trigger_rules")
op.drop_table("scenario_trigger_rules")
op.drop_index("ix_scenario_schedules_scenario_id", table_name="scenario_schedules")
op.drop_table("scenario_schedules")
# #endregion Alembic.ScenarioAutomationTables

67
backend/alembic/versions/aa1b2c3d4e5f_drop_deprecated_translate_columns.py
View File

@@ -1,67 +0,0 @@
"""Drop all deprecated columns from translate models
Deprecated columns removed:
- translation_jobs.target_language (use target_languages JSON instead)
- translation_records.llm_translation, user_edit, final_value (use TranslationLanguage instead)
- terminology_dictionaries.source_language, target_language (use per-entry DictionaryEntry instead)
Revision ID: aa1b2c3d4e5f
Revises: 2a7b8c9d0e1f
Create Date: 2026-05-14 23:59:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "aa1b2c3d4e5f"
down_revision: str | Sequence[str] | None = "2a7b8c9d0e1f"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Drop all deprecated columns using DROP COLUMN IF EXISTS for safety."""
bind = op.get_bind()
if bind.engine.name == "sqlite":
inspector = sa.inspect(bind)
_drop_if_exists_sqlite(inspector, "translation_jobs", "target_language")
_drop_if_exists_sqlite(inspector, "translation_records", "llm_translation")
_drop_if_exists_sqlite(inspector, "translation_records", "user_edit")
_drop_if_exists_sqlite(inspector, "translation_records", "final_value")
_drop_if_exists_sqlite(inspector, "terminology_dictionaries", "source_language")
_drop_if_exists_sqlite(inspector, "terminology_dictionaries", "target_language")
else:
# PostgreSQL and others: use IF EXISTS
op.execute("ALTER TABLE translation_jobs DROP COLUMN IF EXISTS target_language")
op.execute("ALTER TABLE translation_records DROP COLUMN IF EXISTS llm_translation")
op.execute("ALTER TABLE translation_records DROP COLUMN IF EXISTS user_edit")
op.execute("ALTER TABLE translation_records DROP COLUMN IF EXISTS final_value")
op.execute("ALTER TABLE terminology_dictionaries DROP COLUMN IF EXISTS source_language")
op.execute("ALTER TABLE terminology_dictionaries DROP COLUMN IF EXISTS target_language")
def downgrade() -> None:
"""Restore all deprecated columns (for rollback)."""
op.add_column("translation_jobs", sa.Column("target_language", sa.String(), nullable=True,
comment="Target language code (e.g. en, ru) [DEPRECATED: use target_languages]"))
op.add_column("translation_records", sa.Column("llm_translation", sa.Text(), nullable=True,
comment="[DEPRECATED: use TranslationLanguage]"))
op.add_column("translation_records", sa.Column("user_edit", sa.Text(), nullable=True,
comment="[DEPRECATED: use TranslationLanguage]"))
op.add_column("translation_records", sa.Column("final_value", sa.Text(), nullable=True,
comment="[DEPRECATED: use TranslationLanguage]"))
op.add_column("terminology_dictionaries", sa.Column("source_language", sa.String(), nullable=True,
comment="[DEPRECATED: use per-entry source_language]"))
op.add_column("terminology_dictionaries", sa.Column("target_language", sa.String(), nullable=True,
comment="[DEPRECATED: use per-entry target_language]"))
def _drop_if_exists_sqlite(inspector, table: str, column: str) -> None:
"""Drop a column from a SQLite table only if it exists."""
columns = [c["name"] for c in inspector.get_columns(table)]
if column in columns:
op.drop_column(table, column)

82
backend/alembic/versions/b0c1d2e3f4a5_cascade_ondelete_translate_fks.py
View File

@@ -1,82 +0,0 @@
"""cascade ondelete for all translate FK references (translation_jobs/runs/batches chain)
Revision ID: b0c1d2e3f4a5
Revises: f0e9d8c7b6a5
Create Date: 2026-05-21 19:05:00.000000
"""
from collections.abc import Sequence
from alembic import op
# revision identifiers, used by Alembic.
revision: str = 'b0c1d2e3f4a5'
down_revision: str | Sequence[str] | None = 'f0e9d8c7b6a5'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
# Each entry: (table, constraint_name, column, ref_table, ondelete_rule)
FK_FIXES = [
# Direct FKs to translation_jobs.id
('translation_runs', 'translation_runs_job_id_fkey', 'job_id', 'CASCADE'),
('translation_events', 'translation_events_job_id_fkey', 'job_id', 'CASCADE'),
('translation_preview_sessions', 'translation_preview_sessions_job_id_fkey', 'job_id', 'CASCADE'),
('translation_schedules', 'translation_schedules_job_id_fkey', 'job_id', 'CASCADE'),
('translation_job_dictionaries', 'translation_job_dictionaries_job_id_fkey', 'job_id', 'CASCADE'),
('translation_metric_snapshots', 'translation_metric_snapshots_job_id_fkey', 'job_id', 'CASCADE'),
# FKs to translation_runs.id
('translation_batches', 'translation_batches_run_id_fkey', 'run_id', 'CASCADE'),
('translation_records', 'translation_records_run_id_fkey', 'run_id', 'CASCADE'),
('translation_events', 'translation_events_run_id_fkey', 'run_id', 'SET NULL'),
('translation_preview_sessions', 'translation_preview_sessions_run_id_fkey', 'run_id', 'SET NULL'),
('translation_metric_snapshots', 'translation_metric_snapshots_run_id_fkey', 'run_id', 'SET NULL'),
('translation_run_language_stats', 'translation_run_language_stats_run_id_fkey', 'run_id', 'CASCADE'),
# FKs to translation_batches.id
('translation_records', 'translation_records_batch_id_fkey', 'batch_id', 'CASCADE'),
# FKs to translation_records.id
('translation_languages', 'translation_languages_record_id_fkey', 'record_id', 'CASCADE'),
# FKs to translation_preview_sessions.id
('translation_preview_records', 'translation_preview_records_session_id_fkey', 'session_id', 'CASCADE'),
# FKs to translation_preview_records.id
('translation_preview_languages', 'translation_preview_languages_preview_record_id_fkey', 'preview_record_id', 'CASCADE'),
]
def upgrade() -> None:
"""Upgrade schema."""
for table, constraint, column, ondelete in FK_FIXES:
op.drop_constraint(constraint, table, type_='foreignkey')
op.create_foreign_key(
constraint, table, 'translation_jobs' if 'job_id' in column else (
'translation_runs' if 'run_id' in column else (
'translation_batches' if 'batch_id' in column else (
'translation_records' if 'record_id' in column and 'preview' not in constraint else (
'translation_preview_sessions' if 'session_id' in column else (
'translation_preview_records' if 'preview_record_id' in column else 'unknown'
)
)
)
)
),
[column], ['id'],
ondelete=ondelete,
)
def downgrade() -> None:
"""Downgrade schema."""
ref_map = {
'job_id': 'translation_jobs',
'run_id': 'translation_runs',
'batch_id': 'translation_batches',
'record_id': 'translation_records',
'session_id': 'translation_preview_sessions',
'preview_record_id': 'translation_preview_records',
}
for table, constraint, column, _ondelete in reversed(FK_FIXES):
ref_table = ref_map[column]
op.drop_constraint(constraint, table, type_='foreignkey')
op.create_foreign_key(
constraint, table, ref_table,
[column], ['id'],
)

60
backend/alembic/versions/b1c2d3e4f5a6_add_source_hash_to_translation_records.py
View File

@@ -1,60 +0,0 @@
"""Add source_hash column to translation_records for cache dedup
Adds source_hash (SHA256 of source_text + source_data + dict/config hashes)
enabling cache-hit lookups: if the same source row + dictionary + config
combination has already been successfully translated, the LLM call is skipped.
Revision ID: b1c2d3e4f5a6
Revises: aa1b2c3d4e5f
Create Date: 2026-05-15 23:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "b1c2d3e4f5a6"
down_revision: str | Sequence[str] | None = "aa1b2c3d4e5f"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Add source_hash column with indices."""
bind = op.get_bind()
if bind.engine.name == "sqlite":
# SQLite — check existence first
inspector = sa.inspect(bind)
columns = [c["name"] for c in inspector.get_columns("translation_records")]
if "source_hash" not in columns:
op.add_column("translation_records",
sa.Column("source_hash", sa.String(), nullable=True,
comment="SHA256(source_text+source_data+dict_snapshot_hash+config_hash) for cache dedup"))
op.create_index("ix_translation_records_source_hash_status",
"translation_records", ["source_hash", "status"])
else:
# PostgreSQL and others
op.add_column("translation_records",
sa.Column("source_hash", sa.String(), nullable=True,
comment="SHA256(source_text+source_data+dict_snapshot_hash+config_hash) for cache dedup"))
op.create_index("ix_translation_records_source_hash_status",
"translation_records", ["source_hash", "status"])
def downgrade() -> None:
"""Drop source_hash column and index."""
bind = op.get_bind()
if bind.engine.name == "sqlite":
inspector = sa.inspect(bind)
columns = [c["name"] for c in inspector.get_columns("translation_records")]
if "source_hash" in columns:
op.drop_index("ix_translation_records_source_hash_status",
table_name="translation_records")
op.drop_column("translation_records", "source_hash")
else:
op.drop_index("ix_translation_records_source_hash_status",
table_name="translation_records")
op.drop_column("translation_records", "source_hash")

76
backend/alembic/versions/b2a3c4d5e6f7_add_agent_lifecycle_events.py
View File

@@ -1,76 +0,0 @@
# #region Alembic.AddAgentLifecycleEvents [C:3] [TYPE Module] [SEMANTICS alembic,agent,lifecycle,audit]
# @ingroup Alembic
# @BRIEF Add agent_lifecycle_events table for Phase 3 durable agent lifecycle audit.
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.Agent.AgentLifecycleEvent]
# @INVARIANT table has composite indexes for common query patterns (user+type+created, conv+type+created).
# @RATIONALE Immutable, indexed events make trace/conversation diagnostics queryable without
# keeping raw prompts or tool output in application logs.
# @REJECTED Reusing agent_messages was rejected — message content has a separate retention and
# privacy contract and cannot represent request/tool lifecycle boundaries safely.
"""add agent_lifecycle_events table
Revision ID: b2a3c4d5e6f7
Revises: 7eaf84b7f6be
Create Date: 2026-07-15 10:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "b2a3c4d5e6f7"
down_revision: Union[str, Sequence[str], None] = "7eaf84b7f6be"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"agent_lifecycle_events",
sa.Column("id", sa.String(), nullable=False),
sa.Column("trace_id", sa.String(), nullable=False, index=True),
sa.Column("conversation_id", sa.String(), nullable=False, index=True),
sa.Column("user_id", sa.String(), nullable=False, index=True),
sa.Column("environment_id", sa.String(), nullable=True, index=True),
sa.Column("event_type", sa.String(), nullable=False, index=True),
sa.Column("tool_name", sa.String(), nullable=True, index=True),
sa.Column("status", sa.String(), nullable=True, index=True),
# Store UTC timestamps explicitly; the model normalizes values to UTC
# before serialization, independent of the database session timezone.
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("elapsed_ms", sa.Float(), nullable=True),
sa.Column("payload", sa.JSON(), nullable=True),
sa.Column("error_code", sa.String(), nullable=True, index=True),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
"ix_agent_lifecycle_events_user_type_created",
"agent_lifecycle_events",
["user_id", "event_type", "created_at"],
unique=False,
)
op.create_index(
"ix_agent_lifecycle_events_conv_type_created",
"agent_lifecycle_events",
["conversation_id", "event_type", "created_at"],
unique=False,
)
op.create_index(
"ix_agent_lifecycle_events_created",
"agent_lifecycle_events",
["created_at"],
unique=False,
)
def downgrade() -> None:
op.drop_index("ix_agent_lifecycle_events_created", table_name="agent_lifecycle_events")
op.drop_index("ix_agent_lifecycle_events_conv_type_created", table_name="agent_lifecycle_events")
op.drop_index("ix_agent_lifecycle_events_user_type_created", table_name="agent_lifecycle_events")
op.drop_table("agent_lifecycle_events")
# #endregion Alembic.AddAgentLifecycleEvents

70
backend/alembic/versions/b4c5d6e7f8a9_add_include_source_reference_to_translation_jobs.py
View File

@@ -1,70 +0,0 @@
"""Add include_source_reference column to translation_jobs
Revision ID: b4c5d6e7f8a9
Revises: ed310b33f02c
Create Date: 2026-07-12 12:30:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "b4c5d6e7f8a9"
down_revision: str | Sequence[str] | None = "f2b3c4d5e6f7"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Add include_source_reference column to translation_jobs.
This column was added to the TranslationJob model (for controlling
whether to insert original source rows alongside translated ones)
but the corresponding Alembic migration was missed. This additive
migration safely adds it if missing.
"""
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = [c["name"] for c in inspector.get_columns("translation_jobs")]
if "include_source_reference" in columns:
# Already present
return
if bind.engine.name == "sqlite":
op.add_column(
"translation_jobs",
sa.Column(
"include_source_reference",
sa.Boolean(),
nullable=False,
server_default=sa.true(),
comment="If true, insert original/source rows alongside translated rows",
),
)
else:
# PostgreSQL etc. — safe IF NOT EXISTS
op.execute(
"ALTER TABLE translation_jobs "
"ADD COLUMN IF NOT EXISTS include_source_reference BOOLEAN NOT NULL DEFAULT TRUE"
)
def downgrade() -> None:
"""Drop include_source_reference column from translation_jobs."""
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = [c["name"] for c in inspector.get_columns("translation_jobs")]
if "include_source_reference" not in columns:
return
if bind.engine.name == "sqlite":
op.drop_column("translation_jobs", "include_source_reference")
else:
op.execute(
"ALTER TABLE translation_jobs DROP COLUMN IF EXISTS include_source_reference"
)

28
backend/alembic/versions/b9c0d1e2f3a4_add_investigation_evidence_snapshot.py
View File

@@ -1,28 +0,0 @@
# #region Alembic.ScenarioInvestigationEvidence [C:2] [TYPE Module] [SEMANTICS alembic,scenario,investigation,evidence]
"""add immutable evidence snapshots to scenario investigation projections"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "b9c0d1e2f3a4"
down_revision: str | Sequence[str] | None = "a8b9c0d1e2f3"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.add_column("scenario_investigation_queue", sa.Column("evidence_snapshot", sa.JSON(), nullable=False, server_default=sa.text("'{}'")))
op.add_column("scenario_investigation_cases", sa.Column("evidence_snapshot", sa.JSON(), nullable=False, server_default=sa.text("'{}'")))
op.add_column("scenario_investigation_cases", sa.Column("linked_run_ids", sa.JSON(), nullable=False, server_default=sa.text("'[]'")))
op.add_column("scenario_investigation_cases", sa.Column("owner_id", sa.String(length=128), nullable=True))
op.create_index("ix_scenario_investigation_cases_owner_id", "scenario_investigation_cases", ["owner_id"])
def downgrade() -> None:
op.drop_index("ix_scenario_investigation_cases_owner_id", table_name="scenario_investigation_cases")
op.drop_column("scenario_investigation_cases", "owner_id")
op.drop_column("scenario_investigation_cases", "linked_run_ids")
op.drop_column("scenario_investigation_cases", "evidence_snapshot")
op.drop_column("scenario_investigation_queue", "evidence_snapshot")
# #endregion Alembic.ScenarioInvestigationEvidence

34
backend/alembic/versions/c0d1e2f3a4b5_add_insert_method_and_connection_fields.py
View File

@@ -1,34 +0,0 @@
"""add insert_method and connection fields to translation tables
Revision ID: c0d1e2f3a4b5
Revises: f2b3c4d5e6f7
Create Date: 2026-06-10 14:45:00
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'c0d1e2f3a4b5'
down_revision: Union[str, None] = '351afb8f961a'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Add insert_method and connection_id to translation_jobs
op.add_column('translation_jobs', sa.Column('insert_method', sa.String(), nullable=False, server_default='sqllab'))
op.add_column('translation_jobs', sa.Column('connection_id', sa.String(), nullable=True))
# Add insert_method and connection_snapshot to translation_runs
op.add_column('translation_runs', sa.Column('insert_method', sa.String(), nullable=True))
op.add_column('translation_runs', sa.Column('connection_snapshot', sa.JSON(), nullable=True))
def downgrade() -> None:
op.drop_column('translation_runs', 'connection_snapshot')
op.drop_column('translation_runs', 'insert_method')
op.drop_column('translation_jobs', 'connection_id')
op.drop_column('translation_jobs', 'insert_method')

41
backend/alembic/versions/c1d2e3f4a5b6_add_scenario_live_execution_binding.py
View File

@@ -1,41 +0,0 @@
# #region Alembic.ScenarioLiveExecutionBinding [C:3] [TYPE Module] [SEMANTICS alembic,migration,scenario,execution,live-binding]
# @ingroup Alembic
# @BRIEF Add nullable immutable live-execution binding identity fields to durable scenario runs.
# @RELATION DEPENDS_ON -> [Models.ScenarioExecution.Run]
# @INVARIANT Existing runs remain valid with no binding and therefore fail closed as unavailable live I/O.
# @RATIONALE The new fields are nullable and additive so historical runs preserve their original lifecycle evidence.
# @REJECTED Backfilling binding authority from environment_id was rejected — an environment label is not a pinned RLS/principal/model capability.
"""add nullable ScenarioRun live execution binding identity snapshot (044)"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "c1d2e3f4a5b6"
down_revision: str | Sequence[str] | None = "b9c0d1e2f3a4"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
# #region Alembic.ScenarioLiveExecutionBinding.Upgrade [C:4] [TYPE Function] [SEMANTICS alembic,migration,scenario,live-binding,additive]
# @BRIEF Add nullable persisted live-binding identity fields without deriving legacy authority.
# @POST Existing scenario runs retain valid null binding fields; a lookup index exists for binding_ref.
# @SIDE_EFFECT Schema mutation: two nullable columns and one index on scenario_runs.
def upgrade() -> None:
op.add_column("scenario_runs", sa.Column("live_execution_binding_ref", sa.String(128), nullable=True))
op.add_column("scenario_runs", sa.Column("live_execution_binding_snapshot", sa.JSON(), nullable=True))
op.create_index("ix_scenario_runs_live_execution_binding_ref", "scenario_runs", ["live_execution_binding_ref"])
# #endregion Alembic.ScenarioLiveExecutionBinding.Upgrade
# #region Alembic.ScenarioLiveExecutionBinding.Downgrade [C:3] [TYPE Function] [SEMANTICS alembic,migration,scenario,live-binding,rollback]
# @BRIEF Remove only the additive live-binding fields in reverse dependency order.
# @SIDE_EFFECT Schema mutation: binding-ref index and nullable columns are removed.
def downgrade() -> None:
op.drop_index("ix_scenario_runs_live_execution_binding_ref", table_name="scenario_runs")
op.drop_column("scenario_runs", "live_execution_binding_snapshot")
op.drop_column("scenario_runs", "live_execution_binding_ref")
# #endregion Alembic.ScenarioLiveExecutionBinding.Downgrade
# #endregion Alembic.ScenarioLiveExecutionBinding

50
backend/alembic/versions/c3d4e5f6a7b8_add_user_id_to_task_records.py
View File

@@ -1,50 +0,0 @@
# #region Alembic.TaskRecordsUserId [C:3] [TYPE Module] [SEMANTICS alembic,migration,task,postgres]
# @BRIEF Adds nullable task ownership to persistent task records.
# @RELATION DEPENDS_ON -> [EXT:SQLAlchemy:Alembic]
# @POST Existing task_records rows retain data and gain a nullable user_id column.
# @RATIONALE Task ownership must persist so task list queries can be scoped to a user after restart.
# @REJECTED Runtime create_all() or manual ALTER TABLE was rejected because schema evolution must
# remain reproducible through the Alembic migration chain.
"""add user_id column to task_records table
Revision ID: c3d4e5f6a7b8
Revises: b2a3c4d5e6f7
Create Date: 2026-07-15 19:06:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from sqlalchemy import inspect
from alembic import op
# revision identifiers, used by Alembic.
revision: str = 'c3d4e5f6a7b8'
down_revision: str | Sequence[str] | None = 'b2a3c4d5e6f7'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _table_exists(table_name: str) -> bool:
conn = op.get_bind()
inspector = inspect(conn)
return table_name in inspector.get_table_names()
def upgrade() -> None:
"""Add user_id column to task_records table."""
if not _table_exists("task_records"):
return
op.add_column(
"task_records",
sa.Column("user_id", sa.String(), nullable=True),
)
def downgrade() -> None:
"""Remove user_id column from task_records table."""
op.drop_column("task_records", "user_id")
# #endregion Alembic.TaskRecordsUserId

39
backend/alembic/versions/c4a3a2f74bfe_add_missing_columns_needs_review_.py
View File

@@ -1,39 +0,0 @@
"""add missing columns needs_review language_overridden per_language_metrics
Revision ID: c4a3a2f74bfe
Revises: ed310b33f02c
Create Date: 2026-05-14 17:30:36.865269
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = 'c4a3a2f74bfe'
down_revision: str | Sequence[str] | None = 'ed310b33f02c'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# CR1: needs_review + language_overridden on translation_languages
op.add_column('translation_languages', sa.Column('needs_review', sa.Boolean(), nullable=True,
comment='Flagged because source language could not be determined'))
op.add_column('translation_languages', sa.Column('language_overridden', sa.Boolean(), nullable=True,
comment='User manually overrode the auto-detected source language'))
# CR1b: needs_review on translation_preview_languages
op.add_column('translation_preview_languages', sa.Column('needs_review', sa.Boolean(), nullable=True,
comment='Flagged because source language could not be determined'))
# CR2: per_language_metrics on translation_metric_snapshots
op.add_column('translation_metric_snapshots', sa.Column('per_language_metrics', sa.JSON(), nullable=True,
comment='Per-language cumulative metrics: {lang: {cumulative_tokens, cumulative_cost, runs}}'))
def downgrade() -> None:
op.drop_column('translation_metric_snapshots', 'per_language_metrics')
op.drop_column('translation_preview_languages', 'needs_review')
op.drop_column('translation_languages', 'language_overridden')
op.drop_column('translation_languages', 'needs_review')

37
backend/alembic/versions/c7d8e9f0a1b2_add_run_source_hash_composite_index.py
View File

@@ -1,37 +0,0 @@
"""Add composite index (run_id, source_hash) for NOT EXISTS dedup
Adds ix_translation_records_run_source_hash on translation_records(run_id, source_hash)
to accelerate the correlated NOT EXISTS subquery in orchestrator_query.get_run_records
when deduplicate=true.
Revision ID: c7d8e9f0a1b2
Revises: a1b2c3d4e5f6
Create Date: 2026-06-04 13:30:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "c7d8e9f0a1b2"
down_revision: str | Sequence[str] | None = "a1b2c3d4e5f6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Create composite index."""
op.create_index(
"ix_translation_records_run_source_hash",
"translation_records",
["run_id", "source_hash"],
postgresql_using="btree",
)
def downgrade() -> None:
"""Drop composite index."""
op.drop_index("ix_translation_records_run_source_hash",
table_name="translation_records")

389
backend/alembic/versions/c9d8e7f6a5b4_add_v2_validation_models.py
View File

@@ -1,389 +0,0 @@
"""Add v2 LLM validation models (tables, columns, indexes)
Adds:
- validation_sources table
- validation_runs table
- v2 columns to validation_policies
- v2 columns + FK + indexes to llm_validation_results
- index on validation_policies.provider_id
- optional data backfill: creates ValidationRun placeholders for existing
ValidationRecord rows that have a policy_id but no run_id
@ADR [MIGRATION-001] Backfill creates one run per distinct policy_id.
@RATIONALE Existing records were created before the ValidationRun model existed.
Grouping by policy_id avoids creating excessive runs for what is essentially
one historic execution batch per policy. The run status is set to 'completed'
to avoid confusing downstream consumers that filter on 'running'.
Revision ID: c9d8e7f6a5b4
Revises: 2df63b7ce038
Create Date: 2026-05-31 12:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = "c9d8e7f6a5b4"
down_revision: str | Sequence[str] | None = "2df63b7ce038"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _table_exists(table_name: str) -> bool:
conn = op.get_bind()
inspector = inspect(conn)
return table_name in inspector.get_table_names()
def upgrade() -> None:
"""Upgrade schema to v2 validation models."""
_create_validation_sources()
_create_validation_runs()
_extend_validation_policies()
_extend_validation_results()
_backfill_validation_runs()
def downgrade() -> None:
"""Downgrade schema by removing v2 additions in reverse order."""
# Drop FK constraints from llm_validation_results
op.drop_constraint(
"fk_llm_validation_results_source_id",
"llm_validation_results",
type_="foreignkey",
)
op.drop_constraint(
"fk_llm_validation_results_run_id",
"llm_validation_results",
type_="foreignkey",
)
op.drop_index(
op.f("ix_llm_validation_results_run_id"),
table_name="llm_validation_results",
)
# Remove v2 columns from llm_validation_results
_v2_result_columns = [
"screenshot_paths",
"timings",
"token_usage",
"logs_sent_to_llm",
"tab_screenshots",
"chart_data_results",
"dataset_health",
"execution_path",
"source_id",
"run_id",
]
for col in _v2_result_columns:
op.drop_column("llm_validation_results", col)
# Remove columns + index from validation_policies
op.drop_index(
op.f("ix_validation_policies_provider_id"),
table_name="validation_policies",
)
_v2_policy_columns = [
"source_snapshot",
"policy_dashboard_concurrency_limit",
"llm_batch_size",
"execute_chart_data",
"logs_enabled",
"screenshot_enabled",
"prompt_template",
]
for col in _v2_policy_columns:
op.drop_column("validation_policies", col)
# Drop validation_runs table (index first, then table)
op.drop_index(
op.f("ix_validation_runs_policy_id"),
table_name="validation_runs",
)
op.drop_table("validation_runs")
# Drop validation_sources table (index first, then table)
op.drop_index(
op.f("ix_validation_sources_policy_id"),
table_name="validation_sources",
)
op.drop_table("validation_sources")
# ---------------------------------------------------------------------------
# Upgrade helpers (grouped for readability)
# ---------------------------------------------------------------------------
def _create_validation_sources() -> None:
"""Create the validation_sources table (no FK dependencies)."""
if not _table_exists("validation_policies"):
return
op.create_table(
"validation_sources",
sa.Column("id", sa.String(), nullable=False),
sa.Column("policy_id", sa.String(), nullable=False),
sa.Column("type", sa.String(), nullable=False),
sa.Column("value", sa.String(), nullable=False),
sa.Column("parsed_context", sa.JSON(), nullable=True),
sa.Column(
"status", sa.String(), nullable=False, server_default="valid"
),
sa.Column("resolved_dashboard_id", sa.String(), nullable=True),
sa.Column("title", sa.String(), nullable=True),
sa.Column("last_error", sa.String(), nullable=True),
sa.Column("last_checked_at", sa.DateTime(), nullable=True),
sa.Column(
"created_at",
sa.DateTime(),
nullable=False,
server_default=sa.func.now(),
),
sa.ForeignKeyConstraint(
["policy_id"],
["validation_policies.id"],
),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
op.f("ix_validation_sources_policy_id"),
"validation_sources",
["policy_id"],
)
def _create_validation_runs() -> None:
"""Create the validation_runs table."""
if not _table_exists("validation_policies"):
return
op.create_table(
"validation_runs",
sa.Column("id", sa.String(), nullable=False),
sa.Column("policy_id", sa.String(), nullable=False),
sa.Column("task_id", sa.String(), nullable=True),
sa.Column(
"started_at",
sa.DateTime(),
nullable=False,
server_default=sa.func.now(),
),
sa.Column("finished_at", sa.DateTime(), nullable=True),
sa.Column(
"trigger", sa.String(), nullable=False, server_default="manual"
),
sa.Column(
"status", sa.String(), nullable=False, server_default="running"
),
sa.Column(
"dashboard_count",
sa.Integer(),
nullable=False,
server_default="0",
),
sa.Column(
"pass_count", sa.Integer(), nullable=False, server_default="0"
),
sa.Column(
"warn_count", sa.Integer(), nullable=False, server_default="0"
),
sa.Column(
"fail_count", sa.Integer(), nullable=False, server_default="0"
),
sa.Column(
"unknown_count", sa.Integer(), nullable=False, server_default="0"
),
sa.Column(
"created_at",
sa.DateTime(),
nullable=False,
server_default=sa.func.now(),
),
sa.ForeignKeyConstraint(
["policy_id"],
["validation_policies.id"],
),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
op.f("ix_validation_runs_policy_id"),
"validation_runs",
["policy_id"],
)
def _extend_validation_policies() -> None:
"""Add v2 columns and index to validation_policies."""
if not _table_exists("validation_policies"):
return
op.add_column(
"validation_policies",
sa.Column("prompt_template", sa.Text(), nullable=True),
)
op.add_column(
"validation_policies",
sa.Column(
"screenshot_enabled",
sa.Boolean(),
nullable=False,
server_default="true",
),
)
op.add_column(
"validation_policies",
sa.Column(
"logs_enabled",
sa.Boolean(),
nullable=False,
server_default="true",
),
)
op.add_column(
"validation_policies",
sa.Column(
"execute_chart_data",
sa.Boolean(),
nullable=False,
server_default="false",
),
)
op.add_column(
"validation_policies",
sa.Column(
"llm_batch_size",
sa.Integer(),
nullable=False,
server_default="1",
),
)
op.add_column(
"validation_policies",
sa.Column(
"policy_dashboard_concurrency_limit",
sa.Integer(),
nullable=False,
server_default="3",
),
)
op.add_column(
"validation_policies",
sa.Column("source_snapshot", sa.JSON(), nullable=True),
)
op.create_index(
op.f("ix_validation_policies_provider_id"),
"validation_policies",
["provider_id"],
)
def _extend_validation_results() -> None:
"""Add v2 columns, FKs, and indexes to llm_validation_results."""
if not _table_exists("llm_validation_results"):
return
op.add_column(
"llm_validation_results",
sa.Column(
"run_id",
sa.String(),
sa.ForeignKey("validation_runs.id", name="fk_llm_validation_results_run_id"),
nullable=True,
),
)
op.add_column(
"llm_validation_results",
sa.Column(
"source_id",
sa.String(),
sa.ForeignKey("validation_sources.id", name="fk_llm_validation_results_source_id"),
nullable=True,
),
)
op.add_column(
"llm_validation_results",
sa.Column("execution_path", sa.String(), nullable=True),
)
op.add_column(
"llm_validation_results",
sa.Column("dataset_health", sa.JSON(), nullable=True),
)
op.add_column(
"llm_validation_results",
sa.Column("chart_data_results", sa.JSON(), nullable=True),
)
op.add_column(
"llm_validation_results",
sa.Column("tab_screenshots", sa.JSON(), nullable=True),
)
op.add_column(
"llm_validation_results",
sa.Column("logs_sent_to_llm", sa.JSON(), nullable=True),
)
op.add_column(
"llm_validation_results",
sa.Column("token_usage", sa.JSON(), nullable=True),
)
op.add_column(
"llm_validation_results",
sa.Column("timings", sa.JSON(), nullable=True),
)
op.add_column(
"llm_validation_results",
sa.Column("screenshot_paths", sa.JSON(), nullable=True),
)
op.create_index(
op.f("ix_llm_validation_results_run_id"),
"llm_validation_results",
["run_id"],
)
def _backfill_validation_runs() -> None:
"""Create placeholder ValidationRun entries for existing records.
For each distinct policy_id that has ValidationRecord rows without a
run_id, creates one ValidationRun (status='completed') and links all
matching records to it.
Safe no-op when no such rows exist (e.g. fresh database).
"""
if not _table_exists("llm_validation_results"):
return
conn = op.get_bind()
# Check for existing records that need backfill
result = conn.execute(
sa.text(
"SELECT DISTINCT v.policy_id "
"FROM llm_validation_results v "
"WHERE v.policy_id IS NOT NULL "
"AND v.run_id IS NULL"
)
)
rows = result.fetchall()
if not rows:
return
import uuid
for (policy_id,) in rows:
run_id = str(uuid.uuid4())
conn.execute(
sa.text(
"INSERT INTO validation_runs "
"(id, policy_id, started_at, trigger, status, created_at) "
"VALUES (:run_id, :policy_id, NOW(), 'manual', 'completed', NOW())"
),
{"run_id": run_id, "policy_id": policy_id},
)
conn.execute(
sa.text(
"UPDATE llm_validation_results "
"SET run_id = :run_id "
"WHERE policy_id = :policy_id AND run_id IS NULL"
),
{"run_id": run_id, "policy_id": policy_id},
)

55
backend/alembic/versions/d1e2f3a4b5c6_create_task_records_table.py
View File

@@ -1,55 +0,0 @@
"""create task_records table
Previously task_records was created at runtime by init_db() →
Base.metadata.create_all(bind=tasks_engine). This broke Alembic
migrations that reference task_records (e.g. a5b6c7d8e9f0) — on a
fresh database, the table didn't exist when migrations ran.
This migration brings task_records into the Alembic-managed schema.
create_all() becomes a no-op for this table.
Revision ID: d1e2f3a4b5c6
Revises: c4a3a2f74bfe
Create Date: 2026-06-11 17:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = 'd1e2f3a4b5c6'
down_revision: str | Sequence[str] | None = 'c4a3a2f74bfe'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Create task_records table matching TaskRecord model."""
op.create_table(
'task_records',
sa.Column('id', sa.String(), nullable=False),
sa.Column('type', sa.String(), nullable=False),
sa.Column('status', sa.String(), nullable=False),
sa.Column('environment_id', sa.String(), nullable=True),
sa.Column('started_at', sa.DateTime(timezone=True), nullable=True),
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
sa.Column('logs', sa.JSON(), nullable=True),
sa.Column('error', sa.String(), nullable=True),
sa.Column('result', sa.JSON(), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=True),
sa.Column('params', sa.JSON(), nullable=True),
sa.ForeignKeyConstraint(
['environment_id'],
['environments.id'],
ondelete='SET NULL',
name='task_records_environment_id_fkey',
),
sa.PrimaryKeyConstraint('id'),
)
def downgrade() -> None:
"""Drop task_records table."""
op.drop_table('task_records')

50
backend/alembic/versions/d2e3f4a5b6c7_add_scenario_artifact_attempt_projection.py
View File

@@ -1,50 +0,0 @@
# #region Alembic.ScenarioArtifactAttemptProjection [C:3] [TYPE Module] [SEMANTICS alembic,scenario,artifact,retry,provenance]
# @ingroup Alembic
# @BRIEF Add additive retry-attempt projection fields without rewriting historical artifact rows.
# @RELATION DEPENDS_ON -> [Models.ScenarioExecution.Artifact]
# @INVARIANT Existing evidence defaults to active and remains inspectable; retries retire only
# future step-bound projections, never content/digest audit data.
# @REJECTED Deleting or backfilling artifact content for retry provenance was rejected — legacy
# rows remain valid historical evidence with nullable step/attempt linkage.
"""add active ScenarioArtifact attempt projection for retry closure (044)"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "d2e3f4a5b6c7"
down_revision: str | Sequence[str] | None = "c1d2e3f4a5b6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
# #region Alembic.ScenarioArtifactAttemptProjection.Upgrade [C:4] [TYPE Function] [SEMANTICS alembic,scenario,artifact,retry,additive]
# @BRIEF Add nullable producer linkage and an active projection flag for durable retry evidence.
# @POST Legacy rows are active=true; no historical SHA/ref is changed or removed.
# @SIDE_EFFECT Schema mutation: two nullable provenance columns, active projection flag, invalidation timestamp, and lookup indexes.
def upgrade() -> None:
op.add_column("scenario_artifacts", sa.Column("logical_step_id", sa.String(128), nullable=True))
op.add_column("scenario_artifacts", sa.Column("attempt", sa.Integer(), nullable=True))
op.add_column(
"scenario_artifacts",
sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()),
)
op.add_column("scenario_artifacts", sa.Column("invalidated_at", sa.DateTime(), nullable=True))
op.create_index("ix_scenario_artifacts_logical_step_id", "scenario_artifacts", ["logical_step_id"])
op.create_index("ix_scenario_artifacts_is_active", "scenario_artifacts", ["is_active"])
# #endregion Alembic.ScenarioArtifactAttemptProjection.Upgrade
# #region Alembic.ScenarioArtifactAttemptProjection.Downgrade [C:3] [TYPE Function] [SEMANTICS alembic,scenario,artifact,retry,rollback]
# @BRIEF Remove only additive projection metadata; retained evidence rows and digests are untouched.
def downgrade() -> None:
op.drop_index("ix_scenario_artifacts_is_active", table_name="scenario_artifacts")
op.drop_index("ix_scenario_artifacts_logical_step_id", table_name="scenario_artifacts")
op.drop_column("scenario_artifacts", "invalidated_at")
op.drop_column("scenario_artifacts", "is_active")
op.drop_column("scenario_artifacts", "attempt")
op.drop_column("scenario_artifacts", "logical_step_id")
# #endregion Alembic.ScenarioArtifactAttemptProjection.Downgrade
# #endregion Alembic.ScenarioArtifactAttemptProjection

52
backend/alembic/versions/d4e5f6a7b8c9_add_active_translation_run_guard.py
View File

@@ -1,52 +0,0 @@
# #region Alembic.TranslateActiveRunGuard [C:3] [TYPE Module] [SEMANTICS alembic,migration,translate,concurrency]
# @BRIEF Enforces at most one pending/running translation run per job.
# @RELATION DEPENDS_ON -> [EXT:SQLAlchemy:Alembic]
# @POST Concurrent manual/scheduled triggers cannot create duplicate active runs.
"""add unique active translation run guard
Revision ID: d4e5f6a7b8c9
Revises: c3d4e5f6a7b8
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
revision: str = "d4e5f6a7b8c9"
down_revision: str | Sequence[str] | None = "c3d4e5f6a7b8"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Create a partial unique index on supported application databases."""
bind = op.get_bind()
dialect = bind.dialect.name
if dialect == "postgresql":
op.create_index(
"uq_translation_runs_one_active_per_job",
"translation_runs",
["job_id"],
unique=True,
postgresql_where=sa.text("status IN ('PENDING', 'RUNNING')"),
)
elif dialect == "sqlite":
op.create_index(
"uq_translation_runs_one_active_per_job",
"translation_runs",
["job_id"],
unique=True,
sqlite_where=sa.text("status IN ('PENDING', 'RUNNING')"),
)
def downgrade() -> None:
"""Drop the active-run guard."""
bind = op.get_bind()
if bind.dialect.name in {"postgresql", "sqlite"}:
op.drop_index("uq_translation_runs_one_active_per_job", table_name="translation_runs")
# #endregion Alembic.TranslateActiveRunGuard

40
backend/alembic/versions/dabc97097e0e_add_cache_hits_column_to_translation_.py
View File

@@ -1,40 +0,0 @@
"""add cache_hits column to translation_runs
Revision ID: dabc97097e0e
Revises: ed28d34edde7
Create Date: 2026-06-02 11:54:03.550164
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = 'dabc97097e0e'
down_revision: str | Sequence[str] | None = 'ed28d34edde7'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _table_exists(table_name: str) -> bool:
conn = op.get_bind()
inspector = inspect(conn)
return table_name in inspector.get_table_names()
def upgrade() -> None:
"""Add cache_hits column to translation_runs table."""
if not _table_exists("translation_runs"):
return
op.add_column("translation_runs", sa.Column(
"cache_hits", sa.Integer(), nullable=False, server_default=sa.text("0"),
comment="Number of rows served from translation cache",
))
def downgrade() -> None:
"""Remove cache_hits column from translation_runs table."""
op.drop_column("translation_runs", "cache_hits")

103
backend/alembic/versions/e3a4b5c6d7e8_add_deployment_records.py
View File

@@ -1,103 +0,0 @@
# #region Alembic.AddDeploymentRecords [C:3] [TYPE Module] [SEMANTICS alembic,migration,deployment,versioning]
# @BRIEF Add deployment_records table for version tracking (Phase 0).
# @RELATION DEPENDS_ON -> [Models.Deployment.DeploymentModels]
# @POST Creates deployment dependency tables before adding foreign-key-constrained records.
# @RATIONALE DeploymentEnvironment and GitRepository were historically created by runtime metadata,
# which left a fresh Alembic upgrade without the foreign-key targets required here.
# @REJECTED Relying on Base.metadata.create_all() before Alembic was rejected because production
# startup must be able to initialize its schema through migrations alone.
"""add deployment_records table
Revision ID: e3a4b5c6d7e8
Revises: f2b3c4d5e6f7
Create Date: 2026-07-10 13:30:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import JSON
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "e3a4b5c6d7e8"
down_revision: str | None = "f2b3c4d5e6f7"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_table(
"git_server_configs",
sa.Column("id", sa.String(36), nullable=False),
sa.Column("name", sa.String(255), nullable=False),
sa.Column("provider", sa.String(20), nullable=False),
sa.Column("url", sa.String(255), nullable=False),
sa.Column("pat", sa.String(255), nullable=False),
sa.Column("default_repository", sa.String(255), nullable=True),
sa.Column("default_branch", sa.String(255), nullable=True),
sa.Column("status", sa.String(20), nullable=True),
sa.Column("last_validated", sa.DateTime(), nullable=True),
sa.PrimaryKeyConstraint("id"),
)
op.create_table(
"git_repositories",
sa.Column("id", sa.String(36), nullable=False),
sa.Column("dashboard_id", sa.Integer(), nullable=False),
sa.Column("config_id", sa.String(36), nullable=False),
sa.Column("remote_url", sa.String(255), nullable=False),
sa.Column("local_path", sa.String(255), nullable=False),
sa.Column("current_branch", sa.String(255), nullable=True),
sa.Column("sync_status", sa.String(20), nullable=True),
sa.ForeignKeyConstraint(["config_id"], ["git_server_configs.id"]),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("dashboard_id"),
)
op.create_table(
"deployment_environments",
sa.Column("id", sa.String(36), nullable=False),
sa.Column("name", sa.String(255), nullable=False),
sa.Column("superset_url", sa.String(255), nullable=False),
sa.Column("superset_token", sa.String(255), nullable=False),
sa.Column("is_active", sa.Boolean(), nullable=True),
sa.PrimaryKeyConstraint("id"),
)
op.create_table(
"deployment_records",
sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
sa.Column("repository_id", sa.String(36), sa.ForeignKey("git_repositories.id", ondelete="CASCADE"), nullable=False),
sa.Column("environment_id", sa.String(36), sa.ForeignKey("deployment_environments.id", ondelete="CASCADE"), nullable=False),
sa.Column("commit_hash", sa.String(40), nullable=False),
sa.Column("content_hash", sa.String(64), nullable=False),
sa.Column("deployed_at", sa.DateTime(), nullable=False, server_default=sa.func.now()),
sa.Column("deployed_by", sa.String(255), nullable=True),
sa.Column("status", sa.String(20), nullable=False, server_default="success"),
sa.Column("error_message", sa.Text(), nullable=True),
sa.Column("resources_changed", JSON(), nullable=True),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
op.f("ix_deployment_records_repository_env"),
"deployment_records",
["repository_id", "environment_id"],
unique=False,
)
op.create_index(
op.f("ix_deployment_records_content_hash"),
"deployment_records",
["content_hash"],
unique=False,
)
def downgrade() -> None:
op.drop_index(op.f("ix_deployment_records_content_hash"), table_name="deployment_records")
op.drop_index(op.f("ix_deployment_records_repository_env"), table_name="deployment_records")
op.drop_table("deployment_records")
op.drop_table("deployment_environments")
op.drop_table("git_repositories")
op.drop_table("git_server_configs")
# #endregion Alembic.AddDeploymentRecords

43
backend/alembic/versions/e3f4a5b6c7d8_add_scenario_cancel_drain_deadline.py
View File

@@ -1,43 +0,0 @@
# #region Alembic.ScenarioCancelDrainDeadline [C:3] [TYPE Module] [SEMANTICS alembic,scenario,execution,cancel,drain]
# @ingroup Alembic
# @BRIEF Add nullable persisted cancellation timing so a worker/scheduler can finish a bounded drain after restart.
# @RELATION DEPENDS_ON -> [Models.ScenarioExecution.Run]
# @INVARIANT Existing runs retain null timing fields; only a cancel request pins a deadline.
# @REJECTED Deriving a deadline from mutable target snapshot or an in-memory process timer was rejected.
"""add durable ScenarioRun cancellation drain timing (044)"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "e3f4a5b6c7d8"
down_revision: str | Sequence[str] | None = "d2e3f4a5b6c7"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
# #region Alembic.ScenarioCancelDrainDeadline.Upgrade [C:4] [TYPE Function] [SEMANTICS alembic,scenario,execution,cancel,additive]
# @BRIEF Add nullable request/deadline timestamps and a deadline lookup index.
# @POST Historical ScenarioRuns preserve null timing until explicitly cancelled.
# @SIDE_EFFECT Schema mutation: two nullable lifecycle timestamps and one index.
def upgrade() -> None:
op.add_column("scenario_runs", sa.Column("cancel_requested_at", sa.DateTime(), nullable=True))
op.add_column("scenario_runs", sa.Column("cancel_drain_deadline_at", sa.DateTime(), nullable=True))
op.create_index(
"ix_scenario_runs_cancel_drain_deadline_at",
"scenario_runs",
["cancel_drain_deadline_at"],
)
# #endregion Alembic.ScenarioCancelDrainDeadline.Upgrade
# #region Alembic.ScenarioCancelDrainDeadline.Downgrade [C:3] [TYPE Function] [SEMANTICS alembic,scenario,execution,cancel,rollback]
# @BRIEF Remove only additive timing metadata; no run/step/evidence history is altered.
def downgrade() -> None:
op.drop_index("ix_scenario_runs_cancel_drain_deadline_at", table_name="scenario_runs")
op.drop_column("scenario_runs", "cancel_drain_deadline_at")
op.drop_column("scenario_runs", "cancel_requested_at")
# #endregion Alembic.ScenarioCancelDrainDeadline.Downgrade
# #endregion Alembic.ScenarioCancelDrainDeadline

44
backend/alembic/versions/e4f5a6b7c8d9_canonical_task_log_events.py
View File

@@ -1,44 +0,0 @@
"""replace legacy task log fields with canonical Molecular CoT fields"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "e4f5a6b7c8d9"
down_revision: str | Sequence[str] | None = "e3f4a5b6c7d8"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.add_column("task_logs", sa.Column("trace_id", sa.String(64), nullable=False, server_default=""))
op.add_column("task_logs", sa.Column("span_id", sa.String(128), nullable=True))
op.add_column("task_logs", sa.Column("src", sa.String(255), nullable=False, server_default="task.system"))
op.add_column("task_logs", sa.Column("marker", sa.String(16), nullable=False, server_default="REASON"))
op.add_column("task_logs", sa.Column("intent", sa.Text(), nullable=False, server_default=""))
op.add_column("task_logs", sa.Column("payload", sa.JSON(), nullable=True))
op.add_column("task_logs", sa.Column("error", sa.Text(), nullable=True))
bind = op.get_bind()
bind.execute(sa.text("""
UPDATE task_logs
SET src = COALESCE(source, 'task.system'),
intent = COALESCE(message, ''),
marker = CASE WHEN UPPER(level) IN ('WARNING', 'ERROR') THEN 'EXPLORE' ELSE 'REASON' END,
error = CASE WHEN UPPER(level) IN ('WARNING', 'ERROR') THEN COALESCE(message, 'Task event') ELSE NULL END
"""))
if bind.dialect.name == "postgresql":
bind.execute(sa.text("UPDATE task_logs SET payload = metadata_json::json WHERE metadata_json IS NOT NULL"))
elif bind.dialect.name == "sqlite":
bind.execute(sa.text("UPDATE task_logs SET payload = json(metadata_json) WHERE metadata_json IS NOT NULL AND json_valid(metadata_json)"))
op.create_index("ix_task_logs_task_src", "task_logs", ["task_id", "src"])
op.drop_index("ix_task_logs_task_source", table_name="task_logs")
op.drop_column("task_logs", "source")
op.drop_column("task_logs", "message")
op.drop_column("task_logs", "metadata_json")
for column in ("trace_id", "src", "marker", "intent"):
op.alter_column("task_logs", column, server_default=None)
def downgrade() -> None:
raise NotImplementedError("Canonical task log migration is a breaking migration")

40
backend/alembic/versions/e5f4d3c2b1a_add_policy_id_to_validation_results.py
View File

@@ -1,40 +0,0 @@
"""Add policy_id column to llm_validation_results
Revision ID: e5f4d3c2b1a
Revises: a7b1c2d3e4f5
Create Date: 2026-05-21 12:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = "e5f4d3c2b1a"
down_revision: str | Sequence[str] | None = "a7b1c2d3e4f5"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _table_exists(table_name: str) -> bool:
conn = op.get_bind()
inspector = inspect(conn)
return table_name in inspector.get_table_names()
def upgrade() -> None:
"""Add policy_id column to llm_validation_results as nullable indexed."""
if not _table_exists("llm_validation_results"):
return
op.add_column(
"llm_validation_results",
sa.Column("policy_id", sa.String(), nullable=True, index=True),
)
def downgrade() -> None:
"""Remove policy_id column from llm_validation_results."""
op.drop_column("llm_validation_results", "policy_id")

49
backend/alembic/versions/e6f7a8b9c0d1_add_translation_run_observability_metrics.py
View File

@@ -1,49 +0,0 @@
# #region Alembic.AddTranslationRunObservabilityMetrics [C:3] [TYPE Module] [SEMANTICS alembic,translate,metrics]
# @defgroup Alembic Persist independent source, translation, and insert metrics for translation runs.
"""add translation run observability metrics
Revision ID: e6f7a8b9c0d1
Revises: f5e6d7c8b9a0
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "e6f7a8b9c0d1"
down_revision: str | Sequence[str] | None = "f5e6d7c8b9a0"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
# #region Alembic.AddTranslationRunObservabilityMetrics.Upgrade [C:3] [TYPE Function] [SEMANTICS alembic,translate,metrics]
# @ingroup Alembic
# @BRIEF Add nullable run metrics so historical runs remain explicitly unknown.
def upgrade() -> None:
op.add_column("translation_runs", sa.Column("source_records_read", sa.Integer(), nullable=True))
op.add_column("translation_runs", sa.Column("eligible_records", sa.Integer(), nullable=True))
op.add_column("translation_runs", sa.Column("translated_records", sa.Integer(), nullable=True))
op.add_column("translation_runs", sa.Column("same_language_skipped_records", sa.Integer(), nullable=True))
op.add_column("translation_runs", sa.Column("insert_rows_prepared", sa.Integer(), nullable=True))
op.add_column("translation_runs", sa.Column("insert_rows_affected", sa.Integer(), nullable=True))
# #endregion Alembic.AddTranslationRunObservabilityMetrics.Upgrade
# #region Alembic.AddTranslationRunObservabilityMetrics.Downgrade [C:2] [TYPE Function] [SEMANTICS alembic,translate,metrics]
# @ingroup Alembic
# @BRIEF Remove translation-run observability metrics.
def downgrade() -> None:
op.drop_column("translation_runs", "insert_rows_affected")
op.drop_column("translation_runs", "insert_rows_prepared")
op.drop_column("translation_runs", "same_language_skipped_records")
op.drop_column("translation_runs", "translated_records")
op.drop_column("translation_runs", "eligible_records")
op.drop_column("translation_runs", "source_records_read")
# #endregion Alembic.AddTranslationRunObservabilityMetrics.Downgrade
# #endregion Alembic.AddTranslationRunObservabilityMetrics

35
backend/alembic/versions/ed28d34edde7_add_max_images_to_llm_providers.py
View File

@@ -1,35 +0,0 @@
"""add_max_images_to_llm_providers
Revision ID: ed28d34edde7
Revises: 7703bbc038bd
Create Date: 2026-05-31 22:19:21.922928
"""
from typing import Sequence, Union
from alembic import op
from sqlalchemy import inspect
import sqlalchemy as sa
revision: str = 'ed28d34edde7'
down_revision: Union[str, Sequence[str], None] = '7703bbc038bd'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# The llm_providers table is defined in the ORM model and created by
# Base.metadata.create_all() at app startup, not via migrations.
# If the table doesn't exist yet (fresh DB), skip — the model definition
# already includes max_images, so create_all() will create it with the column.
conn = op.get_bind()
inspector = inspect(conn)
if "llm_providers" not in inspector.get_table_names():
return
op.add_column('llm_providers', sa.Column('max_images', sa.Integer(), nullable=True))
def downgrade() -> None:
op.drop_column('llm_providers', 'max_images')

392
backend/alembic/versions/ed310b33f02c_multi_language_translation_tables.py
View File

@@ -1,392 +0,0 @@
"""multi-language translation tables
Revision ID: ed310b33f02c
Revises:
Create Date: 2026-05-14 15:35:03.167031
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = 'ed310b33f02c'
down_revision: str | Sequence[str] | None = None
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('environments',
sa.Column('id', sa.String(), nullable=False),
sa.Column('name', sa.String(), nullable=False),
sa.Column('url', sa.String(), nullable=False),
sa.Column('credentials_id', sa.String(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('terminology_dictionaries',
sa.Column('id', sa.String(), nullable=False),
sa.Column('name', sa.String(), nullable=False),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('source_dialect', sa.String(), nullable=False),
sa.Column('target_dialect', sa.String(), nullable=False),
sa.Column('source_language', sa.String(), nullable=True, comment='[DEPRECATED: use per-entry source_language]'),
sa.Column('target_language', sa.String(), nullable=True, comment='[DEPRECATED: use per-entry target_language]'),
sa.Column('is_active', sa.Boolean(), nullable=True),
sa.Column('created_by', sa.String(), nullable=True),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('translation_jobs',
sa.Column('id', sa.String(), nullable=False),
sa.Column('name', sa.String(), nullable=False),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('source_dialect', sa.String(), nullable=False),
sa.Column('target_dialect', sa.String(), nullable=False),
sa.Column('database_dialect', sa.String(), nullable=True, comment='Detected dialect from Superset connection at save time'),
sa.Column('status', sa.String(), nullable=False),
sa.Column('source_datasource_id', sa.String(), nullable=True, comment='Superset datasource ID'),
sa.Column('source_table', sa.String(), nullable=True, comment='Source table name resolved from datasource'),
sa.Column('target_schema', sa.String(), nullable=True, comment='Target table schema'),
sa.Column('target_table', sa.String(), nullable=True, comment='Target table name'),
sa.Column('source_key_cols', sa.JSON(), nullable=True, comment='Source key column names for composite key'),
sa.Column('target_key_cols', sa.JSON(), nullable=True, comment='Target key column names for composite key'),
sa.Column('translation_column', sa.String(), nullable=True, comment='Source column whose values will be translated'),
sa.Column('target_column', sa.String(), nullable=True, comment='Target column for translated output (defaults to translation_column)'),
sa.Column('context_columns', sa.JSON(), nullable=True, comment='Context column names included in LLM prompt'),
sa.Column('target_language', sa.String(), nullable=True, comment='Target language code (e.g. en, ru) [DEPRECATED: use target_languages]'),
sa.Column('source_language', sa.String(), nullable=True, comment='Fallback source language hint [DEPRECATED: auto-detected per row]'),
sa.Column('target_languages', sa.JSON(), nullable=True, comment='List of BCP-47 target language codes (multi-language support)'),
sa.Column('provider_id', sa.String(), nullable=True, comment='LLM provider ID'),
sa.Column('batch_size', sa.Integer(), nullable=False, comment='Records per batch'),
sa.Column('upsert_strategy', sa.String(), nullable=False, comment='MERGE, INSERT, UPDATE'),
sa.Column('environment_id', sa.String(), nullable=True, comment='Superset environment ID for datasource access'),
sa.Column('target_database_id', sa.String(), nullable=True, comment='Superset database ID for SQL Lab insert target'),
sa.Column('created_by', sa.String(), nullable=True),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('database_mappings',
sa.Column('id', sa.String(), nullable=False),
sa.Column('source_env_id', sa.String(), nullable=False),
sa.Column('target_env_id', sa.String(), nullable=False),
sa.Column('source_db_uuid', sa.String(), nullable=False),
sa.Column('target_db_uuid', sa.String(), nullable=False),
sa.Column('source_db_name', sa.String(), nullable=False),
sa.Column('target_db_name', sa.String(), nullable=False),
sa.Column('engine', sa.String(), nullable=True),
sa.ForeignKeyConstraint(['source_env_id'], ['environments.id'], ),
sa.ForeignKeyConstraint(['target_env_id'], ['environments.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('dictionary_entries',
sa.Column('id', sa.String(), nullable=False),
sa.Column('dictionary_id', sa.String(), nullable=False),
sa.Column('source_term', sa.String(), nullable=False),
sa.Column('source_term_normalized', sa.String(), nullable=False),
sa.Column('target_term', sa.String(), nullable=False),
sa.Column('source_language', sa.String(), nullable=False, comment='BCP-47 source language code'),
sa.Column('target_language', sa.String(), nullable=False, comment='BCP-47 target language code'),
sa.Column('context_notes', sa.Text(), nullable=True),
sa.Column('context_data', sa.JSON(), nullable=True, comment='Structured context for term usage'),
sa.Column('usage_notes', sa.Text(), nullable=True, comment='Usage guidance for the term mapping'),
sa.Column('has_context', sa.Boolean(), nullable=True, comment='Whether context_data is populated'),
sa.Column('context_source', sa.String(), nullable=True, comment='auto|auto_with_edits|manual|bulk'),
sa.Column('origin_source_language', sa.String(), nullable=True, comment='Original source language of the term'),
sa.Column('origin_run_id', sa.String(), nullable=True, comment='Run ID from which this correction originated'),
sa.Column('origin_row_key', sa.String(), nullable=True, comment='Row key within the run that triggered this correction'),
sa.Column('origin_user_id', sa.String(), nullable=True, comment='User who submitted the correction'),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['dictionary_id'], ['terminology_dictionaries.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('dictionary_id', 'source_term_normalized', 'source_language', 'target_language', name='uq_dict_source_term_lang')
)
op.create_index('idx_dict_entry_lang', 'dictionary_entries', ['source_language', 'target_language'], unique=False)
op.create_index('idx_dict_has_context', 'dictionary_entries', ['has_context'], unique=False)
op.create_index(op.f('ix_dictionary_entries_dictionary_id'), 'dictionary_entries', ['dictionary_id'], unique=False)
op.create_table('migration_jobs',
sa.Column('id', sa.String(), nullable=False),
sa.Column('source_env_id', sa.String(), nullable=False),
sa.Column('target_env_id', sa.String(), nullable=False),
sa.Column('status', sa.Enum('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', 'AWAITING_MAPPING', name='migrationstatus'), nullable=True),
sa.Column('replace_db', sa.Boolean(), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
sa.ForeignKeyConstraint(['source_env_id'], ['environments.id'], ),
sa.ForeignKeyConstraint(['target_env_id'], ['environments.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('resource_mappings',
sa.Column('id', sa.String(), nullable=False),
sa.Column('environment_id', sa.String(), nullable=False),
sa.Column('resource_type', sa.Enum('CHART', 'DATASET', 'DASHBOARD', name='resourcetype'), nullable=False),
sa.Column('uuid', sa.String(), nullable=False),
sa.Column('remote_integer_id', sa.String(), nullable=False),
sa.Column('resource_name', sa.String(), nullable=True),
sa.Column('last_synced_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
sa.ForeignKeyConstraint(['environment_id'], ['environments.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('translation_job_dictionaries',
sa.Column('id', sa.String(), nullable=False),
sa.Column('job_id', sa.String(), nullable=False),
sa.Column('dictionary_id', sa.String(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['dictionary_id'], ['terminology_dictionaries.id'], ),
sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('job_id', 'dictionary_id', name='uq_job_dictionary')
)
op.create_index(op.f('ix_translation_job_dictionaries_dictionary_id'), 'translation_job_dictionaries', ['dictionary_id'], unique=False)
op.create_index(op.f('ix_translation_job_dictionaries_job_id'), 'translation_job_dictionaries', ['job_id'], unique=False)
op.create_table('translation_runs',
sa.Column('id', sa.String(), nullable=False),
sa.Column('job_id', sa.String(), nullable=False),
sa.Column('status', sa.String(), nullable=False),
sa.Column('trigger_type', sa.String(), nullable=True, comment='manual, scheduled, retry, baseline_expired'),
sa.Column('started_at', sa.DateTime(), nullable=True),
sa.Column('completed_at', sa.DateTime(), nullable=True),
sa.Column('error_message', sa.Text(), nullable=True),
sa.Column('total_records', sa.Integer(), nullable=True),
sa.Column('successful_records', sa.Integer(), nullable=True),
sa.Column('failed_records', sa.Integer(), nullable=True),
sa.Column('skipped_records', sa.Integer(), nullable=True),
sa.Column('insert_status', sa.String(), nullable=True, comment='Status of the Superset insert/update operation'),
sa.Column('superset_execution_id', sa.String(), nullable=True, comment='Superset execution/task ID'),
sa.Column('superset_execution_log', sa.JSON(), nullable=True, comment='Superset execution log output'),
sa.Column('config_snapshot', sa.JSON(), nullable=True, comment='Snapshot of job config at run creation time'),
sa.Column('key_hash', sa.String(), nullable=True, comment='Hash of source key fields for dedup'),
sa.Column('config_hash', sa.String(), nullable=True, comment='Hash of translation configuration state'),
sa.Column('dict_snapshot_hash', sa.String(), nullable=True, comment='Hash of dictionary state at run time'),
sa.Column('created_by', sa.String(), nullable=True),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_translation_runs_job_id'), 'translation_runs', ['job_id'], unique=False)
op.create_table('translation_schedules',
sa.Column('id', sa.String(), nullable=False),
sa.Column('job_id', sa.String(), nullable=False),
sa.Column('cron_expression', sa.String(), nullable=False),
sa.Column('timezone', sa.String(), nullable=False),
sa.Column('is_active', sa.Boolean(), nullable=True),
sa.Column('last_run_at', sa.DateTime(), nullable=True),
sa.Column('next_run_at', sa.DateTime(), nullable=True),
sa.Column('execution_mode', sa.String(), nullable=False, comment='full, new_key_only'),
sa.Column('created_by', sa.String(), nullable=True),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_translation_schedules_job_id'), 'translation_schedules', ['job_id'], unique=False)
op.create_table('translation_batches',
sa.Column('id', sa.String(), nullable=False),
sa.Column('run_id', sa.String(), nullable=False),
sa.Column('batch_index', sa.Integer(), nullable=False),
sa.Column('status', sa.String(), nullable=False),
sa.Column('total_records', sa.Integer(), nullable=True),
sa.Column('successful_records', sa.Integer(), nullable=True),
sa.Column('failed_records', sa.Integer(), nullable=True),
sa.Column('started_at', sa.DateTime(), nullable=True),
sa.Column('completed_at', sa.DateTime(), nullable=True),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_translation_batches_run_id'), 'translation_batches', ['run_id'], unique=False)
op.create_table('translation_events',
sa.Column('id', sa.String(), nullable=False),
sa.Column('job_id', sa.String(), nullable=False),
sa.Column('run_id', sa.String(), nullable=True),
sa.Column('event_type', sa.String(), nullable=False),
sa.Column('event_data', sa.JSON(), nullable=True),
sa.Column('created_by', sa.String(), nullable=True),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ),
sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_translation_events_job_id'), 'translation_events', ['job_id'], unique=False)
op.create_index(op.f('ix_translation_events_run_id'), 'translation_events', ['run_id'], unique=False)
op.create_table('translation_metric_snapshots',
sa.Column('id', sa.String(), nullable=False),
sa.Column('job_id', sa.String(), nullable=False),
sa.Column('run_id', sa.String(), nullable=True),
sa.Column('key_hash', sa.String(), nullable=False, comment='Hash of dimension key fields for aggregation'),
sa.Column('config_hash', sa.String(), nullable=True, comment='Hash of translation configuration state'),
sa.Column('dict_snapshot_hash', sa.String(), nullable=True, comment='Hash of dictionary state at capture time'),
sa.Column('covers_events_before', sa.DateTime(), nullable=True, comment='Indicates snapshot covers events before this timestamp'),
sa.Column('total_jobs', sa.Integer(), nullable=True),
sa.Column('total_runs', sa.Integer(), nullable=True),
sa.Column('total_records', sa.Integer(), nullable=True),
sa.Column('successful_records', sa.Integer(), nullable=True),
sa.Column('failed_records', sa.Integer(), nullable=True),
sa.Column('skipped_records', sa.Integer(), nullable=True),
sa.Column('avg_duration_ms', sa.Integer(), nullable=True),
sa.Column('p50_duration_ms', sa.Integer(), nullable=True),
sa.Column('p95_duration_ms', sa.Integer(), nullable=True),
sa.Column('p99_duration_ms', sa.Integer(), nullable=True),
sa.Column('snapshot_date', sa.DateTime(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ),
sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index('ix_metric_snapshots_job_date', 'translation_metric_snapshots', ['job_id', 'snapshot_date'], unique=False)
op.create_index(op.f('ix_translation_metric_snapshots_job_id'), 'translation_metric_snapshots', ['job_id'], unique=False)
op.create_index(op.f('ix_translation_metric_snapshots_run_id'), 'translation_metric_snapshots', ['run_id'], unique=False)
op.create_table('translation_preview_sessions',
sa.Column('id', sa.String(), nullable=False),
sa.Column('job_id', sa.String(), nullable=False),
sa.Column('run_id', sa.String(), nullable=True),
sa.Column('status', sa.String(), nullable=False),
sa.Column('created_by', sa.String(), nullable=True),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('expires_at', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ),
sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_translation_preview_sessions_job_id'), 'translation_preview_sessions', ['job_id'], unique=False)
op.create_table('translation_run_language_stats',
sa.Column('id', sa.String(), nullable=False),
sa.Column('run_id', sa.String(), nullable=False),
sa.Column('language_code', sa.String(), nullable=False, comment='BCP-47 language code'),
sa.Column('total_rows', sa.Integer(), nullable=True),
sa.Column('translated_rows', sa.Integer(), nullable=True),
sa.Column('failed_rows', sa.Integer(), nullable=True),
sa.Column('skipped_rows', sa.Integer(), nullable=True),
sa.Column('token_count', sa.Integer(), nullable=True),
sa.Column('estimated_cost', sa.Float(), nullable=True),
sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('run_id', 'language_code', name='uq_run_language')
)
op.create_index('idx_rls_run', 'translation_run_language_stats', ['run_id'], unique=False)
op.create_table('translation_preview_records',
sa.Column('id', sa.String(), nullable=False),
sa.Column('session_id', sa.String(), nullable=False),
sa.Column('source_sql', sa.Text(), nullable=True),
sa.Column('target_sql', sa.Text(), nullable=True),
sa.Column('source_object_type', sa.String(), nullable=True),
sa.Column('source_object_id', sa.String(), nullable=True),
sa.Column('source_object_name', sa.String(), nullable=True),
sa.Column('source_data', sa.JSON(), nullable=True, comment='Original source row key columns for upsert matching'),
sa.Column('status', sa.String(), nullable=False),
sa.Column('feedback', sa.Text(), nullable=True),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['session_id'], ['translation_preview_sessions.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_translation_preview_records_session_id'), 'translation_preview_records', ['session_id'], unique=False)
op.create_table('translation_records',
sa.Column('id', sa.String(), nullable=False),
sa.Column('batch_id', sa.String(), nullable=False),
sa.Column('run_id', sa.String(), nullable=False),
sa.Column('source_sql', sa.Text(), nullable=True),
sa.Column('target_sql', sa.Text(), nullable=True),
sa.Column('source_object_type', sa.String(), nullable=True),
sa.Column('source_object_id', sa.String(), nullable=True),
sa.Column('source_object_name', sa.String(), nullable=True),
sa.Column('source_data', sa.JSON(), nullable=True, comment='Original source row key columns for upsert matching'),
sa.Column('status', sa.String(), nullable=False),
sa.Column('error_message', sa.Text(), nullable=True),
sa.Column('token_count_input', sa.Integer(), nullable=True),
sa.Column('token_count_output', sa.Integer(), nullable=True),
sa.Column('translation_duration_ms', sa.Integer(), nullable=True),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('llm_translation', sa.Text(), nullable=True, comment='[DEPRECATED: use TranslationLanguage]'),
sa.Column('user_edit', sa.Text(), nullable=True, comment='[DEPRECATED: use TranslationLanguage]'),
sa.Column('final_value', sa.Text(), nullable=True, comment='[DEPRECATED: use TranslationLanguage]'),
sa.ForeignKeyConstraint(['batch_id'], ['translation_batches.id'], ),
sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_translation_records_batch_id'), 'translation_records', ['batch_id'], unique=False)
op.create_index(op.f('ix_translation_records_run_id'), 'translation_records', ['run_id'], unique=False)
op.create_index('ix_translation_records_run_status', 'translation_records', ['run_id', 'status'], unique=False)
op.create_table('translation_languages',
sa.Column('id', sa.String(), nullable=False),
sa.Column('record_id', sa.String(), nullable=False),
sa.Column('language_code', sa.String(), nullable=False, comment='BCP-47 language code'),
sa.Column('source_language_detected', sa.String(), nullable=True, comment="BCP-47 or 'und' for undetermined"),
sa.Column('translated_value', sa.Text(), nullable=True, comment='LLM-generated translation'),
sa.Column('user_edit', sa.Text(), nullable=True, comment='User-edited translation'),
sa.Column('final_value', sa.Text(), nullable=True, comment='Final resolved value (translated or user edit)'),
sa.Column('status', sa.String(), nullable=True, comment='pending|translated|approved|edited|rejected|failed|skipped'),
sa.Column('error_message', sa.Text(), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=True),
sa.ForeignKeyConstraint(['record_id'], ['translation_records.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('record_id', 'language_code', name='uq_record_language')
)
op.create_index('idx_tl_language', 'translation_languages', ['language_code'], unique=False)
op.create_index('idx_tl_record_lang', 'translation_languages', ['record_id', 'language_code'], unique=False)
op.create_table('translation_preview_languages',
sa.Column('id', sa.String(), nullable=False),
sa.Column('preview_record_id', sa.String(), nullable=False),
sa.Column('language_code', sa.String(), nullable=False, comment='BCP-47 language code'),
sa.Column('source_language_detected', sa.String(), nullable=True, comment="BCP-47 or 'und'"),
sa.Column('translated_value', sa.Text(), nullable=True),
sa.Column('user_edit', sa.Text(), nullable=True),
sa.Column('final_value', sa.Text(), nullable=True),
sa.Column('status', sa.String(), nullable=True, comment='pending|approved|edited|rejected'),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=True),
sa.ForeignKeyConstraint(['preview_record_id'], ['translation_preview_records.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('preview_record_id', 'language_code', name='uq_preview_record_language')
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('translation_preview_languages')
op.drop_index('idx_tl_record_lang', table_name='translation_languages')
op.drop_index('idx_tl_language', table_name='translation_languages')
op.drop_table('translation_languages')
op.drop_index('ix_translation_records_run_status', table_name='translation_records')
op.drop_index(op.f('ix_translation_records_run_id'), table_name='translation_records')
op.drop_index(op.f('ix_translation_records_batch_id'), table_name='translation_records')
op.drop_table('translation_records')
op.drop_index(op.f('ix_translation_preview_records_session_id'), table_name='translation_preview_records')
op.drop_table('translation_preview_records')
op.drop_index('idx_rls_run', table_name='translation_run_language_stats')
op.drop_table('translation_run_language_stats')
op.drop_index(op.f('ix_translation_preview_sessions_job_id'), table_name='translation_preview_sessions')
op.drop_table('translation_preview_sessions')
op.drop_index(op.f('ix_translation_metric_snapshots_run_id'), table_name='translation_metric_snapshots')
op.drop_index(op.f('ix_translation_metric_snapshots_job_id'), table_name='translation_metric_snapshots')
op.drop_index('ix_metric_snapshots_job_date', table_name='translation_metric_snapshots')
op.drop_table('translation_metric_snapshots')
op.drop_index(op.f('ix_translation_events_run_id'), table_name='translation_events')
op.drop_index(op.f('ix_translation_events_job_id'), table_name='translation_events')
op.drop_table('translation_events')
op.drop_index(op.f('ix_translation_batches_run_id'), table_name='translation_batches')
op.drop_table('translation_batches')
op.drop_index(op.f('ix_translation_schedules_job_id'), table_name='translation_schedules')
op.drop_table('translation_schedules')
op.drop_index(op.f('ix_translation_runs_job_id'), table_name='translation_runs')
op.drop_table('translation_runs')
op.drop_index(op.f('ix_translation_job_dictionaries_job_id'), table_name='translation_job_dictionaries')
op.drop_index(op.f('ix_translation_job_dictionaries_dictionary_id'), table_name='translation_job_dictionaries')
op.drop_table('translation_job_dictionaries')
op.drop_table('resource_mappings')
op.drop_table('migration_jobs')
op.drop_index(op.f('ix_dictionary_entries_dictionary_id'), table_name='dictionary_entries')
op.drop_index('idx_dict_has_context', table_name='dictionary_entries')
op.drop_index('idx_dict_entry_lang', table_name='dictionary_entries')
op.drop_table('dictionary_entries')
op.drop_table('database_mappings')
op.drop_table('translation_jobs')
op.drop_table('terminology_dictionaries')
op.drop_table('environments')
# ### end Alembic commands ###

74
backend/alembic/versions/f0e9d8c7b6a5_cascade_ondelete_all_env_fks.py
View File

@@ -1,74 +0,0 @@
"""cascade ondelete for all remaining environments FK references
Revision ID: f0e9d8c7b6a5
Revises: a5b6c7d8e9f0, e5f4d3c2b1a
Create Date: 2026-05-21 19:00:00.000000
@RATIONALE The dataset_review_sessions table is created at runtime by
init_db() → Base.metadata.create_all(), not by any Alembic migration.
On a fresh database, it does not exist when Alembic runs (entrypoint
runs alembic upgrade head BEFORE the backend starts). The model already
defines the FK with ondelete='CASCADE', so on fresh databases the FK is
correct. This migration only needs to alter the FK on databases that
were upgraded from before the FK had CASCADE.
Guard: skip dataset_review_sessions FK if the table doesn't exist.
"""
from collections.abc import Sequence
from alembic import op
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = 'f0e9d8c7b6a5'
down_revision: str | Sequence[str] | None = ('a5b6c7d8e9f0', 'e5f4d3c2b1a')
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
FK_DEFS = [
('resource_mappings', 'resource_mappings_environment_id_fkey', 'environment_id'),
('database_mappings', 'database_mappings_source_env_id_fkey', 'source_env_id'),
('database_mappings', 'database_mappings_target_env_id_fkey', 'target_env_id'),
('migration_jobs', 'migration_jobs_source_env_id_fkey', 'source_env_id'),
('migration_jobs', 'migration_jobs_target_env_id_fkey', 'target_env_id'),
('dataset_review_sessions', 'dataset_review_sessions_environment_id_fkey', 'environment_id'),
]
def _table_exists(table: str) -> bool:
"""Check if a table exists in the current database connection."""
conn = op.get_bind()
inspector = inspect(conn)
return inspector.has_table(table)
def upgrade() -> None:
"""Upgrade schema."""
for table, constraint_name, column in FK_DEFS:
if not _table_exists(table):
continue
op.drop_constraint(constraint_name, table, type_='foreignkey')
op.create_foreign_key(
constraint_name,
table,
'environments',
[column],
['id'],
ondelete='CASCADE',
)
def downgrade() -> None:
"""Downgrade schema."""
for table, constraint_name, column in reversed(FK_DEFS):
if not _table_exists(table):
continue
op.drop_constraint(constraint_name, table, type_='foreignkey')
op.create_foreign_key(
constraint_name,
table,
'environments',
[column],
['id'],
)

44
backend/alembic/versions/f1a2b3c4d5e6_drop_dict_dialect_columns.py
View File

@@ -1,44 +0,0 @@
"""drop source_dialect/target_dialect from terminology_dictionaries
Revision ID: f1a2b3c4d5e6
Revises: dabc97097e0e
Create Date: 2026-06-02 12:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = 'f1a2b3c4d5e6'
down_revision: str | Sequence[str] | None = 'dabc97097e0e'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _column_exists(table_name: str, column_name: str) -> bool:
conn = op.get_bind()
inspector = inspect(conn)
columns = [c["name"] for c in inspector.get_columns(table_name)]
return column_name in columns
def upgrade() -> None:
"""Drop source_dialect and target_dialect from terminology_dictionaries."""
if not _column_exists("terminology_dictionaries", "source_dialect"):
return
op.drop_column("terminology_dictionaries", "source_dialect")
op.drop_column("terminology_dictionaries", "target_dialect")
def downgrade() -> None:
"""Re-add source_dialect and target_dialect to terminology_dictionaries."""
import sqlalchemy as sa
op.add_column("terminology_dictionaries", sa.Column(
"source_dialect", sa.String(), nullable=False, server_default="",
))
op.add_column("terminology_dictionaries", sa.Column(
"target_dialect", sa.String(), nullable=False, server_default="",
))

75
backend/alembic/versions/f2b3c4d5e6f7_add_agent_conversations.py
View File

@@ -1,75 +0,0 @@
# #region Alembic.AddAgentConversations [C:2] [TYPE Function] [SEMANTICS alembic,migration,agent]
# @BRIEF Add agent_conversations and agent_messages tables for Gradio Agent Chat.
# @RELATION DEPENDS_ON -> [Models.Agent]
"""add agent conversations
Revision ID: f2b3c4d5e6f7
Revises: f0e9d8c7b6a5
Create Date: 2026-06-09 13:30:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "f2b3c4d5e6f7"
down_revision: Union[str, None] = "f0e9d8c7b6a5"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"agent_conversations",
sa.Column("id", sa.String(), nullable=False),
sa.Column("user_id", sa.String(), nullable=False),
sa.Column("title", sa.String(256), nullable=False, server_default="New Conversation"),
sa.Column("is_archived", sa.Boolean(), nullable=False, server_default=sa.text("false")),
sa.Column("created_at", sa.DateTime(), server_default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(), server_default=sa.func.now()),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
op.f("ix_agent_conversations_user_id"),
"agent_conversations",
["user_id"],
unique=False,
)
op.create_table(
"agent_messages",
sa.Column("id", sa.String(), nullable=False),
sa.Column(
"conversation_id",
sa.String(),
sa.ForeignKey("agent_conversations.id"),
nullable=False,
),
sa.Column("role", sa.String(16), nullable=False),
sa.Column("text", sa.Text(), nullable=True),
sa.Column("state", sa.String(32), nullable=True),
sa.Column("tool_calls", sa.JSON(), nullable=True),
sa.Column("attachments", sa.JSON(), nullable=True),
sa.Column("created_at", sa.DateTime(), server_default=sa.func.now()),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
op.f("ix_agent_messages_conversation_id"),
"agent_messages",
["conversation_id"],
unique=False,
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f("ix_agent_messages_conversation_id"), table_name="agent_messages")
op.drop_table("agent_messages")
op.drop_index(op.f("ix_agent_conversations_user_id"), table_name="agent_conversations")
op.drop_table("agent_conversations")
# ### end Alembic commands ###
# #endregion Alembic.AddAgentConversations

48
backend/alembic/versions/f4a5b6c7d8e9_add_deployment_validation.py
View File

@@ -1,48 +0,0 @@
# #region Alembic.AddDeploymentValidation [C:3] [TYPE Module] [SEMANTICS alembic,deployment,validation]
# @defgroup Alembic Persist PREPROD validation against each deployed dashboard version.
# @LAYER Database
"""add preproduction validation to deployment records
Revision ID: f4a5b6c7d8e9
Revises: e3a4b5c6d7e8
Create Date: 2026-07-12 18:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "f4a5b6c7d8e9"
down_revision: Union[str, None] = "e3a4b5c6d7e8"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
# #region Alembic.AddDeploymentValidation.Upgrade [C:2] [TYPE Function] [SEMANTICS alembic,deployment,validation]
# @ingroup Alembic
# @BRIEF Add validation metadata without changing existing deployment history.
def upgrade() -> None:
op.add_column(
"deployment_records",
sa.Column("validation_status", sa.String(20), nullable=False, server_default="pending"),
)
op.add_column("deployment_records", sa.Column("validated_at", sa.DateTime(), nullable=True))
op.add_column("deployment_records", sa.Column("validated_by", sa.String(255), nullable=True))
# #endregion Alembic.AddDeploymentValidation.Upgrade
# #region Alembic.AddDeploymentValidation.Downgrade [C:2] [TYPE Function] [SEMANTICS alembic,deployment,validation]
# @ingroup Alembic
# @BRIEF Remove PREPROD validation metadata.
def downgrade() -> None:
op.drop_column("deployment_records", "validated_by")
op.drop_column("deployment_records", "validated_at")
op.drop_column("deployment_records", "validation_status")
# #endregion Alembic.AddDeploymentValidation.Downgrade
# #endregion Alembic.AddDeploymentValidation

63
backend/alembic/versions/f5e6d7c8b9a0_add_dashboard_releases.py
View File

@@ -1,63 +0,0 @@
# #region Alembic.AddDashboardReleases [C:3] [TYPE Module] [SEMANTICS alembic,git,release]
# @defgroup Alembic Persist dashboard release records and repository policy overrides.
"""add dashboard releases
Revision ID: f5e6d7c8b9a0
Revises: d4e5f6a7b8c9
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
revision: str = "f5e6d7c8b9a0"
down_revision: str | Sequence[str] | None = "d4e5f6a7b8c9"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
# #region Alembic.AddDashboardReleases.Upgrade [C:3] [TYPE Function] [SEMANTICS alembic,git,release]
# @ingroup Alembic
# @BRIEF Add release ledger and optional per-repository policy JSON.
def upgrade() -> None:
op.add_column("git_repositories", sa.Column("release_policy", sa.JSON(), nullable=True))
op.create_table(
"dashboard_releases",
sa.Column("id", sa.String(length=36), primary_key=True),
sa.Column("repository_id", sa.String(length=36), sa.ForeignKey("git_repositories.id", ondelete="CASCADE"), nullable=False),
sa.Column("deployment_id", sa.Integer(), sa.ForeignKey("deployment_records.id", ondelete="RESTRICT"), nullable=False, unique=True),
sa.Column("name", sa.String(length=255), nullable=False),
sa.Column("version", sa.String(length=100), nullable=False),
sa.Column("notes", sa.Text(), nullable=False),
sa.Column("commit_hash", sa.String(length=40), nullable=False),
sa.Column("content_hash", sa.String(length=64), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("created_by", sa.String(length=255), nullable=False),
sa.Column("approved_at", sa.DateTime(), nullable=True),
sa.Column("approved_by", sa.String(length=255), nullable=True),
sa.Column("approval_comment", sa.Text(), nullable=True),
sa.Column("published_at", sa.DateTime(), nullable=True),
sa.Column("published_by", sa.String(length=255), nullable=True),
sa.UniqueConstraint("repository_id", "version", name="uq_dashboard_release_repository_version"),
)
op.create_index("ix_dashboard_releases_repository_id", "dashboard_releases", ["repository_id"])
# #endregion Alembic.AddDashboardReleases.Upgrade
# #region Alembic.AddDashboardReleases.Downgrade [C:2] [TYPE Function] [SEMANTICS alembic,git,release]
# @ingroup Alembic
# @BRIEF Remove dashboard release persistence.
def downgrade() -> None:
op.drop_index("ix_dashboard_releases_repository_id", table_name="dashboard_releases")
op.drop_table("dashboard_releases")
op.drop_column("git_repositories", "release_policy")
# #endregion Alembic.AddDashboardReleases.Downgrade
# #endregion Alembic.AddDashboardReleases

184
backend/alembic/versions/f7a8b9c0d1e2_add_translate_performance_knobs.py
View File

@@ -1,184 +0,0 @@
# #region Alembic.AddTranslatePerformanceKnobs [C:3] [TYPE Module] [SEMANTICS alembic,translate,performance]
# @defgroup Alembic Persist translation performance knobs (job policy + provider capabilities).
"""Add translation performance knobs (job + provider capabilities).
Revision ID: f7a8b9c0d1e2
Revises: e6f7a8b9c0d1
Create Date: 2026-07-20 10:30:00.000000
NULL defaults preserve legacy algorithm behaviour (serial LLM, auto hard caps).
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "f7a8b9c0d1e2"
down_revision: str | Sequence[str] | None = "e6f7a8b9c0d1"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
# #region Alembic.AddTranslatePerformanceKnobs.AddColIfMissing [C:2] [TYPE Function] [SEMANTICS alembic,translate,idempotent]
# @ingroup Alembic
# @BRIEF Add a column only when absent — keeps the migration re-runnable.
def _add_col_if_missing(table: str, column: sa.Column) -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table(table):
return
existing = {c["name"] for c in inspector.get_columns(table)}
if column.name in existing:
return
op.add_column(table, column)
# #endregion Alembic.AddTranslatePerformanceKnobs.AddColIfMissing
# #region Alembic.AddTranslatePerformanceKnobs.Upgrade [C:3] [TYPE Function] [SEMANTICS alembic,translate,performance]
# @ingroup Alembic
# @BRIEF Add nullable performance knobs to translation_jobs and llm_providers.
# @RATIONALE NULL defaults preserve legacy algorithm behaviour (serial LLM, auto hard caps);
# capabilities are stored in DB, not inferred from brand/host at runtime.
# @RATIONALE llm_providers is an ORM-owned table created by Base.metadata.create_all()
# after Alembic on a fresh install; absent tables must therefore be skipped here.
# @REJECTED Non-nullable columns with server defaults — would silently flip legacy jobs to new behaviour.
def upgrade() -> None:
# ── translation_jobs (job policy / performance) ────────────────────────
_add_col_if_missing(
"translation_jobs",
sa.Column(
"llm_batch_max_rows",
sa.Integer(),
nullable=True,
comment="Max source rows per LLM batch (NULL = algorithm default)",
),
)
_add_col_if_missing(
"translation_jobs",
sa.Column(
"llm_concurrency",
sa.Integer(),
nullable=True,
comment="Parallel LLM batch workers (NULL = 1, legacy serial)",
),
)
_add_col_if_missing(
"translation_jobs",
sa.Column(
"insert_concurrency",
sa.Integer(),
nullable=True,
comment="Parallel insert workers (NULL = 1)",
),
)
_add_col_if_missing(
"translation_jobs",
sa.Column(
"multi_lang_mode",
sa.String(),
nullable=True,
comment="single_call | per_language (NULL = single_call)",
),
)
_add_col_if_missing(
"translation_jobs",
sa.Column(
"batch_aggressiveness",
sa.String(),
nullable=True,
comment="safe | balanced | fast (NULL = balanced legacy constants)",
),
)
_add_col_if_missing(
"translation_jobs",
sa.Column(
"max_in_flight_batches",
sa.Integer(),
nullable=True,
comment="Backpressure queue depth for parallel results (NULL = 32)",
),
)
# ── llm_providers (capabilities, not brand heuristics) ─────────────────
_add_col_if_missing(
"llm_providers",
sa.Column(
"throughput_class",
sa.String(),
nullable=True,
comment="standard | local (NULL = derive later via capability, not host sniff at runtime)",
),
)
_add_col_if_missing(
"llm_providers",
sa.Column(
"reasoning_control",
sa.String(),
nullable=True,
comment="off|generic_none|openai_effort|deepseek_thinking|llamacpp_think|auto",
),
)
_add_col_if_missing(
"llm_providers",
sa.Column(
"supports_json_object",
sa.Boolean(),
nullable=True,
comment="If true, send response_format=json_object (NULL = true for openai-compatible)",
),
)
_add_col_if_missing(
"llm_providers",
sa.Column(
"default_llm_concurrency",
sa.Integer(),
nullable=True,
comment="Default job llm_concurrency when job field is NULL",
),
)
_add_col_if_missing(
"llm_providers",
sa.Column(
"max_llm_concurrency",
sa.Integer(),
nullable=True,
comment="Hard ceiling for job llm_concurrency",
),
)
# #endregion Alembic.AddTranslatePerformanceKnobs.Upgrade
# #region Alembic.AddTranslatePerformanceKnobs.Downgrade [C:2] [TYPE Function] [SEMANTICS alembic,translate,performance]
# @ingroup Alembic
# @BRIEF Drop performance knobs conditionally (idempotent).
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
job_cols = {c["name"] for c in inspector.get_columns("translation_jobs")}
for name in (
"max_in_flight_batches",
"batch_aggressiveness",
"multi_lang_mode",
"insert_concurrency",
"llm_concurrency",
"llm_batch_max_rows",
):
if name in job_cols:
op.drop_column("translation_jobs", name)
if not inspector.has_table("llm_providers"):
return
prov_cols = {c["name"] for c in inspector.get_columns("llm_providers")}
for name in (
"max_llm_concurrency",
"default_llm_concurrency",
"supports_json_object",
"reasoning_control",
"throughput_class",
):
if name in prov_cols:
op.drop_column("llm_providers", name)
# #endregion Alembic.AddTranslatePerformanceKnobs.Downgrade
# #endregion Alembic.AddTranslatePerformanceKnobs

116
backend/alembic/versions/g1h2i3j4k5l6_add_agent_runs.py
View File

@@ -1,116 +0,0 @@
# #region Alembic.AddAgentRuns [C:3] [TYPE Module] [SEMANTICS alembic,agent-run,durable]
# @ingroup Alembic
# @BRIEF Add agent_runs, agent_run_events, draft_artifacts, approval_gates tables.
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.AgentRun]
# @INVARIANT agent_run_events has unique (run_id, sequence); drafts use opaque content_ref.
# @RATIONALE Durable run state is the backend-of-record for scenario runs — survives Gradio restarts.
"""add agent runs tables
Revision ID: g1h2i3j4k5l6
Revises: 8e9f0a1b2c3d
Create Date: 2026-07-28 12:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "g1h2i3j4k5l6"
down_revision: Union[str, Sequence[str], None] = "8e9f0a1b2c3d"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"agent_runs",
sa.Column("id", sa.String(), nullable=False),
sa.Column("conversation_id", sa.String(128), nullable=True),
sa.Column("user_id", sa.String(), nullable=False),
sa.Column("intent", sa.String(64), nullable=False, server_default="dashboard_scenario_build"),
sa.Column("trigger", sa.String(64), nullable=False, server_default="manual"),
sa.Column("dashboard_id", sa.String(64), nullable=False),
sa.Column("environment_id", sa.String(128), nullable=False),
sa.Column("context_snapshot", sa.JSON(), nullable=False),
sa.Column("status", sa.String(32), nullable=False, server_default="CREATED"),
sa.Column("current_stage", sa.String(32), nullable=True),
sa.Column("last_sequence", sa.Integer(), nullable=False, server_default="0"),
sa.Column("error_code", sa.String(64), nullable=True),
sa.Column("error_detail", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("updated_at", sa.DateTime(), nullable=False),
sa.Column("finished_at", sa.DateTime(), nullable=True),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_agent_runs_user_status", "agent_runs", ["user_id", "status"])
op.create_index("ix_agent_runs_dashboard", "agent_runs", ["dashboard_id", "environment_id"])
op.create_table(
"agent_run_events",
sa.Column("id", sa.String(), nullable=False),
sa.Column("run_id", sa.String(), nullable=False),
sa.Column("sequence", sa.Integer(), nullable=False),
sa.Column("event_type", sa.String(32), nullable=False),
sa.Column("stage", sa.String(32), nullable=True),
sa.Column("status", sa.String(32), nullable=True),
sa.Column("payload", sa.JSON(), nullable=True),
sa.Column("payload_hash", sa.String(64), nullable=True),
sa.Column("occurred_at", sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(["run_id"], ["agent_runs.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_agent_run_events_run_seq", "agent_run_events", ["run_id", "sequence"], unique=True)
op.create_table(
"draft_artifacts",
sa.Column("id", sa.String(), nullable=False),
sa.Column("run_id", sa.String(), nullable=False),
sa.Column("kind", sa.String(32), nullable=False),
sa.Column("name", sa.String(255), nullable=False),
sa.Column("intended_path", sa.String(512), nullable=False),
sa.Column("content_ref", sa.String(256), nullable=False),
sa.Column("sha256", sa.String(64), nullable=False),
sa.Column("validation_status", sa.String(16), nullable=False, server_default="pending"),
sa.Column("warnings", sa.JSON(), nullable=True),
sa.Column("persisted_at", sa.DateTime(), nullable=True),
sa.Column("capture_meta", sa.JSON(), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(["run_id"], ["agent_runs.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_draft_artifacts_run", "draft_artifacts", ["run_id"])
op.create_table(
"approval_gates",
sa.Column("id", sa.String(), nullable=False),
sa.Column("run_id", sa.String(), nullable=False),
sa.Column("operation", sa.String(32), nullable=False),
sa.Column("request_hash", sa.String(64), nullable=False),
sa.Column("target_paths", sa.JSON(), nullable=False),
sa.Column("risk_level", sa.String(16), nullable=False, server_default="guarded"),
sa.Column("required_permission", sa.String(64), nullable=False),
sa.Column("status", sa.String(16), nullable=False, server_default="pending"),
sa.Column("reason_required", sa.Boolean(), nullable=False, server_default="false"),
sa.Column("reason", sa.Text(), nullable=True),
sa.Column("actor_id", sa.String(), nullable=True),
sa.Column("decided_at", sa.DateTime(), nullable=True),
sa.Column("expires_at", sa.DateTime(), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(["run_id"], ["agent_runs.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_approval_gates_run", "approval_gates", ["run_id"])
op.create_index("ix_approval_gates_status", "approval_gates", ["status"])
def downgrade() -> None:
op.drop_table("approval_gates")
op.drop_table("draft_artifacts")
op.drop_table("agent_run_events")
op.drop_table("agent_runs")
# #endregion Alembic.AddAgentRuns

83
backend/alembic/versions/h2i3j4k5l6m7_add_verification_runs.py
View File

@@ -1,83 +0,0 @@
# #region Alembic.AddVerificationRuns [C:3] [TYPE Module] [SEMANTICS alembic,verification,run,persistence]
# @ingroup Alembic
# @BRIEF Add verification_runs table for per-category verification outcome persistence.
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.VerificationRun]
# @INVARIANT FK on agent_run_id uses ON DELETE SET NULL — run survives agent run deletion.
# @INVARIANT FK on release_id uses ON DELETE SET NULL — run survives release deletion (audit retention).
# @RATIONALE Verification runs have their own identity and lifecycle independent of agent
# scenarios. The table stores immutable per-category outcomes with evidence refs.
# Release FK uses SET NULL so historical runs are preserved for audit when a release
# is deleted (audit-retention requirement).
# @REJECTED Embedding outcomes in AgentRun.events was rejected — verification runs have
# their own lifecycle. Storing as JSON on DashboardRelease was rejected — a
# release may have multiple verification runs over time.
"""add verification_runs table
Revision ID: h2i3j4k5l6m7
Revises: g1h2i3j4k5l6
Create Date: 2026-07-30 12:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "h2i3j4k5l6m7"
down_revision: str | Sequence[str] | None = "g1h2i3j4k5l6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Create verification_runs table with FK to agent_runs and dashboard_releases."""
op.create_table(
"verification_runs",
sa.Column("id", sa.String(), nullable=False),
sa.Column(
"agent_run_id",
sa.String(),
sa.ForeignKey("agent_runs.id", ondelete="SET NULL"),
nullable=True,
index=True,
),
sa.Column("repository_id", sa.String(), nullable=False),
sa.Column(
"release_id",
sa.String(),
sa.ForeignKey("dashboard_releases.id", ondelete="SET NULL"),
nullable=True,
index=True,
),
sa.Column("trigger", sa.String(), nullable=False),
sa.Column("environment_id", sa.String(), nullable=False),
sa.Column("categories_run", sa.JSON(), nullable=False),
sa.Column("category_outcomes", sa.JSON(), nullable=False),
sa.Column("overall_status", sa.String(), nullable=False),
sa.Column("summary", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("created_by", sa.String(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
"ix_verification_runs_agent_run",
"verification_runs",
["agent_run_id"],
)
op.create_index(
"ix_verification_runs_created",
"verification_runs",
["created_at"],
)
def downgrade() -> None:
"""Drop verification_runs table and its indexes."""
op.drop_index("ix_verification_runs_agent_run", table_name="verification_runs")
op.drop_index("ix_verification_runs_created", table_name="verification_runs")
op.drop_table("verification_runs")
# #endregion Alembic.AddVerificationRuns

93
backend/alembic/versions/i2j3k4l5m6n7_add_verification_runs_repository_fk.py
View File

@@ -1,93 +0,0 @@
# #region Alembic.AddVerificationRunsRepositoryFK [C:3] [TYPE Module] [SEMANTICS alembic,verification,run,repository,fk,ondelete,set-null]
# @ingroup Alembic
# @BRIEF Add FK on verification_runs.repository_id -> git_repositories.id with ON DELETE SET NULL.
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.VerificationRun]
# @INVARIANT FK on repository_id uses ON DELETE SET NULL — run survives repository deletion.
# @INVARIANT repository_id becomes nullable (was NOT NULL) to support SET NULL semantics
# and future migration flexibility.
# @RATIONALE The original verification_runs table had repository_id as a bare String with
# no FK constraint. This migration adds referential integrity so the service can
# validate repository existence before creating a run, and ON DELETE SET NULL
# ensures historical runs are preserved for audit when a repository is deleted.
# @REJECTED ON DELETE CASCADE was rejected — audit retention requires preserving verification
# run records even after the repository is removed from the system. CASCADE would
# silently lose audit history. Keeping NOT NULL was rejected — SET NULL requires
# a nullable column, and a deleted repository should not cascade-delete runs.
"""add FK verification_runs.repository_id -> git_repositories.id
Revision ID: i2j3k4l5m6n7
Revises: h2i3j4k5l6m7
Create Date: 2026-07-30 14:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "i2j3k4l5m6n7"
down_revision: str | Sequence[str] | None = "h2i3j4k5l6m7"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Add FK on verification_runs.repository_id -> git_repositories.id with SET NULL."""
# Step 1: Make repository_id nullable (was NOT NULL)
op.alter_column(
"verification_runs", "repository_id",
existing_type=sa.String(36),
nullable=True,
schema=None,
)
# Step 2: Add FK constraint with ON DELETE SET NULL
op.create_foreign_key(
"fk_verification_runs_repository",
"verification_runs",
"git_repositories",
["repository_id"],
["id"],
ondelete="SET NULL",
source_schema=None,
referent_schema=None,
)
# Step 3: Add index on repository_id for query performance
op.create_index(
"ix_verification_runs_repository",
"verification_runs",
["repository_id"],
)
def downgrade() -> None:
"""Drop FK and revert repository_id to NOT NULL.
WARNING: If rows have NULL repository_id (from SET NULL on parent delete),
the downgrade will FAIL because NOT NULL cannot be re-applied. Handle NULLs
before downgrading.
"""
# Step 1: Drop index
op.drop_index("ix_verification_runs_repository", table_name="verification_runs")
# Step 2: Drop FK constraint
op.drop_constraint(
"fk_verification_runs_repository",
"verification_runs",
type_="foreignkey",
)
# Step 3: Revert repository_id to NOT NULL
# NOTE: Will fail if any NULL repository_id values exist (from SET NULL on delete).
op.alter_column(
"verification_runs", "repository_id",
existing_type=sa.String(36),
nullable=False,
schema=None,
)
# #endregion Alembic.AddVerificationRunsRepositoryFK

72
backend/alembic/versions/j1k2l3m4n5o6_add_prior_release_id_to_dashboard_releases.py
View File

@@ -1,72 +0,0 @@
# #region Alembic.AddPriorReleaseId [C:3] [TYPE Module] [SEMANTICS alembic,inheritance,dashboard-release,fk]
# @ingroup Alembic
# @BRIEF Add prior_release_id FK on dashboard_releases -> dashboard_releases.id with ON DELETE SET NULL.
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.DashboardRelease]
# @INVARIANT FK on prior_release_id uses ON DELETE SET NULL — prior release survives current release deletion.
# @INVARIANT prior_release_id is nullable — first release in a chain has no prior.
# @RATIONALE FR-013: Baseline inheritance requires chaining releases so the inheritance service can
# load the prior release's content_hash values. ON DELETE SET NULL ensures the current
# release is not cascade-deleted when the prior is removed.
# @REJECTED ON DELETE CASCADE was rejected — deleting a prior release should not cascade-delete
# all subsequent releases in the chain. Keeping NOT NULL was rejected — the first release
# in a chain has no prior.
"""add prior_release_id FK on dashboard_releases -> dashboard_releases.id
Revision ID: j1k2l3m4n5o6
Revises: i2j3k4l5m6n7
Create Date: 2026-07-30 15:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "j1k2l3m4n5o6"
down_revision: str | Sequence[str] | None = "i2j3k4l5m6n7"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Add prior_release_id column with FK and index."""
# Step 1: Add nullable column
op.add_column(
"dashboard_releases",
sa.Column("prior_release_id", sa.String(length=36), nullable=True),
)
# Step 2: Add FK constraint with ON DELETE SET NULL
op.create_foreign_key(
"fk_dashboard_releases_prior_release",
"dashboard_releases",
"dashboard_releases",
["prior_release_id"],
["id"],
ondelete="SET NULL",
source_schema=None,
referent_schema=None,
)
# Step 3: Add index for query performance on inheritance lookups
op.create_index(
"ix_dashboard_releases_prior_release",
"dashboard_releases",
["prior_release_id"],
)
def downgrade() -> None:
"""Drop FK, index, and column."""
op.drop_index("ix_dashboard_releases_prior_release", table_name="dashboard_releases")
op.drop_constraint(
"fk_dashboard_releases_prior_release",
"dashboard_releases",
type_="foreignkey",
)
op.drop_column("dashboard_releases", "prior_release_id")
# #endregion Alembic.AddPriorReleaseId

73
backend/alembic/versions/k5l6m7n8o9p0_maintenance_settings_and_banner_snapshot.py
View File

@@ -1,73 +0,0 @@
# #region Alembic.MaintenanceSettingsAndBannerSnapshot [C:3] [TYPE Module] [SEMANTICS alembic,maintenance,banner-height,timezone,snapshot]
# @ingroup Alembic
# @BRIEF Add maintenance_settings.banner_height and maintenance_dashboard_banners.original_position_json;
# migrate default timezone 'UTC' -> 'Europe/Moscow' for the untouched default settings row.
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.Maintenance.MaintenanceModels]
# @INVARIANT banner_height is nullable — NULL/0 means auto-estimate (unchanged behavior).
# @INVARIANT original_position_json is nullable — legacy banners (pre-release) have no snapshot
# and keep the surgical removal path.
# @INVARIANT The timezone UPDATE touches only rows still equal to 'UTC' — a deliberate user choice
# of 'UTC' is indistinguishable from the old default and therefore preserved as-is;
# rows already customized to another zone are never overwritten.
# @RATIONALE The banner is removed by restoring the pre-mutation position_json verbatim instead of
# rebuilding the layout structure, which broke dashboards (y-shift drift, unreverted
# ROOT->TABS -> ROOT->GRID normalization). The snapshot column stores that JSON.
# @REJECTED Restoring only the banner keys (surgical removal) was rejected as the primary path —
# it cannot revert structural normalization and drifts y-coordinates. Dropping the
# snapshot after restore was rejected — keeping it is harmless and aids debugging.
"""add banner_height + original_position_json, migrate timezone default
Revision ID: k5l6m7n8o9p0
Revises: j1k2l3m4n5o6
Create Date: 2026-08-04 10:30:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from sqlalchemy import inspect
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "k5l6m7n8o9p0"
down_revision: str | Sequence[str] | None = "j1k2l3m4n5o6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _table_exists(table: str) -> bool:
"""Check if a table exists in the current database connection."""
return inspect(op.get_bind()).has_table(table)
def upgrade() -> None:
"""Add the two columns and migrate the untouched timezone default."""
# maintenance_settings / maintenance_dashboard_banners are create_all()-only tables on a
# fresh install (no base migration creates them); guard so clean-DB upgrade is a no-op and
# the ORM schema initialization creates them with the final shape.
if _table_exists("maintenance_settings"):
op.add_column(
"maintenance_settings",
sa.Column("banner_height", sa.Integer(), nullable=True),
)
op.execute(
"UPDATE maintenance_settings SET display_timezone = 'Europe/Moscow' "
"WHERE id = 'default' AND display_timezone = 'UTC'"
)
if _table_exists("maintenance_dashboard_banners"):
op.add_column(
"maintenance_dashboard_banners",
sa.Column("original_position_json", sa.Text(), nullable=True),
)
def downgrade() -> None:
"""Drop the two columns (timezone values are left as-is)."""
if _table_exists("maintenance_dashboard_banners"):
op.drop_column("maintenance_dashboard_banners", "original_position_json")
if _table_exists("maintenance_settings"):
op.drop_column("maintenance_settings", "banner_height")
# #endregion Alembic.MaintenanceSettingsAndBannerSnapshot

48
backend/alembic/versions/l6m7n8o9p1q2_maintenance_event_auto_end.py
View File

@@ -1,48 +0,0 @@
# #region Alembic.MaintenanceEventAutoEnd [C:3] [TYPE Module] [SEMANTICS alembic,maintenance,auto-end,column]
# @ingroup Alembic
# @BRIEF Add maintenance_events.auto_end flag (opt-in automatic ending at end_time).
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.Maintenance.MaintenanceModels]
# @INVARIANT auto_end defaults to false — end_time stays informational unless the caller opts in.
# @RATIONALE end_time alone is used for idempotency and banner text; an explicit flag lets the
# scheduler end maintenance automatically only when the caller asks for it.
# @REJECTED Auto-ending on end_time presence alone was rejected — ETL tools may pass a window
# purely informational and would be surprised by an automatic end.
"""add auto_end flag to maintenance_events
Revision ID: l6m7n8o9p1q2
Revises: k5l6m7n8o9p0
Create Date: 2026-08-04 15:55:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from sqlalchemy import inspect
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "l6m7n8o9p1q2"
down_revision: str | Sequence[str] | None = "k5l6m7n8o9p0"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Add auto_end column with server default false (existing rows become informational)."""
# maintenance_events is a create_all()-only table on a fresh install; guard so a clean-DB
# upgrade is a no-op and ORM schema initialization creates it with the final shape.
if inspect(op.get_bind()).has_table("maintenance_events"):
op.add_column(
"maintenance_events",
sa.Column("auto_end", sa.Boolean(), nullable=False, server_default=sa.false()),
)
def downgrade() -> None:
"""Drop the auto_end column."""
if inspect(op.get_bind()).has_table("maintenance_events"):
op.drop_column("maintenance_events", "auto_end")
# #endregion Alembic.MaintenanceEventAutoEnd

198
backend/alembic/versions/m7n8o9p1q2r3_add_dataset_lineage_tables.py
View File

@@ -1,198 +0,0 @@
# #region Alembic.AddDatasetLineageTables [C:3] [TYPE Module] [SEMANTICS alembic,lineage,blast-radius,dataset,deprecation,fanout]
# @ingroup Alembic
# @BRIEF Add dataset lineage and blast-radius tables (041) + additive verification_runs.fanout_plan_id.
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.Lineage]
# @RELATION DEPENDS_ON -> [Models.VerificationRun]
# @INVARIANT verification_runs.fanout_plan_id is nullable with ON DELETE SET NULL — a run
# survives plan deletion (R8); null fanout_plan_id = ordinary run.
# @INVARIANT FanoutPlan.dataset_impact_id FK is NOT NULL — a plan always names its impact cause.
# @INVARIANT Edge unique key (environment_id, dataset_uuid, chart_uuid, dashboard_uuid) matches
# the ORM DatasetUsageEdge constraint (data-model).
# @RATIONALE JSON columns hold consumed sets read whole (R1); the trigger value dataset_updated
# is data-level on a plain String column (R5) so no DB enum migration is required.
# @REJECTED A dedicated lineage scheduler table was rejected (LIN-FR-002) — refresh rides the
# existing ID-synchronization cycle. Storing lineage in ResourceMapping was rejected
# (R1) — stale-deletion semantics differ from identity rows.
"""add dataset lineage tables
Revision ID: m7n8o9p1q2r3
Revises: l6m7n8o9p1q2
Create Date: 2026-08-04 19:50:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "m7n8o9p1q2r3"
down_revision: str | Sequence[str] | None = "l6m7n8o9p1q2"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Create lineage tables and add the additive verification_runs.fanout_plan_id column."""
op.create_table(
"dataset_usage_edges",
sa.Column("id", sa.String(), nullable=False),
sa.Column(
"environment_id",
sa.String(),
sa.ForeignKey("environments.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("dataset_uuid", sa.String(), nullable=False),
sa.Column("chart_uuid", sa.String(), nullable=False),
sa.Column("dashboard_uuid", sa.String(), nullable=False),
sa.Column("dashboard_title", sa.String(), nullable=True),
sa.Column("consumed_columns", sa.JSON(), nullable=False),
sa.Column("consumed_metrics", sa.JSON(), nullable=False),
sa.Column("projection_confidence", sa.String(length=16), nullable=False),
sa.Column("unresolved_refs", sa.JSON(), nullable=False),
sa.Column("edge_fingerprint", sa.String(length=64), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint(
"environment_id",
"dataset_uuid",
"chart_uuid",
"dashboard_uuid",
name="uq_dataset_usage_edge_env_dataset_chart_dash",
),
)
op.create_index(
"ix_dataset_usage_edges_dataset_env",
"dataset_usage_edges",
["environment_id", "dataset_uuid"],
)
op.create_index(
"ix_dataset_usage_edges_dashboard",
"dataset_usage_edges",
["environment_id", "dashboard_uuid"],
)
op.create_table(
"lineage_index_snapshots",
sa.Column("environment_id", sa.String(), nullable=False),
sa.Column("index_fingerprint", sa.String(length=64), nullable=False),
sa.Column("built_at", sa.DateTime(), nullable=False),
sa.Column("stale_index", sa.Boolean(), nullable=False),
sa.Column("last_error", sa.JSON(), nullable=True),
sa.PrimaryKeyConstraint("environment_id"),
)
op.create_table(
"dataset_schema_observations",
sa.Column("id", sa.String(), nullable=False),
sa.Column("environment_id", sa.String(), nullable=False),
sa.Column("dataset_uuid", sa.String(), nullable=False),
sa.Column("dataset_physical_key", sa.JSON(), nullable=False),
sa.Column("schema_hash", sa.String(length=64), nullable=False),
sa.Column("schema_payload", sa.JSON(), nullable=False),
sa.Column("observed_at", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
"ix_dataset_schema_observations_env_dataset_time",
"dataset_schema_observations",
["environment_id", "dataset_uuid", "observed_at"],
)
op.create_table(
"dataset_impact_records",
sa.Column("id", sa.String(), nullable=False),
sa.Column("environment_id", sa.String(), nullable=False),
sa.Column("dataset_uuid", sa.String(), nullable=False),
sa.Column("diff", sa.JSON(), nullable=False),
sa.Column("rules_version", sa.String(length=16), nullable=False),
sa.Column("max_severity", sa.String(length=16), nullable=False),
sa.Column("affected_dashboards", sa.JSON(), nullable=False),
sa.Column("observation_pair", sa.JSON(), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
"ix_dataset_impact_records_env_dataset_time",
"dataset_impact_records",
["environment_id", "dataset_uuid", "created_at"],
)
op.create_table(
"dataset_deprecations",
sa.Column("environment_id", sa.String(), nullable=False),
sa.Column("dataset_uuid", sa.String(), nullable=False),
sa.Column("successor_dataset_uuid", sa.String(), nullable=True),
sa.Column("deprecated_at", sa.DateTime(), nullable=False),
sa.Column("grace_window_days", sa.Integer(), nullable=False),
sa.Column("escalation_state", sa.String(length=16), nullable=False),
sa.Column("expired_at", sa.DateTime(), nullable=True),
sa.Column("dependent_status", sa.JSON(), nullable=False),
sa.PrimaryKeyConstraint("environment_id", "dataset_uuid"),
)
op.create_table(
"fanout_plans",
sa.Column("id", sa.String(), nullable=False),
sa.Column("environment_id", sa.String(), nullable=False),
sa.Column(
"dataset_impact_id",
sa.String(),
sa.ForeignKey("dataset_impact_records.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column("trigger", sa.String(length=32), nullable=False),
sa.Column("status", sa.String(length=16), nullable=False),
sa.Column("entries", sa.JSON(), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_fanout_plans_env", "fanout_plans", ["environment_id"])
op.create_index("ix_fanout_plans_created", "fanout_plans", ["created_at"])
# Additive 041 amendment: nullable fanout_plan_id on verification_runs (null = ordinary run).
op.add_column(
"verification_runs",
sa.Column(
"fanout_plan_id",
sa.String(),
sa.ForeignKey("fanout_plans.id", ondelete="SET NULL"),
nullable=True,
),
)
op.create_index(
"ix_verification_runs_fanout_plan",
"verification_runs",
["fanout_plan_id"],
)
def downgrade() -> None:
"""Drop the additive column and lineage tables in reverse dependency order."""
op.drop_index("ix_verification_runs_fanout_plan", table_name="verification_runs")
op.drop_column("verification_runs", "fanout_plan_id")
op.drop_index("ix_fanout_plans_created", table_name="fanout_plans")
op.drop_index("ix_fanout_plans_env", table_name="fanout_plans")
op.drop_table("fanout_plans")
op.drop_table("dataset_deprecations")
op.drop_index("ix_dataset_impact_records_env_dataset_time", table_name="dataset_impact_records")
op.drop_table("dataset_impact_records")
op.drop_index(
"ix_dataset_schema_observations_env_dataset_time",
table_name="dataset_schema_observations",
)
op.drop_table("dataset_schema_observations")
op.drop_table("lineage_index_snapshots")
op.drop_index("ix_dataset_usage_edges_dashboard", table_name="dataset_usage_edges")
op.drop_index("ix_dataset_usage_edges_dataset_env", table_name="dataset_usage_edges")
op.drop_table("dataset_usage_edges")
# #endregion Alembic.AddDatasetLineageTables

223
backend/alembic/versions/n1o2p3q4r5s6_add_load_testing_tables.py
View File

@@ -1,223 +0,0 @@
# #region Alembic.AddLoadTestingTables [C:3] [TYPE Module] [SEMANTICS alembic,load-testing,profile,variation,run,execution,aggregate,finding]
# @ingroup Alembic
# @BRIEF Add 040 dashboard-load-testing tables: load_profiles, load_variations, load_runs,
# load_executions, load_run_aggregates, consistency_findings.
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.LoadTesting]
# @INVARIANT load_variations composite PK (run_id, variation_id) matches the ORM — a variation is
# immutable and bound to exactly one run.
# @INVARIANT load_run_aggregates unique (run_id, chart_id, variation_id) matches the ORM
# UniqueConstraint — one aggregate per chart/variation cell.
# @INVARIANT load_runs.profile_id FK is nullable with ON DELETE SET NULL — a run survives profile
# deletion for audit retention (mirrors verification_runs.repository_id semantics).
# @INVARIANT execution/cascade tables use ON DELETE CASCADE — partial results are queryable only
# while the run exists; the run row itself is never cascade-deleted by these tables.
# @RATIONALE JSON columns (variation_axes, circuit_breaker, blast_radius_report, sample_rows,
# cache_counts, execution_ids) hold bounded structures read whole — matching the 041
# lineage JSON convention and R6 persistence decisions. Statuses are plain String
# columns so terminal-immutability is enforced in the ORM layer, not by a DB enum.
# @REJECTED A DB enum for run status was rejected — the ORM set_status guard owns immutability and
# additive future statuses must not require a migration. Storing full response blobs was
# rejected (R6) — the schema keeps digest/count/sample only.
"""add load testing tables
Revision ID: n1o2p3q4r5s6
Revises: m7n8o9p1q2r3
Create Date: 2026-08-04 22:30:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "n1o2p3q4r5s6"
down_revision: str | Sequence[str] | None = "m7n8o9p1q2r3"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Create the load-testing tables in dependency order."""
op.create_table(
"load_profiles",
sa.Column("id", sa.String(), nullable=False),
sa.Column("dashboard_id", sa.Integer(), nullable=False),
sa.Column("environment_id", sa.String(), nullable=False),
sa.Column("revision", sa.Integer(), nullable=False),
sa.Column("concurrency_requested", sa.Integer(), nullable=False),
sa.Column("execution_mode", sa.String(length=16), nullable=False),
sa.Column("iterations", sa.Integer(), nullable=True),
sa.Column("duration_seconds", sa.Integer(), nullable=True),
sa.Column("ramp_steps", sa.JSON(), nullable=False),
sa.Column("variation_axes", sa.JSON(), nullable=False),
sa.Column("circuit_breaker", sa.JSON(), nullable=False),
sa.Column("matrix_seed", sa.Integer(), nullable=False),
sa.Column("max_variations", sa.Integer(), nullable=False),
sa.Column("enabled", sa.Boolean(), nullable=False),
sa.Column("created_by", sa.String(), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("updated_at", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_load_profiles_dashboard", "load_profiles", ["dashboard_id"])
op.create_index("ix_load_profiles_environment", "load_profiles", ["environment_id"])
op.create_table(
"load_runs",
sa.Column("id", sa.String(), nullable=False),
sa.Column(
"profile_id",
sa.String(),
sa.ForeignKey("load_profiles.id", ondelete="SET NULL"),
nullable=True,
),
sa.Column("profile_revision", sa.Integer(), nullable=False),
sa.Column("environment_id", sa.String(), nullable=False),
sa.Column("dashboard_id", sa.Integer(), nullable=False),
sa.Column("status", sa.String(length=24), nullable=False),
sa.Column("phase", sa.String(length=16), nullable=False),
sa.Column("effective_concurrency", sa.Integer(), nullable=False),
sa.Column("matrix_seed", sa.Integer(), nullable=False),
sa.Column("theoretical_variations", sa.Integer(), nullable=False),
sa.Column("selected_variations", sa.Integer(), nullable=False),
sa.Column("total_executions", sa.Integer(), nullable=False),
sa.Column("blast_radius_fingerprint", sa.String(length=64), nullable=True),
sa.Column("blast_radius_report", sa.JSON(), nullable=True),
sa.Column("approval_gate_id", sa.String(), nullable=True),
sa.Column("task_id", sa.String(), nullable=True),
sa.Column("circuit_state", sa.String(length=16), nullable=False),
sa.Column("started_at", sa.DateTime(), nullable=True),
sa.Column("finished_at", sa.DateTime(), nullable=True),
sa.Column("stop_reason", sa.String(), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_load_runs_profile", "load_runs", ["profile_id"])
op.create_index("ix_load_runs_environment", "load_runs", ["environment_id"])
op.create_index("ix_load_runs_task", "load_runs", ["task_id"])
op.create_index("ix_load_runs_created", "load_runs", ["created_at"])
op.create_table(
"load_variations",
sa.Column("run_id", sa.String(), nullable=False),
sa.Column("variation_id", sa.String(length=64), nullable=False),
sa.Column("filters", sa.JSON(), nullable=False),
sa.Column("viewport", sa.JSON(), nullable=False),
sa.Column("role", sa.String(length=32), nullable=False),
sa.Column("time_range", sa.JSON(), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(["run_id"], ["load_runs.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("run_id", "variation_id"),
)
op.create_index("ix_load_variations_run", "load_variations", ["run_id"])
op.create_table(
"load_executions",
sa.Column("id", sa.String(), nullable=False),
sa.Column("run_id", sa.String(), nullable=False),
sa.Column("variation_id", sa.String(length=64), nullable=False),
sa.Column("chart_id", sa.Integer(), nullable=False),
sa.Column("filters_hash", sa.String(length=64), nullable=False),
sa.Column("outcome", sa.String(length=16), nullable=False),
sa.Column("error_taxonomy", sa.String(length=64), nullable=True),
sa.Column("queue_wait_ms", sa.Integer(), nullable=False),
sa.Column("resource_wait_ms", sa.Integer(), nullable=False),
sa.Column("upstream_latency_ms", sa.Integer(), nullable=False),
sa.Column("end_to_end_ms", sa.Integer(), nullable=False),
sa.Column("response_sha256", sa.String(length=64), nullable=True),
sa.Column("row_count", sa.Integer(), nullable=True),
sa.Column("sample_rows", sa.JSON(), nullable=True),
sa.Column("cache_state", sa.String(length=16), nullable=False),
sa.Column("is_cached", sa.Boolean(), nullable=True),
sa.Column("cache_key", sa.String(), nullable=True),
sa.Column("cached_dttm", sa.DateTime(), nullable=True),
sa.Column("queried_dttm", sa.DateTime(), nullable=True),
sa.Column("cache_timeout", sa.Integer(), nullable=True),
sa.Column("cache_state_source", sa.String(length=32), nullable=False),
sa.Column("worker_id", sa.String(length=64), nullable=False),
sa.Column("started_at", sa.DateTime(), nullable=False),
sa.Column("finished_at", sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(["run_id"], ["load_runs.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_load_executions_run", "load_executions", ["run_id"])
op.create_index(
"ix_load_executions_run_chart_var",
"load_executions",
["run_id", "chart_id", "variation_id"],
)
op.create_table(
"load_run_aggregates",
sa.Column("id", sa.String(), nullable=False),
sa.Column("run_id", sa.String(), nullable=False),
sa.Column("chart_id", sa.Integer(), nullable=False),
sa.Column("variation_id", sa.String(length=64), nullable=False),
sa.Column("success_count", sa.Integer(), nullable=False),
sa.Column("error_count", sa.Integer(), nullable=False),
sa.Column("p50_upstream_ms", sa.Integer(), nullable=False),
sa.Column("p90_upstream_ms", sa.Integer(), nullable=False),
sa.Column("p95_upstream_ms", sa.Integer(), nullable=False),
sa.Column("p99_upstream_ms", sa.Integer(), nullable=False),
sa.Column("p95_queue_wait_ms", sa.Integer(), nullable=False),
sa.Column("p95_resource_wait_ms", sa.Integer(), nullable=False),
sa.Column("throughput_per_second", sa.Float(), nullable=False),
sa.Column("cache_counts", sa.JSON(), nullable=False),
sa.ForeignKeyConstraint(["run_id"], ["load_runs.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint(
"run_id",
"chart_id",
"variation_id",
name="uq_load_run_aggregate_run_chart_var",
),
)
op.create_table(
"consistency_findings",
sa.Column("id", sa.String(), nullable=False),
sa.Column("run_id", sa.String(), nullable=False),
sa.Column("chart_id", sa.Integer(), nullable=False),
sa.Column("coordinate_key", sa.String(length=64), nullable=False),
sa.Column("first_response_sha256", sa.String(length=64), nullable=False),
sa.Column("divergent_response_sha256", sa.String(length=64), nullable=False),
sa.Column("execution_ids", sa.JSON(), nullable=False),
sa.Column("severity", sa.String(length=16), nullable=False),
sa.Column("classification", sa.String(length=16), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(["run_id"], ["load_runs.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_consistency_findings_run", "consistency_findings", ["run_id"])
op.create_index("ix_consistency_findings_chart", "consistency_findings", ["chart_id"])
def downgrade() -> None:
"""Drop the load-testing tables in reverse dependency order."""
op.drop_index("ix_consistency_findings_chart", table_name="consistency_findings")
op.drop_index("ix_consistency_findings_run", table_name="consistency_findings")
op.drop_table("consistency_findings")
op.drop_table("load_run_aggregates")
op.drop_index("ix_load_executions_run_chart_var", table_name="load_executions")
op.drop_index("ix_load_executions_run", table_name="load_executions")
op.drop_table("load_executions")
op.drop_index("ix_load_variations_run", table_name="load_variations")
op.drop_table("load_variations")
op.drop_index("ix_load_runs_created", table_name="load_runs")
op.drop_index("ix_load_runs_task", table_name="load_runs")
op.drop_index("ix_load_runs_environment", table_name="load_runs")
op.drop_index("ix_load_runs_profile", table_name="load_runs")
op.drop_table("load_runs")
op.drop_index("ix_load_profiles_environment", table_name="load_profiles")
op.drop_index("ix_load_profiles_dashboard", table_name="load_profiles")
op.drop_table("load_profiles")
# #endregion Alembic.AddLoadTestingTables

148
backend/alembic/versions/o1p2q3r4s5t6_legacy_maintenance_columns.py
View File

@@ -1,148 +0,0 @@
# #region Alembic.LegacyMaintenanceColumns [C:3] [TYPE Module] [SEMANTICS alembic,maintenance,legacy-schema,fanout-approval,idempotent]
# @ingroup Alembic
# @BRIEF Idempotent reconciliation for legacy databases: add maintenance_events.auto_end and
# maintenance_dashboard_banners.original_position_json when absent, and create the
# fanout_approvals table for persisted PROD fan-out approval (041 hardening), and
# load_runs.created_by for ownership checks (040 hardening).
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.Maintenance.MaintenanceModels]
# @RELATION DEPENDS_ON -> [Models.Lineage.FanoutApproval]
# @RELATION DEPENDS_ON -> [Models.LoadTesting.LoadRun]
# @INVARIANT Every schema mutation is guarded by an inspector check — running upgrade twice is a
# no-op on the second run (safe for legacy DBs stamped at head).
# @RATIONALE Legacy installations may reach alembic head without these columns (tables pre-date
# the feature migrations); reconciliation must tolerate both present and absent state.
# @REJECTED Unconditional op.add_column was rejected — it raises on an already-migrated schema
# and breaks the "legacy DB stamped at head" upgrade path (test_alembic_migrations).
# @REJECTED ALTER-only recovery was rejected — the fanout approval table must also be created for
# legacy DBs so PROD fan-out approval can persist.
"""add legacy maintenance columns + fanout approvals (idempotent)
Revision ID: o1p2q3r4s5t6
Revises: n1o2p3q4r5s6
Create Date: 2026-08-06 12:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "o1p2q3r4s5t6"
down_revision: str | Sequence[str] | None = "n1o2p3q4r5s6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _column_names(bind, table: str) -> set[str]:
"""Column names of an existing table; empty set when the table is absent."""
inspector = sa.inspect(bind)
if table not in inspector.get_table_names():
return set()
return {col["name"] for col in inspector.get_columns(table)}
def _table_exists(bind, table: str) -> bool:
"""Whether a table currently exists in the database connection."""
return sa.inspect(bind).has_table(table)
def upgrade() -> None:
"""Safely add the two legacy maintenance columns and the fanout_approvals table if absent."""
bind = op.get_bind()
# maintenance_events.auto_end — Boolean NOT NULL with server default false (informational
# unless the scheduler opts in; matches l6m7n8o9p1q2 for legacy tables that missed it).
# Guard on table existence too: these are create_all()-only tables on a fresh install, and a
# missing table must be left for ORM schema init, not mistaken for a missing column.
if _table_exists(bind, "maintenance_events") and "auto_end" not in _column_names(
bind, "maintenance_events"
):
op.add_column(
"maintenance_events",
sa.Column("auto_end", sa.Boolean(), nullable=False, server_default=sa.false()),
)
# maintenance_dashboard_banners.original_position_json — nullable Text snapshot restored
# verbatim on banner removal (matches k5l6m7n8o9p0).
if _table_exists(bind, "maintenance_dashboard_banners") and "original_position_json" not in _column_names(
bind, "maintenance_dashboard_banners"
):
op.add_column(
"maintenance_dashboard_banners",
sa.Column("original_position_json", sa.Text(), nullable=True),
)
# load_runs.created_by — ownership identity for status/stop/compare authorization.
if _table_exists(bind, "load_runs") and "created_by" not in _column_names(
bind, "load_runs"
):
op.add_column(
"load_runs",
sa.Column("created_by", sa.String(), nullable=False, server_default="system"),
)
inspector = sa.inspect(bind)
if "load_prod_approvals" not in inspector.get_table_names():
op.create_table(
"load_prod_approvals",
sa.Column("id", sa.String(), primary_key=True),
sa.Column("requester_id", sa.String(), nullable=False),
sa.Column("approver_id", sa.String(), nullable=False),
sa.Column("environment_id", sa.String(), nullable=False),
sa.Column("dashboard_id", sa.Integer(), nullable=False),
sa.Column("profile_revision", sa.Integer(), nullable=False, server_default="1"),
sa.Column("effective_cap", sa.Integer(), nullable=False),
sa.Column("request_estimate", sa.Integer(), nullable=False),
sa.Column("fingerprint", sa.String(length=64), nullable=False, server_default=""),
sa.Column("reason", sa.String(), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("consumed_at", sa.DateTime(), nullable=True),
)
# fanout_approvals — persisted PROD fan-out approval (041). FK-free on purpose: legacy
# schemas may not share the environments FK topology; scope is enforced by the service.
inspector = sa.inspect(bind)
if "fanout_approvals" not in inspector.get_table_names():
op.create_table(
"fanout_approvals",
sa.Column("id", sa.String(), primary_key=True),
sa.Column("environment_id", sa.String(), nullable=False),
sa.Column("dataset_uuid", sa.String(), nullable=False),
sa.Column("plan_id", sa.String(), nullable=True),
sa.Column("reason", sa.Text(), nullable=True),
sa.Column("approved_by", sa.String(), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("consumed_at", sa.DateTime(), nullable=True),
)
op.create_index(
"ix_fanout_approvals_env_dataset",
"fanout_approvals",
["environment_id", "dataset_uuid"],
)
def downgrade() -> None:
"""Drop only what this migration could have created; guards mirror upgrade()."""
bind = op.get_bind()
inspector = sa.inspect(bind)
if "fanout_approvals" in inspector.get_table_names():
op.drop_index("ix_fanout_approvals_env_dataset", table_name="fanout_approvals")
op.drop_table("fanout_approvals")
if "load_prod_approvals" in inspector.get_table_names():
op.drop_table("load_prod_approvals")
if "created_by" in _column_names(bind, "load_runs"):
op.drop_column("load_runs", "created_by")
if "original_position_json" in _column_names(bind, "maintenance_dashboard_banners"):
op.drop_column("maintenance_dashboard_banners", "original_position_json")
if "auto_end" in _column_names(bind, "maintenance_events"):
op.drop_column("maintenance_events", "auto_end")
# #endregion Alembic.LegacyMaintenanceColumns

44
backend/alembic/versions/p2q3r4s5t6u7_add_verification_run_dashboard_id.py
View File

@@ -1,44 +0,0 @@
# #region Alembic.VerificationRunDashboardId [C:2] [TYPE Module] [SEMANTICS alembic,verification,dashboard,history]
# @ingroup Alembic
# @BRIEF Add nullable verification_runs.dashboard_id for 037 T081 history filtering by dashboard.
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.VerificationRun]
# @INVARIANT The column is nullable and indexed — existing runs remain valid and history
# filtering by dashboard is a soft filter (runs without dashboard_id excluded only
# when the caller filters on dashboard_id).
# @RATIONALE VerificationRunRecord previously carried repository/release/environment but no
# dashboard identity; frontend getVerificationHistory(dashboardId, envId) needs a
# dashboard-scoped query. dashboard_id is populated from category_params at persist.
# @REJECTED Encoding dashboard identity into environment_id or release_id was rejected — it is
# a distinct dimension and would corrupt existing environment/release semantics.
"""add verification_runs.dashboard_id
Revision ID: p2q3r4s5t6u7
Revises: o1p2q3r4s5t6
Create Date: 2026-08-07 14:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "p2q3r4s5t6u7"
down_revision: str | Sequence[str] | None = "o1p2q3r4s5t6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Add nullable, indexed verification_runs.dashboard_id."""
op.add_column("verification_runs", sa.Column("dashboard_id", sa.Integer(), nullable=True))
op.create_index("ix_verification_runs_dashboard", "verification_runs", ["dashboard_id"])
def downgrade() -> None:
"""Drop the dashboard_id index and column."""
op.drop_index("ix_verification_runs_dashboard", table_name="verification_runs")
op.drop_column("verification_runs", "dashboard_id")
# #endregion Alembic.VerificationRunDashboardId

68
backend/alembic/versions/r2s3t4u5v6w7_maintenance_date_format.py
View File

@@ -1,68 +0,0 @@
# #region Alembic.MaintenanceDateFormat [C:2] [TYPE Module] [SEMANTICS alembic,maintenance,date-format,settings]
# @ingroup Alembic
# @BRIEF Add maintenance_settings.date_format (configurable banner date/time format).
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.Maintenance.MaintenanceModels]
# @INVARIANT date_format is non-nullable with server default 'YYYY.MM.DD hh:mm:ss'.
# @RATIONALE The maintenance banner date rendering was hardcoded to strftime('%Y-%m-%d %H:%M');
# this column makes it user-configurable (friendly YYYY/MM/DD tokens).
# @REJECTED Making the column nullable was rejected — the renderer always needs a format.
"""add date_format to maintenance_settings
Revision ID: r2s3t4u5v6w7
Revises: 9a5a3b802c49
Create Date: 2026-08-10 10:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from sqlalchemy import inspect
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "r2s3t4u5v6w7"
down_revision: str | Sequence[str] | None = "9a5a3b802c49"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _table_exists(table: str) -> bool:
"""Check if a table exists in the current database connection."""
return inspect(op.get_bind()).has_table(table)
def _column_exists(table: str, column: str) -> bool:
"""Check if a column exists in the given table."""
inspector = inspect(op.get_bind())
return any(c["name"] == column for c in inspector.get_columns(table))
def upgrade() -> None:
"""Add date_format column to maintenance_settings if missing."""
# maintenance_settings is a create_all()-only table on a fresh install (no base
# migration creates it); guard so a clean-DB upgrade is a no-op and the ORM schema
# initialization creates it with the final shape.
if _table_exists("maintenance_settings") and not _column_exists(
"maintenance_settings", "date_format"
):
op.add_column(
"maintenance_settings",
sa.Column(
"date_format",
sa.String(),
nullable=False,
server_default="YYYY.MM.DD hh:mm:ss",
),
)
def downgrade() -> None:
"""Drop the date_format column (values are discarded)."""
if _table_exists("maintenance_settings") and _column_exists(
"maintenance_settings", "date_format"
):
op.drop_column("maintenance_settings", "date_format")
# #endregion Alembic.MaintenanceDateFormat

93
backend/alembic/versions/s3t4u5v6w7x8_maintenance_default_message.py
View File

@@ -1,93 +0,0 @@
# #region Alembic.MaintenanceDefaultMessage [C:2] [TYPE Module] [SEMANTICS alembic,maintenance,settings,message]
# @ingroup Alembic
# @BRIEF Add the default message used to prefill new maintenance notices.
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.Maintenance.MaintenanceSettings]
# @INVARIANT default_message is non-nullable and does not alter existing event messages.
# @INVARIANT Only the exact legacy built-in template is migrated; user-edited templates stay intact.
"""add maintenance settings default message
Revision ID: s3t4u5v6w7x8
Revises: r2s3t4u5v6w7
Create Date: 2026-08-10 13:15:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from sqlalchemy import inspect
from alembic import op
revision: str = "s3t4u5v6w7x8"
down_revision: str | Sequence[str] | None = "r2s3t4u5v6w7"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
DEFAULT_MESSAGE = "Выполняется плановое обновление данных. Показатели могут быть временно недоступны."
OLD_DEFAULT_TEMPLATE = (
'<div style="background:#FFF3E0;padding:16px;border-left:4px solid #FF9800;border-radius:4px">\n\n'
"## ⚠️ Технические работы\n\n"
"{message}\n\n"
"**Начало:** {start_time}\n"
"**Конец:** {end_time}\n\n"
"*Данные могут быть неполными или временно недоступны.*\n\n"
"</div>"
)
RUSAL_DEFAULT_TEMPLATE = (
'<div style="background:#F2F7FA;border:1px solid #005B96;border-left:5px solid #005B96;'
'border-radius:8px;padding:14px 16px;margin-bottom:12px;font-family:Arial,sans-serif;color:#1F3344">\n'
'<div style="color:#005B96;font-size:16px;font-weight:700">РУСАЛ · Технические работы</div>\n'
'<p style="margin:8px 0;color:#1F3344">{message}</p>\n'
'<table style="font-size:13px;color:#40586B"><tr><td style="padding-right:12px">Начало:</td>'
'<td><strong>{start_time}</strong></td></tr><tr><td style="padding-right:12px">Окончание:</td>'
'<td><strong>{end_time}</strong></td></tr></table>\n'
'<p style="margin:10px 0 0;font-size:12px;color:#61788A">Данные на дашборде могут быть временно неполными.</p>\n'
"</div>"
)
def _column_exists(table: str, column: str) -> bool:
"""Return whether a column exists on a pre-existing installation."""
inspector = inspect(op.get_bind())
return inspector.has_table(table) and any(
item["name"] == column for item in inspector.get_columns(table)
)
def _table_exists(table: str) -> bool:
"""Return whether a legacy installation already has the table."""
return inspect(op.get_bind()).has_table(table)
def upgrade() -> None:
"""Persist the default message without modifying historical events."""
if _table_exists("maintenance_settings") and not _column_exists(
"maintenance_settings", "default_message"
):
op.add_column(
"maintenance_settings",
sa.Column(
"default_message",
sa.Text(),
nullable=False,
server_default=DEFAULT_MESSAGE,
),
)
if _table_exists("maintenance_settings"):
op.get_bind().execute(
sa.text(
"UPDATE maintenance_settings "
"SET banner_template = :new_template "
"WHERE id = 'default' AND banner_template = :old_template"
),
{"new_template": RUSAL_DEFAULT_TEMPLATE, "old_template": OLD_DEFAULT_TEMPLATE},
)
def downgrade() -> None:
"""Remove the setting; existing event messages remain intact."""
if _column_exists("maintenance_settings", "default_message"):
op.drop_column("maintenance_settings", "default_message")
# #endregion Alembic.MaintenanceDefaultMessage

89
backend/alembic/versions/t0u1v2w3x4y5_drop_orphaned_dataset_review_tables.py
View File

@@ -1,89 +0,0 @@
# #region Alembic.DropOrphanedDatasetReviewTables [C:2] [TYPE Module] [SEMANTICS alembic,dataset,review,cleanup]
# @ingroup Alembic
# @BRIEF Drop orphaned tables left behind by the removed dataset-review feature.
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.DatasetReview] (removed in 110cbb20)
# @RATIONALE The dataset-review feature was removed from the codebase (commit
# 110cbb20 "refactor: remove rejected dataset review feature"), but its tables
# were never dropped from the database. They are unreachable from the app
# (no models register them, so init_db()/create_all() will not recreate them)
# and they actively break environment deletion: dataset_review_sessions carries
# an environment_id FK -> environments.id with ON DELETE CASCADE, but
# dataset_profiles (and other children) reference dataset_review_sessions with
# non-cascading FKs. Cascading an environment delete into dataset_review_sessions
# therefore raises a ForeignKeyViolation and rolls the whole transaction back.
# @INVARIANT Drops only tables that belong to the removed feature. Children are
# dropped before parents so no FK violations occur; each drop is guarded by a
# table-exists check for databases that already cleaned up manually.
# @REJECTED Altering the orphaned FKs to add CASCADE — leaves dead tables behind.
# @REJECTED Dropping with CASCADE in one statement — hides the true dependency order.
"""drop orphaned dataset-review feature tables
Revision ID: t0u1v2w3x4y5
Revises: s3t4u5v6w7x8
Create Date: 2026-08-11 10:50:00.000000
"""
from collections.abc import Sequence
from sqlalchemy import inspect
from alembic import op
revision: str = "t0u1v2w3x4y5"
down_revision: str | Sequence[str] | None = "s3t4u5v6w7x8"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
# Drop order is dependency order: children before parents.
# Leaves referencing dataset_review_sessions are listed before their parents;
# clarification_* are listed before clarification_sessions, and
# semantic_candidates before semantic_field_entries.
ORPHANED_TABLES = [
# -> dataset_review_sessions children
"clarification_answers",
"clarification_options",
"clarification_questions",
"clarification_sessions",
"compiled_previews",
"dataset_run_contexts",
"execution_mappings",
"export_artifacts",
"imported_filters",
"semantic_candidates",
"semantic_field_entries",
"semantic_sources",
"session_collaborators",
"session_events",
"template_variables",
"validation_findings",
"dataset_profiles",
# -> parent, itself orphaned; environment_id FK to environments.id
"dataset_review_sessions",
]
def _table_exists(table: str) -> bool:
"""Return whether a table currently exists in the database."""
return inspect(op.get_bind()).has_table(table)
def upgrade() -> None:
"""Drop the orphaned dataset-review tables, children first."""
for table in ORPHANED_TABLES:
if _table_exists(table):
op.drop_table(table)
def downgrade() -> None:
"""Restore the orphaned tables is intentionally unsupported.
The feature that created these tables was removed, so their original DDL
no longer exists in the codebase. Recreating them here would invent schema.
"""
raise NotImplementedError(
"Dropping the orphaned dataset-review tables is not reversible: the "
"feature that created them was removed (commit 110cbb20)."
)
# #endregion Alembic.DropOrphanedDatasetReviewTables

59
backend/alembic/versions/t1u2v3w4x5y6_drop_orphaned_connection_configs.py
View File

@@ -1,59 +0,0 @@
# #region Alembic.DropOrphanedConnectionConfigs [C:2] [TYPE Module] [SEMANTICS alembic,connections,cleanup]
# @ingroup Alembic
# @BRIEF Drop the orphaned connection_configs table left by the removed connections feature.
# @LAYER Database
# @RELATION DEPENDS_ON -> [Models.Connection] (removed in 36643024)
# @RATIONALE The connection configuration feature was removed from the codebase
# (commit 36643024 "refactor(connections): remove ConnectionConfig, migrate
# mapper to SQL Lab"), but the connection_configs table was never dropped. It
# is unreachable from the app (no model registers it, so init_db()/create_all()
# will not recreate it) and carries no foreign keys, so it is inert — but it is
# dead schema and should be removed for cleanliness.
# @INVARIANT Only drops connection_configs; guarded by a table-exists check so it
# is a no-op on databases that already removed the table manually.
# @REJECTED Keeping the table — dead schema that shadows the removed feature.
"""drop orphaned connection_configs table
Revision ID: t1u2v3w4x5y6
Revises: t0u1v2w3x4y5
Create Date: 2026-08-11 10:55:00.000000
"""
from collections.abc import Sequence
from sqlalchemy import inspect
from alembic import op
revision: str = "t1u2v3w4x5y6"
down_revision: str | Sequence[str] | None = "t0u1v2w3x4y5"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
ORPHANED_TABLE = "connection_configs"
def _table_exists(table: str) -> bool:
"""Return whether a table currently exists in the database."""
return inspect(op.get_bind()).has_table(table)
def upgrade() -> None:
"""Drop the orphaned connection_configs table."""
if _table_exists(ORPHANED_TABLE):
op.drop_table(ORPHANED_TABLE)
def downgrade() -> None:
"""Restore connection_configs is intentionally unsupported.
The feature that created this table was removed (commit 36643024), so the
original DDL no longer exists in the codebase. Recreating it here would
invent schema.
"""
raise NotImplementedError(
"Dropping the orphaned connection_configs table is not reversible: the "
"feature that created it was removed (commit 36643024)."
)
# #endregion Alembic.DropOrphanedConnectionConfigs

44
backend/alembic/versions/u1v2w3x4y5z6_add_sync_duration_to_environments.py
View File

@@ -1,44 +0,0 @@
"""add ID-sync duration metrics to environments
Revision ID: u1v2w3x4y5z6
Revises: t1u2v3w4x5y6
Create Date: 2026-08-18
"""
from collections.abc import Sequence
import sqlalchemy as sa
from sqlalchemy import inspect
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "u1v2w3x4y5z6"
down_revision: str | Sequence[str] | None = "t1u2v3w4x5y6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _table_exists(table_name: str) -> bool:
conn = op.get_bind()
inspector = inspect(conn)
return table_name in inspector.get_table_names()
def upgrade() -> None:
"""Add last/avg sync duration + run count to environments for the pre-migration ID rescan reference."""
if not _table_exists("environments"):
return
op.add_column("environments", sa.Column("last_sync_duration_seconds", sa.Float(), nullable=True))
op.add_column("environments", sa.Column("sync_duration_avg_seconds", sa.Float(), nullable=True))
op.add_column(
"environments",
sa.Column("sync_run_count", sa.Integer(), nullable=False, server_default=sa.text("0")),
)
def downgrade() -> None:
"""Remove the ID-sync duration metrics columns."""
op.drop_column("environments", "sync_run_count")
op.drop_column("environments", "sync_duration_avg_seconds")
op.drop_column("environments", "last_sync_duration_seconds")

182
backend/alembic/versions/v1w2x3y4z5a6_add_scenario_registry_tables.py
View File

@@ -1,182 +0,0 @@
# #region Alembic.ScenarioRegistryTables [C:3] [TYPE Module] [SEMANTICS alembic,scenario,registry,revision,lifecycle,staleness]
# @ingroup Alembic
# @BRIEF Add 042 scenario registry tables: entries, revisions, staleness signals, lifecycle audit.
# @LAYER Database
"""add scenario registry tables (042 dashboard-scenario-registry)
Revision ID: v1w2x3y4z5a6
Revises: u1v2w3x4y5z6
Create Date: 2026-08-19
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "v1w2x3y4z5a6"
down_revision: str | Sequence[str] | None = "u1v2w3x4y5z6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Create scenario_registry_entries, scenario_revisions, scenario_staleness_signals, scenario_lifecycle_audit.
entry.current_revision_id is a plain indexed pointer (no FK): a circular FK
(revisions.scenario_id -> entries) would break sequential inserts under SQLite
FK enforcement and is unnecessary — revisions are append-only and pointer
integrity is owned by the activation service.
"""
# ── scenario_registry_entries ──
op.create_table(
"scenario_registry_entries",
sa.Column("scenario_id", sa.String(length=36), nullable=False),
sa.Column("scenario_key", sa.String(length=255), nullable=False),
sa.Column("name", sa.String(length=255), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("dashboard_id", sa.Integer(), nullable=False),
sa.Column("environment_ids", sa.JSON(), nullable=False),
sa.Column("owner_id", sa.String(length=128), nullable=False),
sa.Column("owner_username", sa.String(length=128), nullable=False),
sa.Column("tags", sa.JSON(), nullable=False),
sa.Column("metadata_version", sa.String(length=64), nullable=False),
sa.Column("current_revision_id", sa.String(length=36), nullable=True),
sa.Column("lifecycle_status", sa.String(length=32), nullable=False),
sa.Column("validation_status", sa.String(length=32), nullable=False),
sa.Column("last_run_id", sa.String(length=36), nullable=True),
sa.Column("last_successful_run_id", sa.String(length=36), nullable=True),
sa.Column("health", sa.String(length=16), nullable=False),
sa.Column("baseline_compatibility", sa.String(length=64), nullable=True),
sa.Column("source_scenario_id", sa.String(length=36), nullable=True),
sa.Column("source_revision_id", sa.String(length=36), nullable=True),
sa.Column("last_modified_at", sa.DateTime(), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("scenario_id"),
)
op.create_index("ix_scenario_registry_entries_key", "scenario_registry_entries", ["scenario_key"])
op.create_index(
"ix_scenario_registry_entries_dashboard", "scenario_registry_entries", ["dashboard_id"]
)
op.create_index(
"ix_scenario_registry_entries_owner", "scenario_registry_entries", ["owner_username"]
)
op.create_index(
"ix_scenario_registry_entries_lifecycle", "scenario_registry_entries", ["lifecycle_status"]
)
op.create_index(
"ix_scenario_registry_entries_current_revision",
"scenario_registry_entries",
["current_revision_id"],
)
# ── scenario_revisions ──
op.create_table(
"scenario_revisions",
sa.Column("revision_id", sa.String(length=36), nullable=False),
sa.Column("scenario_id", sa.String(length=36), nullable=False),
sa.Column("content_hash", sa.String(length=64), nullable=False),
sa.Column("parent_revision_id", sa.String(length=36), nullable=True),
sa.Column("graph_snapshot", sa.JSON(), nullable=False),
sa.Column("execution_template_hash", sa.String(length=64), nullable=False),
sa.Column("template_version", sa.String(length=32), nullable=False),
sa.Column("schema_version", sa.Integer(), nullable=False),
sa.Column("compatibility_family", sa.String(length=64), nullable=False),
sa.Column("change_summary", sa.JSON(), nullable=False),
sa.Column("created_by", sa.String(length=128), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("activation_status", sa.String(length=16), nullable=False),
sa.Column("activated_by", sa.String(length=128), nullable=True),
sa.Column("activated_at", sa.DateTime(), nullable=True),
sa.Column("activation_agent_action_id", sa.String(length=128), nullable=True),
sa.ForeignKeyConstraint(["scenario_id"], ["scenario_registry_entries.scenario_id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(["parent_revision_id"], ["scenario_revisions.revision_id"], ondelete="SET NULL"),
sa.PrimaryKeyConstraint("revision_id"),
)
op.create_index("ix_scenario_revisions_scenario", "scenario_revisions", ["scenario_id"])
op.create_index(
"ix_scenario_revisions_activation", "scenario_revisions", ["activation_status"]
)
# ── entry.current_revision_id is a plain indexed pointer (no FK) — see upgrade() rationale ──
# ── scenario_staleness_signals ──
op.create_table(
"scenario_staleness_signals",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("scenario_id", sa.String(length=36), nullable=False),
sa.Column("source_type", sa.String(length=64), nullable=False),
sa.Column("source_fingerprint", sa.String(length=128), nullable=False),
sa.Column("kind", sa.String(length=32), nullable=False),
sa.Column("severity", sa.String(length=16), nullable=False),
sa.Column("reason", sa.Text(), nullable=False),
sa.Column("affected_ref", sa.String(length=255), nullable=False),
sa.Column("detected_at", sa.DateTime(), nullable=False),
sa.Column("resolved_at", sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(["scenario_id"], ["scenario_registry_entries.scenario_id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint(
"scenario_id",
"source_type",
"source_fingerprint",
"affected_ref",
name="uq_scenario_staleness_signal_identity",
),
)
op.create_index(
"ix_scenario_staleness_signals_scenario", "scenario_staleness_signals", ["scenario_id"]
)
# ── scenario_lifecycle_audit ──
op.create_table(
"scenario_lifecycle_audit",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("scenario_id", sa.String(length=36), nullable=False),
sa.Column("from_state", sa.String(length=32), nullable=True),
sa.Column("to_state", sa.String(length=32), nullable=False),
sa.Column("actor_id", sa.String(length=128), nullable=False),
sa.Column("actor_type", sa.String(length=16), nullable=False),
sa.Column("reason", sa.Text(), nullable=True),
sa.Column("agent_action_id", sa.String(length=128), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(["scenario_id"], ["scenario_registry_entries.scenario_id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
"ix_scenario_lifecycle_audit_scenario", "scenario_lifecycle_audit", ["scenario_id"]
)
op.create_index(
"ix_scenario_lifecycle_audit_created", "scenario_lifecycle_audit", ["created_at"]
)
def downgrade() -> None:
"""Drop the scenario registry tables (reverse order of FK dependencies)."""
op.drop_index("ix_scenario_lifecycle_audit_created", table_name="scenario_lifecycle_audit")
op.drop_index("ix_scenario_lifecycle_audit_scenario", table_name="scenario_lifecycle_audit")
op.drop_table("scenario_lifecycle_audit")
op.drop_index(
"ix_scenario_staleness_signals_scenario", table_name="scenario_staleness_signals"
)
op.drop_table("scenario_staleness_signals")
op.drop_index(
"ix_scenario_registry_entries_current_revision",
table_name="scenario_registry_entries",
)
op.drop_index("ix_scenario_revisions_activation", table_name="scenario_revisions")
op.drop_index("ix_scenario_revisions_scenario", table_name="scenario_revisions")
op.drop_table("scenario_revisions")
op.drop_index("ix_scenario_registry_entries_lifecycle", table_name="scenario_registry_entries")
op.drop_index("ix_scenario_registry_entries_owner", table_name="scenario_registry_entries")
op.drop_index(
"ix_scenario_registry_entries_dashboard", table_name="scenario_registry_entries"
)
op.drop_index("ix_scenario_registry_entries_key", table_name="scenario_registry_entries")
op.drop_table("scenario_registry_entries")
# #endregion Alembic.ScenarioRegistryTables

43
backend/alembic/versions/w2x3y4z5a6_add_scenario_working_drafts.py
View File

@@ -1,43 +0,0 @@
# #region Alembic.ScenarioWorkingDrafts [C:2] [TYPE Module] [SEMANTICS alembic,scenario,editor,draft,persistence]
# @ingroup Alembic
# @BRIEF Add 043 server-owned working drafts table.
# @LAYER Database
"""add server-owned scenario working drafts (043 editor)"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "w2x3y4z5a6"
down_revision: str | Sequence[str] | None = "v1w2x3y4z5a6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_table(
"scenario_working_drafts",
sa.Column("draft_id", sa.String(length=36), nullable=False),
sa.Column("scenario_id", sa.String(length=36), nullable=False),
sa.Column("base_revision_id", sa.String(length=36), nullable=False),
sa.Column("operations", sa.JSON(), nullable=False),
sa.Column("applied_graph", sa.JSON(), nullable=False),
sa.Column("digest", sa.String(length=64), nullable=False),
sa.Column("created_by", sa.String(length=128), nullable=False),
sa.Column("status", sa.String(length=24), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(["scenario_id"], ["scenario_registry_entries.scenario_id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("draft_id"),
)
op.create_index("ix_scenario_working_drafts_scenario", "scenario_working_drafts", ["scenario_id"])
op.create_index("ix_scenario_working_drafts_base_revision", "scenario_working_drafts", ["base_revision_id"])
op.create_index("ix_scenario_working_drafts_status", "scenario_working_drafts", ["status"])
def downgrade() -> None:
op.drop_index("ix_scenario_working_drafts_status", table_name="scenario_working_drafts")
op.drop_index("ix_scenario_working_drafts_base_revision", table_name="scenario_working_drafts")
op.drop_index("ix_scenario_working_drafts_scenario", table_name="scenario_working_drafts")
op.drop_table("scenario_working_drafts")
# #endregion Alembic.ScenarioWorkingDrafts

71
backend/alembic/versions/x3y4z5a6b7c8_add_scenario_run_tables.py
View File

@@ -1,71 +0,0 @@
# #region Alembic.ScenarioRunTables [C:3] [TYPE Module] [SEMANTICS alembic,scenario,run,step,execution]
# @ingroup Alembic
# @BRIEF Add 044 scenario run + step run tables.
# @LAYER Database
"""add durable scenario execution run tables (044)"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "x3y4z5a6b7c8"
down_revision: str | Sequence[str] | None = "w2x3y4z5a6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_table(
"scenario_runs",
sa.Column("id", sa.String(36), nullable=False),
sa.Column("scenario_id", sa.String(36), nullable=False),
sa.Column("scenario_revision_id", sa.String(36), nullable=False),
sa.Column("scenario_content_hash", sa.String(64), nullable=False),
sa.Column("environment_id", sa.String(128), nullable=False),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("phase", sa.String(32), nullable=False),
sa.Column("parameter_bindings", sa.JSON(), nullable=False),
sa.Column("target_snapshot", sa.JSON(), nullable=False),
sa.Column("trigger_source", sa.String(32), nullable=False),
sa.Column("idempotency_key", sa.String(128), nullable=False),
sa.Column("runner_plan", sa.JSON(), nullable=False),
sa.Column("execution_principal_fingerprint", sa.String(64), nullable=True),
sa.Column("error_code", sa.String(64), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("started_at", sa.DateTime(), nullable=True),
sa.Column("finished_at", sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(["scenario_id"], ["scenario_registry_entries.scenario_id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("idempotency_key"),
)
op.create_index("ix_scenario_runs_scenario_status", "scenario_runs", ["scenario_id", "status"])
op.create_index("ix_scenario_runs_revision", "scenario_runs", ["scenario_revision_id"])
op.create_table(
"scenario_step_runs",
sa.Column("id", sa.String(36), nullable=False),
sa.Column("run_id", sa.String(36), nullable=False),
sa.Column("logical_step_id", sa.String(128), nullable=False),
sa.Column("step_position", sa.Integer(), nullable=False),
sa.Column("attempt", sa.Integer(), nullable=False),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("inputs_snapshot", sa.JSON(), nullable=False),
sa.Column("outputs", sa.JSON(), nullable=False),
sa.Column("artifact_refs", sa.JSON(), nullable=False),
sa.Column("progress", sa.Integer(), nullable=False),
sa.Column("error_code", sa.String(64), nullable=True),
sa.Column("started_at", sa.DateTime(), nullable=True),
sa.Column("finished_at", sa.DateTime(), nullable=True),
sa.Column("step_outcome", sa.JSON(), nullable=False),
sa.ForeignKeyConstraint(["run_id"], ["scenario_runs.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_scenario_step_runs_run", "scenario_step_runs", ["run_id"])
def downgrade() -> None:
op.drop_index("ix_scenario_step_runs_run", table_name="scenario_step_runs")
op.drop_table("scenario_step_runs")
op.drop_index("ix_scenario_runs_revision", table_name="scenario_runs")
op.drop_index("ix_scenario_runs_scenario_status", table_name="scenario_runs")
op.drop_table("scenario_runs")
# #endregion Alembic.ScenarioRunTables

40
backend/alembic/versions/y4z5a6b7c8d9_add_scenario_investigation_tables.py
View File

@@ -1,40 +0,0 @@
# #region Alembic.ScenarioInvestigationTables [C:3] [TYPE Module] [SEMANTICS alembic,scenario,investigation,queue,case,action,episode]
# @ingroup Alembic
# @BRIEF Add 047 investigation queue, case, agent action and recurring failure episode tables.
# @LAYER Database
"""add scenario investigation queue and case tables (047)"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "y4z5a6b7c8d9"
down_revision: str | Sequence[str] | None = "x3y4z5a6b7c8"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_table("scenario_investigation_queue",
sa.Column("id", sa.String(36), primary_key=True), sa.Column("fingerprint", sa.String(128), nullable=False),
sa.Column("scenario_id", sa.String(36), nullable=False), sa.Column("run_id", sa.String(36)),
sa.Column("severity", sa.String(16), nullable=False), sa.Column("status", sa.String(24), nullable=False),
sa.Column("occurrence_count", sa.Integer(), nullable=False), sa.Column("created_at", sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(["scenario_id"], ["scenario_registry_entries.scenario_id"], ondelete="CASCADE"))
op.create_index("ix_scenario_investigation_queue_fingerprint", "scenario_investigation_queue", ["fingerprint"])
op.create_index("ix_scenario_investigation_queue_status", "scenario_investigation_queue", ["status"])
op.create_table("scenario_investigation_cases",
sa.Column("id", sa.String(36), primary_key=True), sa.Column("fingerprint", sa.String(128), nullable=False, unique=True),
sa.Column("scenario_id", sa.String(36), nullable=False), sa.Column("status", sa.String(24), nullable=False),
sa.Column("disposition", sa.String(32)), sa.Column("decision_version", sa.Integer(), nullable=False), sa.Column("created_at", sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(["scenario_id"], ["scenario_registry_entries.scenario_id"], ondelete="CASCADE"))
op.create_table("scenario_agent_actions",
sa.Column("id", sa.String(36), primary_key=True), sa.Column("case_id", sa.String(36), nullable=False),
sa.Column("action_type", sa.String(32), nullable=False), sa.Column("payload", sa.JSON(), nullable=False), sa.Column("created_at", sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(["case_id"], ["scenario_investigation_cases.id"], ondelete="CASCADE"))
def downgrade() -> None:
op.drop_table("scenario_agent_actions")
op.drop_table("scenario_investigation_cases")
op.drop_index("ix_scenario_investigation_queue_status", table_name="scenario_investigation_queue")
op.drop_index("ix_scenario_investigation_queue_fingerprint", table_name="scenario_investigation_queue")
op.drop_table("scenario_investigation_queue")
# #endregion Alembic.ScenarioInvestigationTables

74
backend/alembic/versions/z5a6b7c8d9e0_add_scenario_execution_tables.py
View File

@@ -1,74 +0,0 @@
# #region Alembic.ScenarioExecutionTables [C:3] [TYPE Module] [SEMANTICS alembic,scenario,execution,artifact,approval,resume]
# @ingroup Alembic
# @BRIEF Add 044 execution artifacts, PROD approval gates, run idempotency/resume columns.
# @LAYER Database
"""add generic-owner scenario execution artifacts, PROD approval gates, run idempotency/resume columns (044 T014f/T017/T018/T019)"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "z5a6b7c8d9e0"
down_revision: str | Sequence[str] | None = "y4z5a6b7c8d9"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_table(
"scenario_artifacts",
sa.Column("id", sa.String(36), nullable=False),
sa.Column("owner_type", sa.String(32), nullable=False),
sa.Column("owner_id", sa.String(36), nullable=False),
sa.Column("kind", sa.String(32), nullable=False),
sa.Column("name", sa.String(255), nullable=False),
sa.Column("content_ref", sa.String(256), nullable=False),
sa.Column("sha256", sa.String(64), nullable=False),
sa.Column("retention_class", sa.String(32), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_scenario_artifacts_owner", "scenario_artifacts", ["owner_type", "owner_id"])
op.create_index(
"ix_scenario_artifacts_owner_kind", "scenario_artifacts", ["owner_type", "owner_id", "kind"]
)
op.create_table(
"action_approval_gates",
sa.Column("id", sa.String(36), nullable=False),
sa.Column("owner_type", sa.String(32), nullable=False),
sa.Column("owner_id", sa.String(36), nullable=False),
sa.Column("operation", sa.String(64), nullable=False),
sa.Column("required_permission", sa.String(64), nullable=False),
sa.Column("request_hash", sa.String(64), nullable=False),
sa.Column("status", sa.String(16), nullable=False),
sa.Column("decision", sa.String(16), nullable=True),
sa.Column("actor_id", sa.String(128), nullable=True),
sa.Column("comment", sa.String(2000), nullable=True),
sa.Column("expires_at", sa.DateTime(), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("decided_at", sa.DateTime(), nullable=True),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_action_approval_gates_owner", "action_approval_gates", ["owner_type", "owner_id"])
op.create_index("ix_action_approval_gates_status", "action_approval_gates", ["status"])
# Idempotency request fingerprint and infrastructure-pause resume token on existing run rows.
op.add_column("scenario_runs", sa.Column("request_hash", sa.String(64), nullable=True))
op.add_column("scenario_runs", sa.Column("resume_token", sa.String(128), nullable=True))
op.create_index("ix_scenario_runs_request_hash", "scenario_runs", ["request_hash"])
def downgrade() -> None:
op.drop_index("ix_scenario_runs_request_hash", table_name="scenario_runs")
op.drop_column("scenario_runs", "resume_token")
op.drop_column("scenario_runs", "request_hash")
op.drop_index("ix_action_approval_gates_status", table_name="action_approval_gates")
op.drop_index("ix_action_approval_gates_owner", table_name="action_approval_gates")
op.drop_table("action_approval_gates")
op.drop_index("ix_scenario_artifacts_owner_kind", table_name="scenario_artifacts")
op.drop_index("ix_scenario_artifacts_owner", table_name="scenario_artifacts")
op.drop_table("scenario_artifacts")
# #endregion Alembic.ScenarioExecutionTables

45
backend/alembic/versions/z6a7b8c9d0e1_add_scenario_edit_proposals.py
View File

@@ -1,45 +0,0 @@
# #region Alembic.ScenarioEditProposals [C:2] [TYPE Module] [SEMANTICS alembic,scenario,editor,proposal,agent]
# @ingroup Alembic
# @BRIEF Add 043 server-stored agent edit proposals table.
# @LAYER Database
"""add server-stored agent edit proposals (043 US5)"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "z6a7b8c9d0e1"
down_revision: str | Sequence[str] | None = "y4z5a6b7c8d9"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_table(
"scenario_edit_proposals",
sa.Column("proposal_id", sa.String(length=36), nullable=False),
sa.Column("scenario_id", sa.String(length=36), nullable=False),
sa.Column("base_revision_id", sa.String(length=36), nullable=False),
sa.Column("request_text", sa.Text(), nullable=False),
sa.Column("operations", sa.JSON(), nullable=False),
sa.Column("proposed_graph", sa.JSON(), nullable=False),
sa.Column("digest", sa.String(length=64), nullable=False),
sa.Column("created_by", sa.String(length=128), nullable=False),
sa.Column("agent_action_id", sa.String(length=128), nullable=True),
sa.Column("status", sa.String(length=24), nullable=False),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(["scenario_id"], ["scenario_registry_entries.scenario_id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("proposal_id"),
)
op.create_index("ix_scenario_edit_proposals_scenario", "scenario_edit_proposals", ["scenario_id"])
op.create_index("ix_scenario_edit_proposals_base_revision", "scenario_edit_proposals", ["base_revision_id"])
op.create_index("ix_scenario_edit_proposals_status", "scenario_edit_proposals", ["status"])
def downgrade() -> None:
op.drop_index("ix_scenario_edit_proposals_status", table_name="scenario_edit_proposals")
op.drop_index("ix_scenario_edit_proposals_base_revision", table_name="scenario_edit_proposals")
op.drop_index("ix_scenario_edit_proposals_scenario", table_name="scenario_edit_proposals")
op.drop_table("scenario_edit_proposals")
# #endregion Alembic.ScenarioEditProposals

35
backend/alembic/versions/z7a8b9c0d1e2_merge_044_execution_043_edits.py
View File

@@ -1,35 +0,0 @@
# #region Alembic.MergeExecutionAndEditProposals [C:1] [TYPE Module] [SEMANTICS alembic,migration,merge,scenario,execution,edit-proposals]
# @ingroup Alembic
# @BRIEF Merge two parallel heads from y4z5a6b7c8d9: 044 execution tables (z5a6b7c8d9e0) and
# 043 edit proposals (z6a7b8c9d0e1). Restores a single alembic head so `upgrade head`
# applies both branches exactly once.
# @RELATION DEPENDS_ON -> [Alembic.ScenarioExecutionTables]
# @RELATION DEPENDS_ON -> [Alembic.ScenarioEditProposals]
# @RATIONALE 044 execution (z5a6b7c8d9e0) and 043 edit proposals (z6a7b8c9d0e1) landed
# concurrently from y4z5a6b7c8d9. A no-op merge revision collapses them into one
# head so runtime migrations stop failing with "multiple head revisions".
# @REJECTED Editing down_revision of either existing migration was rejected — it would rewrite
# already-applied history on live deployments; a merge revision is the additive fix.
"""merge 044 execution tables and 043 edit proposals into one head
Both z5a6b7c8d9e0 (044 execution: artifacts/approval/resume) and z6a7b8c9d0e1
(043 edit proposals) branch from y4z5a6b7c8d9. This merge restores a single
alembic head so upgrade runs both branches exactly once.
"""
from collections.abc import Sequence
revision: str = "z7a8b9c0d1e2"
down_revision: str | Sequence[str] | None = ("z5a6b7c8d9e0", "z6a7b8c9d0e1")
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# Merge point only — both branches already applied their DDL.
pass
def downgrade() -> None:
pass
# #endregion Alembic.MergeExecutionAndEditProposals

1
backend/conftest.py
View File

@@ -21,7 +21,6 @@ _TEST_DB_PATH = _TEST_DB_FILE.name
_TEST_DB_FILE.close()
os.environ.setdefault("DATABASE_URL", f"sqlite:///{_TEST_DB_PATH}")
os.environ.setdefault("AUTH_DATABASE_URL", f"sqlite:///{_TEST_DB_PATH}")
# Integration-test fixture plugins. Declared at the top-level (rootdir) conftest
# because pytest >= 8.2 raises an error for `pytest_plugins` defined in a

1
backend/pyproject.toml
View File

@@ -16,6 +16,7 @@ include = ["src*"]
[tool.pytest.ini_options]
pythonpath = ["."]
addopts = ["-p", "pytest_early"]
asyncio_mode = "auto"
norecursedirs = ["__tests__"]
markers = [

104
backend/pytest_early.py Normal file
View File

@@ -0,0 +1,104 @@
# #region Test.PytestEarly.IntegrationDatabase [C:5] [TYPE Module] [SEMANTICS test,pytest,early-hook,postgres,testcontainers,alembic]
# @defgroup Test.PytestEarly Prepare the integration database before application imports.
# @PRE pytest is invoked with --run-integration; Docker and Testcontainers are available.
# @POST DATABASE_URL points at a temporary PostgreSQL database migrated to head before conftest loading.
# @INVARIANT Application modules are not imported until the integration DATABASE_URL is installed.
# @RATIONALE pytest_load_initial_conftests is the earliest supported hook for a command-line plugin.
# It runs before root conftest.py, whose pytest_plugins import the integration fixture modules.
# @REJECTED Starting PostgreSQL from a session fixture was rejected because tests/conftest.py and
# plugin imports can import src.core.database before session fixtures are evaluated.
from __future__ import annotations
import os
from pathlib import Path
import subprocess
import sys
import uuid
from sqlalchemy.engine import make_url
_container = None
_database_name: str | None = None
_admin_url: str | None = None
def pytest_load_initial_conftests(early_config, parser, args): # noqa: ARG001
"""Install the migrated PostgreSQL URL before any project conftest loads."""
if "--run-integration" not in args:
return
from testcontainers.postgres import PostgresContainer
global _admin_url, _container, _database_name
_container = PostgresContainer(
image="postgres:16-alpine",
username="test",
password="test",
dbname="test_translate",
)
_container.start()
_admin_url = _container.get_connection_url()
admin = make_url(_admin_url)
_database_name = f"ss_test_global_{uuid.uuid4().hex[:12]}"
import psycopg2
connection = psycopg2.connect(
host=_container.get_container_host_ip(),
port=_container.get_exposed_port(5432),
user=admin.username,
password=admin.password,
dbname=admin.database,
)
connection.autocommit = True
try:
with connection.cursor() as cursor:
cursor.execute(f'CREATE DATABASE "{_database_name}"')
finally:
connection.close()
database_url = admin.set(database=_database_name).render_as_string(hide_password=False)
os.environ["DATABASE_URL"] = database_url
backend_dir = Path(__file__).resolve().parent
subprocess.run(
[sys.executable, "-m", "alembic", "upgrade", "head"],
cwd=backend_dir,
env=os.environ.copy(),
check=True,
)
def pytest_unconfigure(config): # noqa: ARG001
"""Drop the temporary database and stop the container after the test session."""
if _container is None:
return
import psycopg2
if _database_name and _admin_url:
admin = make_url(_admin_url)
connection = psycopg2.connect(
host=_container.get_container_host_ip(),
port=_container.get_exposed_port(5432),
user=admin.username,
password=admin.password,
dbname=admin.database,
)
connection.autocommit = True
try:
with connection.cursor() as cursor:
cursor.execute(
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
"WHERE datname = %s AND pid <> pg_backend_pid()",
(_database_name,),
)
cursor.execute(f'DROP DATABASE IF EXISTS "{_database_name}"')
finally:
connection.close()
_container.stop()
# #endregion Test.PytestEarly.IntegrationDatabase

3
backend/src/api/routes/__tests__/conftest.py
View File

@@ -5,10 +5,9 @@
import os
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing")
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:test_auth")
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:test_auth")
os.environ.setdefault("DEV_MODE", "true")
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:test_main")
os.environ.setdefault("TASKS_DATABASE_URL", "sqlite:///:memory:test_tasks")
class FakeQuery:
"""Shared chainable query stub for route tests.
WARNING: filter() is predicate-blind — all ownership and permission filters are

5
backend/src/api/routes/__tests__/test_assistant_authz.py
View File

@@ -17,10 +17,7 @@ from fastapi import HTTPException
# Force isolated sqlite databases for test module before dependencies import.
os.environ.setdefault("DATABASE_URL", "sqlite:////tmp/ss_tools_assistant_authz.db")
os.environ.setdefault(
"TASKS_DATABASE_URL", "sqlite:////tmp/ss_tools_assistant_authz_tasks.db"
)
os.environ.setdefault(
"AUTH_DATABASE_URL", "sqlite:////tmp/ss_tools_assistant_authz_auth.db"
"DATABASE_URL", "sqlite:////tmp/ss_tools_assistant_authz_tasks.db"
)
from src.api.routes import assistant as assistant_module
from src.models.assistant import (

2
backend/src/api/routes/__tests__/test_clean_release_legacy_compat.py
View File

@@ -11,7 +11,7 @@ from fastapi.testclient import TestClient
os.environ.setdefault("DATABASE_URL", "sqlite:///./test_clean_release_legacy_compat.db")
os.environ.setdefault(
"AUTH_DATABASE_URL", "sqlite:///./test_clean_release_legacy_auth.db"
"DATABASE_URL", "sqlite:///./test_clean_release_legacy_auth.db"
)
from src.app import app
from src.dependencies import get_clean_release_repository

3
backend/src/api/routes/__tests__/test_migration_routes.py
View File

@@ -20,9 +20,8 @@ import os
# @SIDE_EFFECT_WARNING: os.environ mutation at module import time — no teardown. This bleeds into all subsequently collected tests. Migrate to pytest.fixture(autouse=True) with monkeypatch.setenv.
os.environ["DATABASE_URL"] = "sqlite:///:memory:"
# @SIDE_EFFECT_WARNING: os.environ mutation at module import time — no teardown. This bleeds into all subsequently collected tests. Migrate to pytest.fixture(autouse=True) with monkeypatch.setenv.
os.environ["TASKS_DATABASE_URL"] = "sqlite:///:memory:"
os.environ["DATABASE_URL"] = "sqlite:///:memory:"
# @SIDE_EFFECT_WARNING: os.environ mutation at module import time — no teardown. This bleeds into all subsequently collected tests. Migrate to pytest.fixture(autouse=True) with monkeypatch.setenv.
os.environ["AUTH_DATABASE_URL"] = "sqlite:///:memory:"
# @SIDE_EFFECT_WARNING: os.environ mutation at module import time — no teardown. This bleeds into all subsequently collected tests. Migrate to pytest.fixture(autouse=True) with monkeypatch.setenv.
os.environ["ENVIRONMENT"] = "testing"
from fastapi import HTTPException

4
backend/src/api/routes/encryption_health.py
View File

@@ -184,10 +184,10 @@ def _inventory_connections(config_manager: ConfigManager) -> list[dict]:
def _inventory_profile_tokens() -> list[dict]:
items = []
try:
from ...core.database import AuthSessionLocal
from ...core.database import SessionLocal
from ...models.profile import UserDashboardPreference
db_auth = AuthSessionLocal()
db_auth = SessionLocal()
try:
prefs = (
db_auth.query(UserDashboardPreference)

2
backend/src/api/routes/validation/__tests__/test_validation_api.py
View File

@@ -25,7 +25,7 @@
import os
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing")
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:test_auth")
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:test_auth")
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:test_main")
os.environ.setdefault("DEV_MODE", "true")

16
backend/src/app.py
View File

@@ -83,7 +83,7 @@ from .api.routes import (
from .api.routes.validation_tasks import router as validation_tasks
from .core.auth.security import get_password_hash
from ss_tools.shared.cot_logger import build_cot_event, get_trace_id, seed_trace_id, set_trace_id
from .core.database import AuthSessionLocal, init_db
from .core.database import SessionLocal, init_db
from .core.encryption_key import ensure_encryption_key
from .core.logger import belief_scope, logger
from .core.utils.network import NetworkError
@@ -130,11 +130,9 @@ def initialize_live_execution_composition() -> int:
# @INVARIANT The trusted live-composition bootstrap runs after AsyncJobRunner initialization and
# before scheduler startup; queued dispatch therefore cannot observe an unbootstrapped
# default root during normal application startup.
# @RATIONALE Alembic migrations removed from lifespan — they now run exclusively
# in docker/backend.entrypoint.sh (wait_for_db → alembic upgrade head).
# Running migrations in both places added ~5s startup overhead and masked
# partial failures. init_db() remains as a safety net for tables without
# dedicated Alembic migrations (e.g., newly added models during development).
# @RATIONALE Alembic migrations run before the application process starts.
# init_db() only records that schema initialization is owned by Alembic;
# runtime create_all() and inline ALTER TABLE repairs are intentionally absent.
@asynccontextmanager
async def lifespan(app: FastAPI):
# Startup
@@ -142,7 +140,7 @@ async def lifespan(app: FastAPI):
with belief_scope("startup_event"):
logger.reason("Ensure encryption subsystem availability")
ensure_encryption_key()
logger.reason("Initialize persistent database tables")
logger.reason("Verify persistent database schema ownership")
init_db()
logger.reason("Bootstrap initial admin user (idempotent)")
ensure_initial_admin_user()
@@ -174,7 +172,7 @@ async def lifespan(app: FastAPI):
# This generalizes the ValidationRun-specific cleanup above.
try:
from sqlalchemy.orm import Session as _Ses
from src.core.database import TasksSessionLocal as _TasksDb
from src.core.database import SessionLocal as _TasksDb
from src.models.task import TaskRecord as _TR
from datetime import datetime as _dt, timezone as _tz
@@ -303,7 +301,7 @@ def ensure_initial_admin_user() -> None:
"INITIAL_ADMIN_PASSWORD set via env var — visible to other processes",
error="Security concern: password in environment variable",
)
db = AuthSessionLocal()
db = SessionLocal()
try:
admin_role = db.query(Role).filter(Role.name == "Admin").first()
if not admin_role:

1
backend/src/core/__tests__/test_config_manager_compat.py
View File

@@ -163,7 +163,6 @@ def test_save_config_syncs_deletions_to_persistence():
# @BRIEF Ensure loading an existing DB-backed config also mirrors environment rows required by FK-backed runtime flows.
def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypatch):
manager = ConfigManager.__new__(ConfigManager)
manager.config_path = None
manager.raw_payload = {}
manager.config = AppConfig(environments=[], settings=GlobalSettings())
sync_calls = []

20
backend/src/core/auth/config.py
View File

@@ -6,11 +6,10 @@
# @RELATION DEPENDS_ON -> [EXT:Library:pydantic]
#
# @INVARIANT All sensitive configuration must be loaded from environment; no hardcoded secrets.
# @RATIONALE SECRET_KEY and AUTH_DATABASE_URL crash-early if env vars are missing.
# Dev fallback for AUTH_DATABASE_URL removed — Class 1 violation restored.
# @RATIONALE SECRET_KEY fails fast when absent; database selection is centralized in DATABASE_URL.
# @REJECTED Default secrets in source code rejected — Class 1 security violation:
# "super-secret-key-change-in-production" and "postgres:postgres" exposed
# secrets in version control. DEV_MODE fallback for AUTH_DATABASE_URL removed
# secrets in version control. DEV_MODE database fallback removed
# in [SEC:C-4] — hardcoded postgres:postgres is a clear-text credential leak.
from pydantic import Field, field_validator
@@ -34,9 +33,6 @@ class AuthConfig(BaseSettings):
JWT_AUDIENCE: str = Field(default="superset-tools-api", validation_alias="JWT_AUDIENCE")
JWT_ISSUER: str = Field(default="superset-tools", validation_alias="JWT_ISSUER")
# Database Settings
AUTH_DATABASE_URL: str = Field(default="", validation_alias="AUTH_DATABASE_URL")
# ADFS Settings
ADFS_CLIENT_ID: str = Field(default="", validation_alias="ADFS_CLIENT_ID")
ADFS_CLIENT_SECRET: str = Field(default="", validation_alias="ADFS_CLIENT_SECRET")
@@ -52,18 +48,6 @@ class AuthConfig(BaseSettings):
"Set it in .env or export it before starting the server."
)
@field_validator("AUTH_DATABASE_URL", mode="after")
@classmethod
def validate_auth_db_url(cls, v: str) -> str:
if v:
return v
raise ValueError(
"AUTH_DATABASE_URL environment variable is required. "
"Set it in .env or export it before starting the server. "
"For local development, create a .env file with AUTH_DATABASE_URL=postgresql+psycopg2://... "
"or use docker-compose.yml with pre-configured PostgreSQL."
)
# #endregion Core.Config.AuthConfigClass
# #region Core.Config.Auth.Config [TYPE Variable]

125
backend/src/core/config_manager.py
View File

@@ -1,12 +1,12 @@
# #region Core.ConfigManager [C:5] [TYPE Module] [SEMANTICS sqlalchemy, validate, migration, config-manager]
# #region Core.ConfigManager [C:5] [TYPE Module] [SEMANTICS sqlalchemy,validate,config-manager]
# @defgroup Core Module group.
#
# @BRIEF Manages application configuration persistence in DB with one-time migration from legacy JSON.
# @BRIEF Manages application configuration persistence in the database.
# @LAYER Domain
# @PRE Database schema for AppConfigRecord must be initialized.
# @POST Configuration is loaded into memory and logger is configured.
# @SIDE_EFFECT Performs DB I/O and may update global logging level.
# @DATA_CONTRACT Input[json, record] -> Model[AppConfig]
# @DATA_CONTRACT Input[record] -> Model[AppConfig]
# @INVARIANT Configuration must always be representable by AppConfig and persisted under global record id.
# @INVARIANT Environment passwords are encrypted at rest using Fernet (see [SEC:C-1]).
# @RELATION DEPENDS_ON -> [Core.ConfigModels.AppConfig]
@@ -15,16 +15,12 @@
# @RELATION DEPENDS_ON -> [Core.Encryption.EncryptionManager]
# @RELATION CALLS -> [Core.Context.Logger]
# @RELATION CALLS -> [Core.Logger.ConfigureLogger]
# @RATIONALE DB-backed ConfigManager chosen over JSON-file-only configuration because environment
# passwords (Superset DB, SSH keys) need encryption at rest via EncryptionManager. The
# one-time migration from legacy config.json ensures backward compatibility without manual
# reconfiguration. Lazy-loading with in-memory cache avoids DB reads on every get_config() call.
# @REJECTED Keeping all config in config.json was rejected — passwords would be stored in plaintext
# and the file is not synchronized across multiple backend instances. Reading from DB on
# every access was rejected — adds ~5ms latency per config read for no benefit since config
# changes are rare.
# @RATIONALE DB-backed ConfigManager is the sole application-settings source because environment
# passwords (Superset DB, SSH keys) need encryption at rest via EncryptionManager.
# Lazy-loading with in-memory cache avoids DB reads on every get_config() call.
# @REJECTED JSON configuration was rejected — passwords would be stored in plaintext and the file is
# not synchronized across multiple backend instances.
#
import json
import os
from pathlib import Path
from typing import Any
@@ -32,6 +28,7 @@ from typing import Any
from sqlalchemy.orm import Session
from ..models.config import AppConfigRecord
from .env_settings import storage_root_path
from ..models.mapping import Environment as EnvironmentRecord
from .config_models import AppConfig, Environment, GlobalSettings
from .database import SessionLocal
@@ -51,20 +48,13 @@ _AUTH_POLICY_FIELDS = ("auth_max_attempts", "auth_attempt_window", "auth_ban_dur
# @SIDE_EFFECT Performs DB I/O, OS path validation, and logger reconfiguration.
class ConfigManager:
# #region Core.ConfigManager.Init [C:4] [TYPE Function]
# @BRIEF Initialize manager state from persisted or migrated configuration.
# @PRE config_path is a non-empty string path.
# @BRIEF Initialize manager state from persisted database configuration.
# @PRE AppConfigRecord schema is initialized.
# @POST self.config is initialized as AppConfig and logger is configured.
# @SIDE_EFFECT Reads config sources and updates logging configuration.
# @DATA_CONTRACT Input(str config_path) -> Output(None; self.config: AppConfig)
def __init__(self, config_path: str = "config.json"):
# @SIDE_EFFECT Reads the database and updates logging configuration.
# @DATA_CONTRACT Input(AppConfigRecord) -> Output(None; self.config: AppConfig)
def __init__(self):
with belief_scope("ConfigManager.__init__"):
if not isinstance(config_path, str) or not config_path:
logger.explore(
"Invalid config_path provided", extra={"path": config_path}
)
raise ValueError("config_path must be a non-empty string")
logger.reason(f"Initializing ConfigManager with legacy path: {config_path}")
self.config_path = Path(config_path)
self.raw_payload: dict[str, Any] = {}
self.config: AppConfig = self._load_config()
configure_logger(self.config.settings.logging)
@@ -85,7 +75,7 @@ class ConfigManager:
# @SIDE_EFFECT May log a warning if key is missing.
# @RATIONALE Encryption manager cannot be initialized before ensure_encryption_key()
# is called at startup. This lazy init allows early bootstrap paths
# (legacy migration, test setup) to work without encryption.
# (early bootstrap and test setup) to work without encryption.
def _get_encryption_manager(self) -> EncryptionManager | None:
if not hasattr(self, "_encryption"):
try:
@@ -134,7 +124,7 @@ class ConfigManager:
# #region Core.ConfigManager.DecryptEnvPasswords [C:3] [TYPE Function]
# @BRIEF Decrypt all environment passwords in the AppConfig after loading from DB.
# @PRE config.environments is populated from DB payload.
# @POST Password fields are decrypted in-place (skips plaintext legacy values).
# @POST Password fields are decrypted in-place (skips plaintext stored values).
# @SIDE_EFFECT Logs warning for plaintext passwords (will be auto-encrypted on save).
# @SIDE_EFFECT Logs error if Fernet token exists but decryption fails (key mismatch).
# @RATIONALE Differs from ConnectionService._decrypt_password (which raises RuntimeError
@@ -207,6 +197,7 @@ class ConfigManager:
"Applied APP_TIMEZONE from env",
extra={"value": settings.app_timezone},
)
settings.storage.root_path = storage_root_path()
# #endregion Core.ConfigManager.ApplyFeaturesFromEnv
# #region Core.ConfigManager.DefaultConfig [C:2] [TYPE Function]
# @BRIEF Build default application configuration fallback.
@@ -218,14 +209,10 @@ class ConfigManager:
return config
# #endregion Core.ConfigManager.DefaultConfig
# #region Core.ConfigManager.SyncRawPayloadFromConfig [C:3] [TYPE Function]
# @BRIEF Merge typed AppConfig state into raw payload while preserving unsupported legacy sections.
# @BRIEF Serialize typed AppConfig state for the global database record.
def _sync_raw_payload_from_config(self) -> dict[str, Any]:
with belief_scope("ConfigManager._sync_raw_payload_from_config"):
typed_payload = self.config.model_dump()
merged_payload = dict(self.raw_payload or {})
merged_payload["environments"] = typed_payload.get("environments", [])
merged_payload["settings"] = typed_payload.get("settings", {})
self.raw_payload = merged_payload
self.raw_payload = self.config.model_dump()
# Plumbing for payload merge — not a decision; keep off INFO backbone.
logger.debug(
"Synchronized raw payload from typed config",
@@ -235,44 +222,14 @@ class ConfigManager:
"src": "ConfigManager._sync_raw_payload_from_config",
"payload": {
"environments_count": len(
merged_payload.get("environments", []) or []
self.raw_payload.get("environments", []) or []
),
"has_settings": "settings" in merged_payload,
"has_settings": "settings" in self.raw_payload,
},
},
)
return merged_payload
return self.raw_payload
# #endregion Core.ConfigManager.SyncRawPayloadFromConfig
# #region Core.ConfigManager.LoadFromLegacyFile [C:3] [TYPE Function]
# @BRIEF Load legacy JSON configuration for migration fallback path.
def _load_from_legacy_file(self) -> dict[str, Any]:
with belief_scope("ConfigManager._load_from_legacy_file"):
if not self.config_path.exists():
logger.reason(
"Legacy config file not found; using default payload",
extra={"path": str(self.config_path)},
)
return {}
logger.reason(
"Loading legacy config file", extra={"path": str(self.config_path)}
)
with self.config_path.open("r", encoding="utf-8") as fh:
payload = json.load(fh)
if not isinstance(payload, dict):
logger.explore(
"Legacy config payload is not a JSON object",
extra={
"path": str(self.config_path),
"type": type(payload).__name__,
},
)
raise ValueError("Legacy config payload must be a JSON object")
logger.reason(
"Legacy config file loaded successfully",
extra={"path": str(self.config_path), "keys": sorted(payload.keys())},
)
return payload
# #endregion Core.ConfigManager.LoadFromLegacyFile
# #region Core.ConfigManager.GetRecord [C:2] [TYPE Function]
# @BRIEF Resolve global configuration record from DB.
def _get_record(self, session: Session) -> AppConfigRecord | None:
@@ -294,7 +251,7 @@ class ConfigManager:
return record
# #endregion Core.ConfigManager.GetRecord
# #region Core.ConfigManager.LoadConfig [C:4] [TYPE Function]
# @BRIEF Load configuration from DB or perform one-time migration from legacy JSON.
# @BRIEF Load configuration from DB, persisting defaults when the global record is absent.
def _load_config(self) -> AppConfig:
with belief_scope("ConfigManager._load_config"):
session = SessionLocal()
@@ -312,6 +269,7 @@ class ConfigManager:
"settings": self.raw_payload.get("settings", {}),
}
)
self._apply_features_from_env(config.settings)
# Decrypt environment passwords after loading from DB (see [SEC:C-1])
self._decrypt_env_passwords(config)
self._sync_environment_records(session, config)
@@ -325,45 +283,12 @@ class ConfigManager:
)
return config
logger.reason(
"Database configuration record missing; attempting legacy file migration",
extra={"legacy_path": str(self.config_path)},
)
legacy_payload = self._load_from_legacy_file()
if legacy_payload:
self.raw_payload = dict(legacy_payload)
config = AppConfig.model_validate(
{
"environments": self.raw_payload.get("environments", []),
"settings": self.raw_payload.get("settings", {}),
}
)
# Legacy config is plaintext — decrypt is a no-op but safe
self._decrypt_env_passwords(config)
self._apply_features_from_env(config.settings)
logger.reason(
"Legacy payload validated; persisting migrated configuration to database",
extra={
"environments_count": len(config.environments),
"payload_keys": sorted(self.raw_payload.keys()),
},
)
self._save_config_to_db(config, session=session)
return config
logger.reason(
"No persisted config found; falling back to default configuration"
"No persisted database config found; persisting defaults"
)
config = self._default_config()
self.raw_payload = config.model_dump()
self._save_config_to_db(config, session=session)
return config
except (json.JSONDecodeError, TypeError, ValueError) as exc:
logger.explore(
"Recoverable config load failure; falling back to default configuration",
extra={"error": str(exc), "legacy_path": str(self.config_path)},
)
config = self._default_config()
self.raw_payload = config.model_dump()
return config
except Exception as exc:
logger.explore(
"Critical config load failure; re-raising persistence or validation error",

691
backend/src/core/database.py
View File

@@ -1,670 +1,81 @@
# #region Core.Database.DatabaseModule [C:3] [TYPE Module] [SEMANTICS sqlalchemy, connection, session]
# @defgroup Core Module group.
#
# @BRIEF Configures database connection and session management (PostgreSQL-first).
# #region Core.Database.DatabaseModule [C:3] [TYPE Module] [SEMANTICS sqlalchemy,connection,session]
# @defgroup Core Database connection and session compatibility layer.
# @BRIEF Provides one application database engine and one session factory.
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [Models.Mapping.MappingModels]
# @RELATION DEPENDS_ON -> [Core.Config.Auth.Config]
#
# @INVARIANT A single engine instance is used for the entire application.
# @INVARIANT DATABASE_URL is the only database authority; compatibility aliases share its objects.
import os
from pathlib import Path
from sqlalchemy import create_engine, inspect, text
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
# Import models to ensure they're registered with Base
from ..models import (
agent as _agent_models, # noqa: F401
agent_run as _agent_run_models, # noqa: F401
api_key as _api_key_models, # noqa: F401
assistant as _assistant_models, # noqa: F401
auth as _auth_models, # noqa: F401
clean_release as _clean_release_models, # noqa: F401
config as _config_models, # noqa: F401
dashboard_release as _dashboard_release_models, # noqa: F401
deployment as _deployment_models, # noqa: F401
git as _git_models, # noqa: F401
lineage as _lineage_models, # noqa: F401
llm as _llm_models, # noqa: F401
load_testing as _load_testing_models, # noqa: F401
maintenance as _maintenance_models, # noqa: F401
profile as _profile_models, # noqa: F401
scenario_registry as _scenario_registry_models, # noqa: F401
task as _task_models, # noqa: F401
verification_run as _verification_run_models, # noqa: F401
from src.models import ( # noqa: F401
agent as _agent,
agent_run as _agent_run,
api_key as _api_key,
assistant as _assistant,
auth as _auth,
clean_release as _clean_release,
config as _config,
dashboard as _dashboard,
dashboard_release as _dashboard_release,
deployment as _deployment,
filter_state as _filter_state,
git as _git,
lineage as _lineage,
llm as _llm,
load_testing as _load_testing,
maintenance as _maintenance,
profile as _profile,
report as _report,
scenario_approval as _scenario_approval,
scenario_artifact as _scenario_artifact,
scenario_automation as _scenario_automation,
scenario_checkpoint as _scenario_checkpoint,
scenario_investigation as _scenario_investigation,
scenario_registry as _scenario_registry,
scenario_run as _scenario_run,
scenario_worker as _scenario_worker,
storage as _storage,
task as _task,
translate as _translate,
verification_run as _verification_run,
)
from ..models.mapping import Base
from .auth.config import auth_config
from .logger import belief_scope, logger
from src.models.mapping import Base # noqa: F401
from .env_settings import database_url
# #region Core.Database.BASEDIR [C:1] [TYPE Variable]
# @BRIEF Base directory for the backend.
BASE_DIR = Path(__file__).resolve().parent.parent.parent
# #endregion Core.Database.BASEDIR
# #region Core.Database.DATABASEURL [C:1] [TYPE Constant]
# @BRIEF URL for the main application database. Read from env; crashes if unset.
# @RATIONALE DATABASE_URL is required. POSTGRES_URL removed — use only DATABASE_URL.
# Crashes at import if unset.
# @REJECTED Hardcoded postgres:postgres@localhost in source code rejected — exposes
# database credentials in version control (Class 1 security violation).
# DEV_MODE fallback removed — same violation via env toggle.
DATABASE_URL = os.getenv("DATABASE_URL")
if not DATABASE_URL:
raise RuntimeError("DATABASE_URL environment variable is required. Set it before starting the server. For local development, create a .env file or use docker-compose.yml.")
# #endregion Core.Database.DATABASEURL
# #region Core.Database.TASKSDATABASEURL [C:1] [TYPE Constant]
# @BRIEF URL for the tasks execution database.
# Defaults to DATABASE_URL to keep task logs in the same PostgreSQL instance.
TASKS_DATABASE_URL = os.getenv("TASKS_DATABASE_URL", DATABASE_URL)
# #endregion Core.Database.TASKSDATABASEURL
# #region Core.Database.AUTHDATABASEURL [C:1] [TYPE Constant]
# @BRIEF URL for the authentication database.
AUTH_DATABASE_URL = os.getenv("AUTH_DATABASE_URL", auth_config.AUTH_DATABASE_URL)
# #endregion Core.Database.AUTHDATABASEURL
DATABASE_URL = database_url()
# #region Core.Database.Engine [C:1] [TYPE Variable]
# @BRIEF SQLAlchemy engine for mappings database.
# @SIDE_EFFECT Creates database engine and manages connection pool.
def _build_engine(db_url: str):
with belief_scope("_build_engine"):
if db_url.startswith("sqlite"):
return create_engine(db_url, connect_args={"check_same_thread": False})
return create_engine(db_url, pool_pre_ping=True)
if db_url.startswith("sqlite"):
return create_engine(db_url, connect_args={"check_same_thread": False})
return create_engine(db_url, pool_pre_ping=True)
engine = _build_engine(DATABASE_URL)
# #endregion Core.Database.Engine
# #region Core.Database.TasksEngine [C:1] [TYPE Variable]
# @BRIEF SQLAlchemy engine for tasks database.
tasks_engine = _build_engine(TASKS_DATABASE_URL)
# #endregion Core.Database.TasksEngine
# #region Core.Database.AuthEngine [C:1] [TYPE Variable]
# @BRIEF SQLAlchemy engine for authentication database.
auth_engine = _build_engine(AUTH_DATABASE_URL)
# #endregion Core.Database.AuthEngine
# #region Core.Database.SessionLocal [C:1] [TYPE Class]
# @BRIEF A session factory for the main mappings database.
# @PRE engine is initialized.
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
# #endregion Core.Database.SessionLocal
# #region Core.Database.TasksSessionLocal [C:1] [TYPE Class]
# @BRIEF A session factory for the tasks execution database.
# @PRE tasks_engine is initialized.
TasksSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=tasks_engine)
# #endregion Core.Database.TasksSessionLocal
# #region Core.Database.AuthSessionLocal [C:1] [TYPE Class]
# @BRIEF A session factory for the authentication database.
# @PRE auth_engine is initialized.
AuthSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=auth_engine)
# #endregion Core.Database.AuthSessionLocal
# #region Core.Database.EnsureUserDashboardPreferencesColumns [C:3] [TYPE Function]
# @BRIEF Applies additive schema upgrades for user_dashboard_preferences table.
# @PRE bind_engine points to application database where profile table is stored.
# @POST Missing columns are added without data loss.
# @RELATION DEPENDS_ON -> [Core.Database.Engine]
def _ensure_user_dashboard_preferences_columns(bind_engine):
with belief_scope("_ensure_user_dashboard_preferences_columns"):
table_name = "user_dashboard_preferences"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {str(column.get("name") or "").strip() for column in inspector.get_columns(table_name)}
alter_statements = []
if "git_username" not in existing_columns:
alter_statements.append("ALTER TABLE user_dashboard_preferences ADD COLUMN git_username VARCHAR")
if "git_email" not in existing_columns:
alter_statements.append("ALTER TABLE user_dashboard_preferences ADD COLUMN git_email VARCHAR")
if "git_personal_access_token_encrypted" not in existing_columns:
alter_statements.append("ALTER TABLE user_dashboard_preferences ADD COLUMN git_personal_access_token_encrypted VARCHAR")
if "start_page" not in existing_columns:
alter_statements.append("ALTER TABLE user_dashboard_preferences ADD COLUMN start_page VARCHAR NOT NULL DEFAULT 'dashboards'")
if "auto_open_task_drawer" not in existing_columns:
alter_statements.append("ALTER TABLE user_dashboard_preferences ADD COLUMN auto_open_task_drawer BOOLEAN NOT NULL DEFAULT TRUE")
if "dashboards_table_density" not in existing_columns:
alter_statements.append("ALTER TABLE user_dashboard_preferences ADD COLUMN dashboards_table_density VARCHAR NOT NULL DEFAULT 'comfortable'")
if "show_only_slug_dashboards" not in existing_columns:
alter_statements.append("ALTER TABLE user_dashboard_preferences ADD COLUMN show_only_slug_dashboards BOOLEAN NOT NULL DEFAULT FALSE")
if not alter_statements:
return
try:
with bind_engine.begin() as connection:
for statement in alter_statements:
connection.execute(text(statement))
except Exception as migration_error:
logger.explore(
"Profile preference additive migration failed",
extra={"src": "database", "error": str(migration_error)},
)
# #endregion Core.Database.EnsureUserDashboardPreferencesColumns
# #region Core.Database.EnsureUserDashboardPreferencesHealthColumns [C:3] [TYPE Function]
# @BRIEF Applies additive schema upgrades for user_dashboard_preferences table (health fields).
# @RELATION DEPENDS_ON -> [Core.Database.Engine]
def _ensure_user_dashboard_preferences_health_columns(bind_engine):
with belief_scope("_ensure_user_dashboard_preferences_health_columns"):
table_name = "user_dashboard_preferences"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {str(column.get("name") or "").strip() for column in inspector.get_columns(table_name)}
alter_statements = []
if "telegram_id" not in existing_columns:
alter_statements.append("ALTER TABLE user_dashboard_preferences ADD COLUMN telegram_id VARCHAR")
if "email_address" not in existing_columns:
alter_statements.append("ALTER TABLE user_dashboard_preferences ADD COLUMN email_address VARCHAR")
if "notify_on_fail" not in existing_columns:
alter_statements.append("ALTER TABLE user_dashboard_preferences ADD COLUMN notify_on_fail BOOLEAN NOT NULL DEFAULT TRUE")
if not alter_statements:
return
try:
with bind_engine.begin() as connection:
for statement in alter_statements:
connection.execute(text(statement))
except Exception as migration_error:
logger.explore(
"Profile health preference additive migration failed",
extra={"src": "database", "error": str(migration_error)},
)
# #endregion Core.Database.EnsureUserDashboardPreferencesHealthColumns
# #region Core.Database.EnsureLlmValidationResultsColumns [C:3] [TYPE Function]
# @BRIEF Applies additive schema upgrades for llm_validation_results table.
# @RELATION DEPENDS_ON -> [Core.Database.Engine]
def _ensure_llm_validation_results_columns(bind_engine):
with belief_scope("_ensure_llm_validation_results_columns"):
table_name = "llm_validation_results"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {str(column.get("name") or "").strip() for column in inspector.get_columns(table_name)}
alter_statements = []
if "task_id" not in existing_columns:
alter_statements.append("ALTER TABLE llm_validation_results ADD COLUMN task_id VARCHAR")
if "environment_id" not in existing_columns:
alter_statements.append("ALTER TABLE llm_validation_results ADD COLUMN environment_id VARCHAR")
if not alter_statements:
return
try:
with bind_engine.begin() as connection:
for statement in alter_statements:
connection.execute(text(statement))
except Exception as migration_error:
logger.explore(
"ValidationRecord additive migration failed",
extra={"src": "database", "error": str(migration_error)},
)
# #endregion Core.Database.EnsureLlmValidationResultsColumns
# #region Core.Database.EnsureGitServerConfigsColumns [C:3] [TYPE Function]
# @BRIEF Applies additive schema upgrades for git_server_configs table.
# @PRE bind_engine points to application database.
# @POST Missing columns are added without data loss.
# @RELATION DEPENDS_ON -> [Core.Database.Engine]
def _ensure_git_server_configs_columns(bind_engine):
with belief_scope("_ensure_git_server_configs_columns"):
table_name = "git_server_configs"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {str(column.get("name") or "").strip() for column in inspector.get_columns(table_name)}
alter_statements = []
if "default_branch" not in existing_columns:
alter_statements.append("ALTER TABLE git_server_configs ADD COLUMN default_branch VARCHAR NOT NULL DEFAULT 'prod'")
if not alter_statements:
return
try:
with bind_engine.begin() as connection:
for statement in alter_statements:
connection.execute(text(statement))
except Exception as migration_error:
logger.explore(
"GitServerConfig preference additive migration failed",
extra={"src": "database", "error": str(migration_error)},
)
# #endregion Core.Database.EnsureGitServerConfigsColumns
# #region Core.Database.EnsureAuthUsersColumns [C:3] [TYPE Function]
# @BRIEF Applies additive schema upgrades for auth users table.
# @PRE bind_engine points to authentication database.
# @POST Missing columns are added without data loss.
# @RELATION DEPENDS_ON -> [Core.Database.AuthEngine]
def _ensure_auth_users_columns(bind_engine):
with belief_scope("_ensure_auth_users_columns"):
table_name = "users"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {str(column.get("name") or "").strip() for column in inspector.get_columns(table_name)}
alter_statements = []
if "full_name" not in existing_columns:
alter_statements.append("ALTER TABLE users ADD COLUMN full_name VARCHAR")
if "is_ad_user" not in existing_columns:
alter_statements.append("ALTER TABLE users ADD COLUMN is_ad_user BOOLEAN NOT NULL DEFAULT FALSE")
if not alter_statements:
logger.reason(
"Auth users schema already up to date",
extra={"table": table_name, "columns": sorted(existing_columns)},
)
return
logger.reason(
"Applying additive auth users schema migration",
extra={"table": table_name, "statements": alter_statements},
)
try:
with bind_engine.begin() as connection:
for statement in alter_statements:
connection.execute(text(statement))
logger.reason(
"Auth users schema migration completed",
extra={
"table": table_name,
"added_columns": [stmt.split(" ADD COLUMN ", 1)[1].split()[0] for stmt in alter_statements],
},
)
except Exception as migration_error:
logger.explore(
"Auth users additive migration failed",
extra={"src": "database", "error": str(migration_error)},
)
raise
# #endregion Core.Database.EnsureAuthUsersColumns
# #region Core.Database.EnsureRolesIsAdminColumn [C:3] [TYPE Function]
# @BRIEF Add is_admin column to roles table if missing (additive migration).
# @PRE Database connection is active.
# @POST roles.is_admin column exists (BOOLEAN, default FALSE).
# @SIDE_EFFECT Executes ALTER TABLE on the auth database.
# @RATIONALE The is_admin=True backfill for the Admin role is owned by
# AuthRepository.ensure_admin_role() (called at startup self-heal). This migration only
# guarantees the column exists — it no longer duplicates the flag backfill.
def _ensure_roles_is_admin_column(bind_engine):
with belief_scope("_ensure_roles_is_admin_column"):
table_name = "roles"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {str(column.get("name") or "").strip() for column in inspector.get_columns(table_name)}
if "is_admin" in existing_columns:
logger.reason("roles.is_admin column already exists")
return
alter = "ALTER TABLE roles ADD COLUMN is_admin BOOLEAN NOT NULL DEFAULT FALSE"
try:
with bind_engine.begin() as connection:
connection.execute(text(alter))
logger.reason("Added roles.is_admin column", extra={"statement": alter})
except Exception as migration_error:
logger.explore(
"roles.is_admin additive migration failed",
extra={"error": str(migration_error)},
)
raise
# #endregion Core.Database.EnsureRolesIsAdminColumn
# #region Core.Database.EnsureFilterSourceEnumValues [C:3] [TYPE Function]
# @BRIEF Adds missing FilterSource enum values to the PostgreSQL native filtersource type.
# @PRE bind_engine points to application database with imported_filters table.
# @POST New enum values are available without data loss.
# @RELATION DEPENDS_ON -> [Core.Database.Engine]
def _ensure_filter_source_enum_values(bind_engine):
with belief_scope("_ensure_filter_source_enum_values"):
try:
with bind_engine.connect() as connection:
# Check if the native enum type exists
result = connection.execute(text("SELECT t.typname FROM pg_type t JOIN pg_namespace n ON t.typnamespace = n.oid WHERE t.typname = 'filtersource' AND n.nspname = 'public'"))
if result.fetchone() is None:
logger.reason("filtersource enum type does not exist yet; skipping migration")
return
# Get existing enum values
result = connection.execute(text("SELECT e.enumlabel FROM pg_enum e JOIN pg_type t ON e.enumtypid = t.oid WHERE t.typname = 'filtersource' ORDER BY e.enumsortorder"))
existing_values = {row[0] for row in result.fetchall()}
required_values = ["SUPERSET_PERMALINK", "SUPERSET_NATIVE_FILTERS_KEY"]
missing_values = [v for v in required_values if v not in existing_values]
if not missing_values:
logger.reason(
"filtersource enum already up to date",
extra={"existing": sorted(existing_values)},
)
return
logger.reason(
"Adding missing values to filtersource enum",
extra={"missing": missing_values},
)
for value in missing_values:
connection.execute(text(f"ALTER TYPE filtersource ADD VALUE IF NOT EXISTS '{value}'"))
connection.commit()
logger.reason(
"filtersource enum migration completed",
extra={"added": missing_values},
)
except Exception as migration_error:
logger.explore(
"FilterSource enum additive migration failed",
extra={"src": "database", "error": str(migration_error)},
)
# #endregion Core.Database.EnsureFilterSourceEnumValues
# #region Core.Database.EnsureTranslationJobsColumns [C:3] [TYPE Function]
# @BRIEF Applies additive schema upgrades for translation_jobs table.
# @PRE bind_engine points to application database.
# @POST Missing columns are added without data loss.
# @RELATION DEPENDS_ON -> [Core.Database.Engine]
def _ensure_translation_jobs_columns(bind_engine):
with belief_scope("_ensure_translation_jobs_columns"):
table_name = "translation_jobs"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {str(column.get("name") or "").strip() for column in inspector.get_columns(table_name)}
if "environment_id" not in existing_columns:
try:
with bind_engine.begin() as connection:
connection.execute(text("ALTER TABLE translation_jobs ADD COLUMN environment_id VARCHAR"))
logger.reflect(
"Added environment_id column to translation_jobs",
)
except Exception as migration_error:
logger.explore(
"Failed to add environment_id to translation_jobs",
extra={"error": str(migration_error)},
)
raise
if "target_database_id" not in existing_columns:
try:
with bind_engine.begin() as connection:
connection.execute(text("ALTER TABLE translation_jobs ADD COLUMN target_database_id VARCHAR"))
logger.reflect(
"Added target_database_id column to translation_jobs",
)
except Exception as migration_error:
logger.explore(
"Failed to add target_database_id to translation_jobs",
extra={"error": str(migration_error)},
)
# Columns added to model AFTER initial Alembic migration — no Alembic migration exists
if "target_language_column" not in existing_columns:
try:
with bind_engine.begin() as connection:
connection.execute(text("ALTER TABLE translation_jobs ADD COLUMN target_language_column VARCHAR"))
logger.reflect("Added target_language_column to translation_jobs")
except Exception as migration_error:
logger.explore(
"Failed to add target_language_column to translation_jobs",
extra={"error": str(migration_error)},
)
if "target_source_column" not in existing_columns:
try:
with bind_engine.begin() as connection:
connection.execute(text("ALTER TABLE translation_jobs ADD COLUMN target_source_column VARCHAR"))
logger.reflect("Added target_source_column to translation_jobs")
except Exception as migration_error:
logger.explore(
"Failed to add target_source_column to translation_jobs",
extra={"error": str(migration_error)},
)
if "target_source_language_column" not in existing_columns:
try:
with bind_engine.begin() as connection:
connection.execute(text("ALTER TABLE translation_jobs ADD COLUMN target_source_language_column VARCHAR"))
logger.reflect("Added target_source_language_column to translation_jobs")
except Exception as migration_error:
logger.explore(
"Failed to add target_source_language_column to translation_jobs",
extra={"error": str(migration_error)},
)
if "disable_reasoning" not in existing_columns:
try:
with bind_engine.begin() as connection:
connection.execute(text("ALTER TABLE translation_jobs ADD COLUMN disable_reasoning BOOLEAN NOT NULL DEFAULT FALSE"))
logger.reflect("Added disable_reasoning column to translation_jobs")
except Exception as migration_error:
logger.explore(
"Failed to add disable_reasoning to translation_jobs",
extra={"error": str(migration_error)},
)
if "include_source_reference" not in existing_columns:
try:
with bind_engine.begin() as connection:
connection.execute(text("ALTER TABLE translation_jobs ADD COLUMN include_source_reference BOOLEAN NOT NULL DEFAULT TRUE"))
logger.reflect("Added include_source_reference column to translation_jobs")
except Exception as migration_error:
logger.explore(
"Failed to add include_source_reference to translation_jobs",
extra={"error": str(migration_error)},
)
# #endregion Core.Database.EnsureTranslationJobsColumns
# #region Core.Database.EnsureTranslationSchedulesColumns [C:3] [TYPE Function]
# @BRIEF Applies additive schema upgrades for translation_schedules table.
# @PRE bind_engine points to application database.
# @POST Missing columns are added without data loss.
# @RELATION DEPENDS_ON -> [Core.Database.Engine]
def _ensure_translation_schedules_columns(bind_engine):
with belief_scope("_ensure_translation_schedules_columns"):
table_name = "translation_schedules"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {str(column.get("name") or "").strip() for column in inspector.get_columns(table_name)}
alter_statements = []
if "execution_mode" not in existing_columns:
alter_statements.append("ALTER TABLE translation_schedules ADD COLUMN execution_mode VARCHAR NOT NULL DEFAULT 'full'")
if not alter_statements:
return
try:
with bind_engine.begin() as connection:
for statement in alter_statements:
connection.execute(text(statement))
except Exception as migration_error:
logger.explore(
"TranslationSchedule additive migration failed",
extra={"src": "database", "error": str(migration_error)},
)
# #endregion Core.Database.EnsureTranslationSchedulesColumns
# #region Core.Database.EnsureDictionaryEntriesColumns [C:3] [TYPE Function]
# @BRIEF Additive migration for dictionary_entries origin tracking columns.
# @RELATION DEPENDS_ON -> [Core.Database.Engine]
def _ensure_dictionary_entries_columns(bind_engine):
with belief_scope("_ensure_dictionary_entries_columns"):
table_name = "dictionary_entries"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {str(column.get("name") or "").strip() for column in inspector.get_columns(table_name)}
alter_statements = []
if "origin_run_id" not in existing_columns:
alter_statements.append("ALTER TABLE dictionary_entries ADD COLUMN origin_run_id VARCHAR")
if "origin_row_key" not in existing_columns:
alter_statements.append("ALTER TABLE dictionary_entries ADD COLUMN origin_row_key VARCHAR")
if "origin_user_id" not in existing_columns:
alter_statements.append("ALTER TABLE dictionary_entries ADD COLUMN origin_user_id VARCHAR")
if not alter_statements:
return
try:
with bind_engine.begin() as connection:
for statement in alter_statements:
connection.execute(text(statement))
except Exception as migration_error:
logger.explore(
"DictionaryEntry additive migration failed",
extra={"src": "database", "error": str(migration_error)},
)
# #endregion Core.Database.EnsureDictionaryEntriesColumns
# #region Core.Database.InitDb [C:3] [TYPE Function]
# @ingroup Core
# @BRIEF Creates any missing tables via create_all() — safety net for development.
# @PRE engine, tasks_engine and auth_engine are initialized.
# In Docker: Alembic migrations already applied via entrypoint.sh.
# @POST All tables from SQLAlchemy models exist in all databases.
# @SIDE_EFFECT Creates physical database tables if they don't exist.
# @RATIONALE create_all() is idempotent — it only creates tables that don't exist,
# never alters existing ones. In Docker, Alembic runs first in entrypoint.sh
# (wait_for_db → alembic upgrade head), so init_db() is a no-op for production.
# For local development (run.sh without Docker), this provides a safety net.
# @REJECTED Removing init_db() entirely was rejected — local development without
# Docker would require manual Alembic setup, increasing friction for developers.
# @REJECTED _ensure_*() inline additive migrations removed — they were duplicating
# Alembic logic, were not versioned, and created hidden schema drift.
# All schema changes must now go through Alembic.
def init_db():
with belief_scope("init_db"):
Base.metadata.create_all(bind=engine)
Base.metadata.create_all(bind=tasks_engine)
Base.metadata.create_all(bind=auth_engine)
# Safety net for additive column changes that may have been missed in
# Alembic (e.g. include_source_reference on translation_jobs).
# These are no-op if columns already exist. Primary schema changes
# must still go through Alembic.
try:
_ensure_translation_jobs_columns(engine)
except Exception as e:
logger.explore("Failed to run translation_jobs column safety ensure", error=str(e))
"""Compatibility no-op; schema ownership belongs exclusively to Alembic."""
return None
# #endregion Core.Database.InitDb
# #region Core.Database.GetDb [C:3] [TYPE Function]
# @ingroup Core
# @BRIEF Dependency for getting a database session.
# @PRE SessionLocal is initialized.
# @POST Session is closed after use.
# @RELATION DEPENDS_ON -> [Core.Database.SessionLocal]
def get_db():
with belief_scope("get_db"):
db = SessionLocal()
try:
yield db
finally:
db.close()
db = SessionLocal()
try:
yield db
finally:
db.close()
# #endregion Core.Database.GetDb
# #region Core.Database.GetTasksDb [C:3] [TYPE Function]
# @ingroup Core
# @BRIEF Dependency for getting a tasks database session.
# @PRE TasksSessionLocal is initialized.
# @POST Session is closed after use.
# @RELATION DEPENDS_ON -> [Core.Database.TasksSessionLocal]
def get_tasks_db():
with belief_scope("get_tasks_db"):
db = TasksSessionLocal()
try:
yield db
finally:
db.close()
yield from get_db()
# #endregion Core.Database.GetTasksDb
# #region Core.Database.GetAuthDb [C:3] [TYPE Function]
# @ingroup Core
# @BRIEF Dependency for getting an authentication database session.
# @PRE AuthSessionLocal is initialized.
# @POST Session is closed after use.
# @DATA_CONTRACT None -> Output[EXT:Library:sqlalchemy.orm.Session]
# @RELATION DEPENDS_ON -> [Core.Database.AuthSessionLocal]
def get_auth_db():
with belief_scope("get_auth_db"):
db = AuthSessionLocal()
try:
yield db
finally:
db.close()
yield from get_db()
# #endregion Core.Database.GetAuthDb
# #endregion Core.Database.DatabaseModule

49
backend/src/core/env_settings.py Normal file
View File

@@ -0,0 +1,49 @@
# #region Core.EnvSettings [C:3] [TYPE Module] [SEMANTICS infrastructure,environment,settings]
# @defgroup Core Infrastructure environment settings.
# @BRIEF Resolves process environment values used by application infrastructure.
# @INVARIANT DATABASE_URL and STORAGE_ROOT_PATH are the only database and storage authorities.
import os
from pathlib import Path
# #region Core.EnvSettings.DatabaseUrl [C:2] [TYPE Function]
# @ingroup Core
# @BRIEF Return the required application database URL.
def database_url() -> str:
value = os.getenv("DATABASE_URL", "").strip()
if not value:
raise RuntimeError("DATABASE_URL environment variable is required")
return value
# #endregion Core.EnvSettings.DatabaseUrl
# #region Core.EnvSettings.StorageRootPath [C:2] [TYPE Function]
# @ingroup Core
# @BRIEF Return the canonical filesystem storage root.
def storage_root_path() -> str:
value = os.getenv("STORAGE_ROOT_PATH", "/app/storage").strip() or "/app/storage"
path = Path(value)
if not path.is_absolute():
raise RuntimeError("STORAGE_ROOT_PATH must be an absolute path")
resolved = path.resolve(strict=False)
project_root = Path(__file__).resolve().parents[3]
approved_roots = (
Path("/app/storage"),
project_root,
project_root / ".." / "ss-tools-storage",
)
if resolved == Path("/") or not any(
resolved == root.resolve(strict=False) or root.resolve(strict=False) in resolved.parents
for root in approved_roots
):
raise RuntimeError(
"STORAGE_ROOT_PATH must be /app/storage or inside the approved "
"project or sibling ss-tools-storage directory"
)
return str(resolved)
# #endregion Core.EnvSettings.StorageRootPath
# #endregion Core.EnvSettings

4
backend/src/core/scheduler.py
View File

@@ -16,7 +16,7 @@ from ss_tools.shared.cot_logger import seed_trace_id
from .async_job_runner import AsyncJobRunner
from .config_manager import ConfigManager
from .database import TASKS_DATABASE_URL, SessionLocal
from .database import DATABASE_URL, SessionLocal
from .logger import belief_scope, logger
@@ -285,7 +285,7 @@ class SchedulerService:
# Args passed to jobs must be simple (primitives) for reliable pickling.
jobstores = {
'default': SQLAlchemyJobStore(
url=TASKS_DATABASE_URL,
url=DATABASE_URL,
tablename='apscheduler_jobs',
)
}

2
backend/src/core/task_manager/context.py
View File

@@ -57,7 +57,7 @@ class TaskContext:
Usage:
def execute(params: dict, context: TaskContext = None):
if context:
context.logger.info("Starting process")
context.logger.reason("Starting process")
context.logger.progress("Processing items", percent=50)
# ... plugin logic
"""

39
backend/src/core/task_manager/event_bus.py
View File

@@ -25,6 +25,7 @@ from typing import Any
from ss_tools.shared.cot_logger import (
CanonicalCotEvent,
build_cot_event,
seed_trace_id,
)
@@ -130,7 +131,12 @@ class EventBus:
self.log_persistence_service.add_logs, task_id, logs
)
except Exception as e:
logger.error(f"Failed to flush logs for task {task_id}: {e}")
logger.explore(
"Failed to flush buffered task logs",
extra={"src": "EventBus._flush_logs"},
payload={"task_id": task_id, "log_count": len(logs)},
error=str(e),
)
if task_id not in self._log_buffer:
self._log_buffer[task_id] = []
merged = logs + self._log_buffer[task_id]
@@ -151,7 +157,14 @@ class EventBus:
self.log_persistence_service.add_logs, task_id, logs
)
except Exception as e:
logger.error(f"Failed to flush logs for task {task_id}: {e}")
logger.explore(
"Failed to flush task logs",
extra={"src": "EventBus.flush_task_logs"},
payload={"task_id": task_id, "log_count": len(logs)},
error=str(e),
)
pending = self._log_buffer.get(task_id, [])
self._log_buffer[task_id] = self._trim_buffer(logs + pending)
# #endregion Core.EventBus.FlushTaskLogs
def _trim_buffer(self, entries: list[LogEntry]) -> list[LogEntry]:
@@ -181,14 +194,30 @@ class EventBus:
self,
task_id: str,
level: str | None = None,
task_logs_list: list[LogEntry] | None = None,
message: str | None = None,
*,
task_logs_list: list[LogEntry] | None = None,
event: CanonicalCotEvent | None = None,
source: str | None = None,
metadata: dict[str, Any] | None = None,
context: dict[str, Any] | None = None,
):
if not should_log_task_level(level):
# Canonical callers provide level in the event. Legacy callers provide it
# positionally; normalize before applying the configured level filter.
effective_level = event["level"] if event is not None else (level or "INFO")
if not should_log_task_level(effective_level):
return
if event is None:
raise ValueError("EventBus.add_log requires a canonical event")
marker = "EXPLORE" if str(level or "INFO").upper() in {"WARNING", "ERROR"} else "REASON"
event = build_cot_event(
src=source if source and "." in source else f"task.{source or 'system'}",
marker=marker,
intent=message or "Task event",
payload=metadata or context,
error=(message or "Task event") if marker == "EXPLORE" else None,
level=level or "INFO",
)
level = event["level"]
log_entry = LogEntry(
timestamp=datetime.fromisoformat(event["ts"]),
**event,

16
backend/src/core/task_manager/log_export.py
View File

@@ -87,26 +87,32 @@ def row_to_export_record(
plugin_id: str | None = None,
domain: str = "task",
) -> dict[str, Any]:
ts = row.get("timestamp")
ts = row.get("timestamp") or row.get("ts")
if isinstance(ts, datetime):
ts_str = ts.isoformat()
else:
ts_str = str(ts or "")
parsed = parse_cot_message(row.get("message"))
if parsed:
row = {**row, **parsed}
record: dict[str, Any] = {
"id": row.get("id"),
"ts": ts_str,
"level": str(row.get("level") or "INFO").upper(),
"task_id": row.get("task_id"),
"src": str(row.get("src") or "task.system"),
"src": str(row.get("src") or row.get("source") or "task.system"),
"marker": str(row.get("marker") or "REASON"),
"intent": redact_text(str(row.get("intent") or "")),
"intent": redact_text(str(row.get("intent") or row.get("message") or "")),
"trace_id": str(row.get("trace_id") or ""),
}
if row.get("span_id"):
record["span_id"] = row["span_id"]
if row.get("payload") is not None:
record["payload"] = redact_metadata(row["payload"])
payload = row.get("payload")
if payload is None:
payload = row.get("metadata")
if payload is not None:
record["payload"] = redact_metadata(payload)
if row.get("error"):
record["error"] = redact_text(str(row["error"]))
return record

15
backend/src/core/task_manager/manager.py
View File

@@ -124,14 +124,23 @@ class TaskManager:
# #region Core.Manager.MakeAddLogCallback [C:3] [TYPE Function]
# @BRIEF Create an async closure for adding logs that looks up the task and delegates to EventBus.
def _make_add_log_callback(self):
async def _add_log(task_id, event=None):
async def _add_log(task_id, level=None, message=None, source="system", metadata=None, context=None, event=None, **_kwargs):
task = self.graph.get_task(task_id)
if not task:
return
if event is None:
raise ValueError("Task log callback requires a canonical event")
from ss_tools.shared.cot_logger import build_cot_event
marker = "EXPLORE" if str(level or "INFO").upper() in {"WARNING", "ERROR"} else "REASON"
event = build_cot_event(
src=source if source and "." in source else f"task.{source or 'system'}",
marker=marker,
intent=message or "Task event",
payload=metadata or context,
error=(message or "Task event") if marker == "EXPLORE" else None,
level=level or "INFO",
)
await self.event_bus.add_log(
task_id, event=event,
task_id, level=event["level"], event=event,
task_logs_list=task.logs,
)
return _add_log

10
backend/src/core/task_manager/models.py
View File

@@ -71,6 +71,16 @@ class LogEntry(BaseModel):
if not isinstance(data, dict):
return data
values = dict(data)
if "intent" not in values and values.get("message") is not None:
values["intent"] = values["message"]
if "src" not in values and values.get("source") is not None:
values["src"] = values["source"]
if "payload" not in values:
values["payload"] = values.get("metadata") or values.get("context")
if "marker" not in values:
values["marker"] = "EXPLORE" if str(values.get("level", "INFO")).upper() in {"WARNING", "ERROR"} else "REASON"
if values["marker"] == "EXPLORE" and not values.get("error"):
values["error"] = values.get("intent") or "Task event"
return values
@model_validator(mode="after")

37
backend/src/core/task_manager/persistence.py
View File

@@ -8,7 +8,7 @@
# @DATA_CONTRACT Input[Task, LogEntry] -> Model[TaskRecord, TaskLogRecord]
# @RELATION DEPENDS_ON -> [Core.Manager.TaskManager]
# @RELATION DEPENDS_ON -> [Core.Graph.TaskGraph]
# @RELATION DEPENDS_ON -> [Core.Database.TasksSessionLocal]
# @RELATION DEPENDS_ON -> [Core.Database.SessionLocal]
# @INVARIANT Database schema must match the TaskRecord model structure.
# @RATIONALE Uses SQLAlchemy ORM with separate TaskRecord and TaskLogRecord tables — task metadata
# and high-volume log entries are stored in distinct tables so log queries don't scan
@@ -26,7 +26,7 @@ from sqlalchemy.orm import Session
from ...models.mapping import Environment
from ...models.task import TaskLogRecord, TaskRecord
from ..database import TasksSessionLocal
from ..database import SessionLocal
from ..logger import belief_scope, logger
from .models import LogEntry, LogFilter, LogStats, Task, TaskLog, TaskStatus
@@ -34,11 +34,11 @@ from .models import LogEntry, LogFilter, LogStats, Task, TaskLog, TaskStatus
# #region Core.Persistence.TaskPersistenceService [C:5] [TYPE Class] [SEMANTICS persistence, service, database, sqlalchemy]
# @defgroup TaskManager Module group.
# @BRIEF Provides methods to save, load, and delete task records in tasks.db using SQLAlchemy models.
# @PRE TasksSessionLocal must provide an active SQLAlchemy session, Task inputs must expose id/plugin_id/status/params/result/logs fields, and TaskRecord plus Environment schemas must be available.
# @PRE SessionLocal must provide an active SQLAlchemy session, Task inputs must expose id/plugin_id/status/params/result/logs fields, and TaskRecord plus Environment schemas must be available.
# @POST Persist operations leave matching TaskRecord rows committed or rolled back without leaking sessions, load operations return reconstructed Task objects from stored TaskRecord rows, and delete operations remove only the addressed task rows.
# @SIDE_EFFECT Opens SQLAlchemy sessions, reads and writes task_records rows, resolves environment foreign keys against environments, commits or rolls back transactions, and emits error logs on persistence failures.
# @DATA_CONTRACT Input[Task | List[Task] | List[str] | Query(limit:int,status:Optional[TaskStatus])] -> Model[TaskRecord, Environment] -> Output[None | List[Task]]
# @RELATION DEPENDS_ON -> [Core.Database.TasksSessionLocal]
# @RELATION DEPENDS_ON -> [Core.Database.SessionLocal]
# @RELATION DEPENDS_ON -> [Models.Task.TaskRecord]
# @RELATION DEPENDS_ON -> [Core.ConfigModels.Environment]
# @RELATION DEPENDS_ON -> [Core.Manager.TaskManager]
@@ -143,7 +143,7 @@ class TaskPersistenceService:
# @POST Service is ready.
def __init__(self):
with belief_scope("TaskPersistenceService.__init__"):
# We use TasksSessionLocal from database.py
# Use the unified SessionLocal from database.py.
pass
# #endregion Core.Persistence.Init
# #region Core.Persistence.PersistTask [C:3] [TYPE Function] [C:3]
@@ -157,7 +157,7 @@ class TaskPersistenceService:
# @RELATION CALLS -> [Core.Persistence.ResolveEnvironmentId]
def persist_task(self, task: Task) -> None:
with belief_scope("TaskPersistenceService.persist_task", f"task_id={task.id}"):
session: Session = TasksSessionLocal()
session: Session = SessionLocal()
try:
record = (
session.query(TaskRecord).filter(TaskRecord.id == task.id).first()
@@ -249,7 +249,7 @@ class TaskPersistenceService:
self, limit: int = 100, status: TaskStatus | None = None
) -> list[Task]:
with belief_scope("TaskPersistenceService.load_tasks"):
session: Session = TasksSessionLocal()
session: Session = SessionLocal()
try:
query = session.query(TaskRecord)
if status:
@@ -312,7 +312,7 @@ class TaskPersistenceService:
if not task_ids:
return
with belief_scope("TaskPersistenceService.delete_tasks"):
session: Session = TasksSessionLocal()
session: Session = SessionLocal()
try:
session.query(TaskRecord).filter(TaskRecord.id.in_(task_ids)).delete(
synchronize_session=False
@@ -328,12 +328,12 @@ class TaskPersistenceService:
# #region Core.Persistence.TaskLogPersistenceService [C:5] [TYPE Class] [SEMANTICS persistence, service, database, log, sqlalchemy]
# @defgroup TaskManager Module group.
# @BRIEF Provides methods to store, query, summarize, and delete task log rows in the task_logs table.
# @PRE TasksSessionLocal must provide an active SQLAlchemy session, task_id inputs must identify task log rows, LogEntry batches must expose timestamp/level/source/message/metadata fields, and LogFilter inputs must provide pagination and filter attributes used by queries.
# @PRE SessionLocal must provide an active SQLAlchemy session, task_id inputs must identify task log rows, LogEntry batches must expose timestamp/level/source/message/metadata fields, and LogFilter inputs must provide pagination and filter attributes used by queries.
# @POST add_logs commits all provided log entries or rolls back on failure, query methods return TaskLog or LogStats views reconstructed from TaskLogRecord rows, and delete methods remove only log rows matching the supplied task identifiers.
# @SIDE_EFFECT Opens SQLAlchemy sessions, inserts, reads, aggregates, and deletes task_logs rows, serializes log metadata to JSON, commits or rolls back transactions, and emits error logs on persistence failures.
# @DATA_CONTRACT Input[task_id:str, logs:List[LogEntry], log_filter:LogFilter, task_ids:List[str]] -> Model[TaskLogRecord] -> Output[None | List[TaskLog] | LogStats | List[str]]
# @RELATION DEPENDS_ON -> [Models.Task.TaskLogRecord]
# @RELATION DEPENDS_ON -> [Core.Database.TasksSessionLocal]
# @RELATION DEPENDS_ON -> [Core.Database.SessionLocal]
# @RELATION DEPENDS_ON -> [Core.Manager.TaskManager]
# @RELATION DEPENDS_ON -> [Core.EventBus]
# @INVARIANT Log entries are batch-inserted for performance.
@@ -411,7 +411,7 @@ class TaskLogPersistenceService:
)
from sqlalchemy import insert
session: Session = TasksSessionLocal()
session: Session = SessionLocal()
try:
# Core multi-row insert via session (no TaskLogRecord() ORM objects).
# Callers should run this off the event loop (asyncio.to_thread).
@@ -439,7 +439,7 @@ class TaskLogPersistenceService:
# @RELATION DEPENDS_ON -> [Core.Models.TaskLog]
def get_logs(self, task_id: str, log_filter: LogFilter) -> list[TaskLog]:
with belief_scope("TaskLogPersistenceService.get_logs", f"task_id={task_id}"):
session: Session = TasksSessionLocal()
session: Session = SessionLocal()
try:
query = session.query(TaskLogRecord).filter(
TaskLogRecord.task_id == task_id
@@ -460,6 +460,7 @@ class TaskLogPersistenceService:
records = query.offset(log_filter.offset).limit(log_filter.limit).all()
logs = []
for record in records:
payload = record.payload if isinstance(record.payload, dict) else None
logs.append(
TaskLog(
id=record.id,
@@ -471,7 +472,7 @@ class TaskLogPersistenceService:
src=record.src,
marker=record.marker,
intent=record.intent,
payload=record.payload,
payload=payload,
error=record.error,
)
)
@@ -493,7 +494,7 @@ class TaskLogPersistenceService:
with belief_scope(
"TaskLogPersistenceService.get_log_stats", f"task_id={task_id}"
):
session: Session = TasksSessionLocal()
session: Session = SessionLocal()
try:
# Get total count
total_count = (
@@ -537,7 +538,7 @@ class TaskLogPersistenceService:
with belief_scope(
"TaskLogPersistenceService.get_sources", f"task_id={task_id}"
):
session: Session = TasksSessionLocal()
session: Session = SessionLocal()
try:
from sqlalchemy import distinct
sources = (
@@ -567,7 +568,7 @@ class TaskLogPersistenceService:
):
if not task_id:
return
session: Session = TasksSessionLocal()
session: Session = SessionLocal()
try:
query = session.query(TaskLogRecord).filter(
TaskLogRecord.task_id == task_id
@@ -615,7 +616,7 @@ class TaskLogPersistenceService:
with belief_scope(
"TaskLogPersistenceService.delete_logs_for_task", f"task_id={task_id}"
):
session: Session = TasksSessionLocal()
session: Session = SessionLocal()
try:
session.query(TaskLogRecord).filter(
TaskLogRecord.task_id == task_id
@@ -639,7 +640,7 @@ class TaskLogPersistenceService:
if not task_ids:
return
with belief_scope("TaskLogPersistenceService.delete_logs_for_tasks"):
session: Session = TasksSessionLocal()
session: Session = SessionLocal()
try:
session.query(TaskLogRecord).filter(
TaskLogRecord.task_id.in_(task_ids)

Some files were not shown because too many files have changed in this diff Show More