alembic fix
This commit is contained in:
@@ -86,7 +86,10 @@ path_separator = os
|
||||
# database URL. This is consumed by the user-maintained env.py script only.
|
||||
# other means of configuring database URLs may be customized within the env.py
|
||||
# file.
|
||||
sqlalchemy.url = sqlite:///./alembic_test.db
|
||||
# H1: Placeholder — DATABASE_URL env var overrides this in alembic/env.py.
|
||||
# If DATABASE_URL is not set, Alembic will fail with a clear error instead of
|
||||
# silently operating on a SQLite test database.
|
||||
sqlalchemy.url = postgresql+psycopg2://__MUST_SET_DATABASE_URL__:5432/__db__
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
|
||||
@@ -9,6 +9,7 @@ from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import inspect
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '7703bbc038bd'
|
||||
@@ -17,8 +18,16 @@ branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def _table_exists(table_name: str) -> bool:
|
||||
conn = op.get_bind()
|
||||
inspector = inspect(conn)
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add description column to validation_policies."""
|
||||
if not _table_exists("validation_policies"):
|
||||
return
|
||||
op.add_column('validation_policies', sa.Column('description', sa.Text(), nullable=True))
|
||||
|
||||
|
||||
|
||||
@@ -7,9 +7,9 @@ Create Date: 2026-05-26 15:27:06.159151
|
||||
"""
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import inspect
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '86c7b1d6a710'
|
||||
@@ -18,8 +18,16 @@ branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def _table_exists(table_name: str) -> bool:
|
||||
conn = op.get_bind()
|
||||
inspector = inspect(conn)
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add is_admin column to roles table, set True for existing Admin roles."""
|
||||
if not _table_exists("roles"):
|
||||
return
|
||||
op.add_column("roles", sa.Column("is_admin", sa.Boolean(), nullable=False, server_default=sa.text("false")))
|
||||
op.execute("UPDATE roles SET is_admin = true WHERE name = 'Admin'")
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ from collections.abc import Sequence
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "9f8e7d6c5b4a"
|
||||
@@ -55,8 +56,22 @@ def _is_multimodal_heuristic(model_name: str) -> bool:
|
||||
return any(marker in token for marker in multimodal_markers)
|
||||
|
||||
|
||||
def _table_exists(table_name: str) -> bool:
|
||||
"""Check if a table exists in the current database schema."""
|
||||
conn = op.get_bind()
|
||||
inspector = inspect(conn)
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add is_multimodal column and backfill using heuristic."""
|
||||
# The llm_providers table is defined in the ORM model (src/models/llm.py)
|
||||
# and created by Base.metadata.create_all() at app startup, not via migrations.
|
||||
# If the table doesn't exist yet (fresh DB), skip — the model definition
|
||||
# already includes is_multimodal, so create_all() will create it with the column.
|
||||
if not _table_exists("llm_providers"):
|
||||
return
|
||||
|
||||
# Step 1: Add column as nullable first
|
||||
op.add_column(
|
||||
"llm_providers",
|
||||
|
||||
@@ -8,9 +8,9 @@ Create Date: 2026-05-21 10:00:00.000000
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import inspect
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "a7b1c2d3e4f5"
|
||||
@@ -19,8 +19,16 @@ branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def _table_exists(table_name: str) -> bool:
|
||||
conn = op.get_bind()
|
||||
inspector = inspect(conn)
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add provider_id column to validation_policies as nullable."""
|
||||
if not _table_exists("validation_policies"):
|
||||
return
|
||||
op.add_column(
|
||||
"validation_policies",
|
||||
sa.Column("provider_id", sa.String(), nullable=True),
|
||||
|
||||
@@ -22,9 +22,9 @@ Create Date: 2026-05-31 12:00:00.000000
|
||||
"""
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import inspect
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "c9d8e7f6a5b4"
|
||||
@@ -33,6 +33,12 @@ branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def _table_exists(table_name: str) -> bool:
|
||||
conn = op.get_bind()
|
||||
inspector = inspect(conn)
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema to v2 validation models."""
|
||||
_create_validation_sources()
|
||||
@@ -115,6 +121,8 @@ def downgrade() -> None:
|
||||
|
||||
def _create_validation_sources() -> None:
|
||||
"""Create the validation_sources table (no FK dependencies)."""
|
||||
if not _table_exists("validation_policies"):
|
||||
return
|
||||
op.create_table(
|
||||
"validation_sources",
|
||||
sa.Column("id", sa.String(), nullable=False),
|
||||
@@ -150,6 +158,8 @@ def _create_validation_sources() -> None:
|
||||
|
||||
def _create_validation_runs() -> None:
|
||||
"""Create the validation_runs table."""
|
||||
if not _table_exists("validation_policies"):
|
||||
return
|
||||
op.create_table(
|
||||
"validation_runs",
|
||||
sa.Column("id", sa.String(), nullable=False),
|
||||
@@ -207,6 +217,8 @@ def _create_validation_runs() -> None:
|
||||
|
||||
def _extend_validation_policies() -> None:
|
||||
"""Add v2 columns and index to validation_policies."""
|
||||
if not _table_exists("validation_policies"):
|
||||
return
|
||||
op.add_column(
|
||||
"validation_policies",
|
||||
sa.Column("prompt_template", sa.Text(), nullable=True),
|
||||
@@ -269,6 +281,8 @@ def _extend_validation_policies() -> None:
|
||||
|
||||
def _extend_validation_results() -> None:
|
||||
"""Add v2 columns, FKs, and indexes to llm_validation_results."""
|
||||
if not _table_exists("llm_validation_results"):
|
||||
return
|
||||
op.add_column(
|
||||
"llm_validation_results",
|
||||
sa.Column(
|
||||
@@ -335,6 +349,8 @@ def _backfill_validation_runs() -> None:
|
||||
|
||||
Safe no-op when no such rows exist (e.g. fresh database).
|
||||
"""
|
||||
if not _table_exists("llm_validation_results"):
|
||||
return
|
||||
conn = op.get_bind()
|
||||
|
||||
# Check for existing records that need backfill
|
||||
|
||||
@@ -8,9 +8,9 @@ Create Date: 2026-05-21 12:00:00.000000
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import inspect
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "e5f4d3c2b1a"
|
||||
@@ -19,8 +19,16 @@ branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def _table_exists(table_name: str) -> bool:
|
||||
conn = op.get_bind()
|
||||
inspector = inspect(conn)
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add policy_id column to llm_validation_results as nullable indexed."""
|
||||
if not _table_exists("llm_validation_results"):
|
||||
return
|
||||
op.add_column(
|
||||
"llm_validation_results",
|
||||
sa.Column("policy_id", sa.String(), nullable=True, index=True),
|
||||
|
||||
@@ -9,6 +9,7 @@ Create Date: 2026-05-31 22:19:21.922928
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
@@ -19,6 +20,14 @@ depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# The llm_providers table is defined in the ORM model and created by
|
||||
# Base.metadata.create_all() at app startup, not via migrations.
|
||||
# If the table doesn't exist yet (fresh DB), skip — the model definition
|
||||
# already includes max_images, so create_all() will create it with the column.
|
||||
conn = op.get_bind()
|
||||
inspector = inspect(conn)
|
||||
if "llm_providers" not in inspector.get_table_names():
|
||||
return
|
||||
op.add_column('llm_providers', sa.Column('max_images', sa.Integer(), nullable=True))
|
||||
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ from pathlib import Path
|
||||
# project_root is used for static files mounting
|
||||
project_root = Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
from alembic.config import Config as AlembicConfig
|
||||
from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect, status
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
@@ -27,8 +26,6 @@ from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
from typing import Any
|
||||
|
||||
from alembic import command as alembic_command
|
||||
|
||||
from .api import auth
|
||||
from .api.routes import (
|
||||
admin,
|
||||
@@ -67,10 +64,14 @@ from .models.auth import Role, User
|
||||
|
||||
# #region lifespan [C:3] [TYPE Function]
|
||||
# @BRIEF Async context manager for FastAPI startup/shutdown lifecycle.
|
||||
# @RELATION CALLS -> [run_alembic_migrations]
|
||||
# @RELATION CALLS -> [init_db]
|
||||
# @RELATION CALLS -> [AppDependencies]
|
||||
# @POST On startup: schema up-to-date, admin exists, scheduler started. On shutdown: scheduler stopped.
|
||||
# @POST On startup: admin exists, scheduler started. On shutdown: scheduler stopped.
|
||||
# @RATIONALE Alembic migrations removed from lifespan — they now run exclusively
|
||||
# in docker/backend.entrypoint.sh (wait_for_db → alembic upgrade head).
|
||||
# Running migrations in both places added ~5s startup overhead and masked
|
||||
# partial failures. init_db() remains as a safety net for tables without
|
||||
# dedicated Alembic migrations (e.g., newly added models during development).
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# Startup
|
||||
@@ -78,9 +79,7 @@ async def lifespan(app: FastAPI):
|
||||
with belief_scope("startup_event"):
|
||||
logger.info("🔐 Ensuring encryption key...")
|
||||
ensure_encryption_key()
|
||||
logger.info("📦 Running Alembic migrations...")
|
||||
run_alembic_migrations()
|
||||
logger.info("🗄️ Initializing database tables...")
|
||||
logger.info("🗄️ Initializing database tables (safety net)...")
|
||||
init_db()
|
||||
logger.info("👤 Bootstrapping admin user...")
|
||||
ensure_initial_admin_user()
|
||||
@@ -109,7 +108,6 @@ async def lifespan(app: FastAPI):
|
||||
scheduler = get_scheduler_service()
|
||||
scheduler.start()
|
||||
logger.info("✅ Application startup complete")
|
||||
logger.info("✅ Application startup complete")
|
||||
yield
|
||||
# Shutdown
|
||||
scheduler.stop()
|
||||
@@ -185,13 +183,16 @@ def ensure_initial_admin_user() -> None:
|
||||
# #endregion ensure_initial_admin_user
|
||||
# #region run_alembic_migrations [C:2] [TYPE Function]
|
||||
# @BRIEF Applies all pending Alembic migrations against DATABASE_URL.
|
||||
# DEPRECATED: Migrations now run exclusively in docker/backend.entrypoint.sh.
|
||||
# Kept for local development (manual invocation).
|
||||
# @POST All Alembic migrations up to 'head' are applied.
|
||||
# @SIDE_EFFECT Executes ALTER TABLE / CREATE TABLE via Alembic chain.
|
||||
# @RATIONALE Single source of truth for schema changes. All column additions,
|
||||
# drops, renames, and data migrations go through Alembic.
|
||||
# init_db() runs after to create any tables that don't have
|
||||
# a standalone Alembic migration yet.
|
||||
# @DEPRECATED Migrations moved to entrypoint.sh — this function is kept for
|
||||
# local development only. Do NOT call from lifespan or production code.
|
||||
def run_alembic_migrations() -> None:
|
||||
from alembic.config import Config as AlembicConfig
|
||||
from alembic import command as alembic_command
|
||||
|
||||
with belief_scope("run_alembic_migrations"):
|
||||
try:
|
||||
alembic_cfg = AlembicConfig("alembic.ini")
|
||||
|
||||
@@ -760,15 +760,17 @@ def _ensure_dictionary_entries_columns(bind_engine):
|
||||
|
||||
|
||||
# #region init_db [C:3] [TYPE Function]
|
||||
# @BRIEF Creates any missing tables via create_all().
|
||||
# @BRIEF Creates any missing tables via create_all() — safety net for development.
|
||||
# @PRE engine, tasks_engine and auth_engine are initialized.
|
||||
# Alembic migrations (run_alembic_migrations) have already been applied.
|
||||
# In Docker: Alembic migrations already applied via entrypoint.sh.
|
||||
# @POST All tables from SQLAlchemy models exist in all databases.
|
||||
# @SIDE_EFFECT Creates physical database tables if they don't exist.
|
||||
# @RATIONALE Schema migrations are handled by Alembic (run_alembic_migrations runs
|
||||
# before init_db in startup_event). create_all() only creates NEW tables
|
||||
# that don't exist yet — it does NOT alter existing ones. All column
|
||||
# additions must go through Alembic migrations.
|
||||
# @RATIONALE create_all() is idempotent — it only creates tables that don't exist,
|
||||
# never alters existing ones. In Docker, Alembic runs first in entrypoint.sh
|
||||
# (wait_for_db → alembic upgrade head), so init_db() is a no-op for production.
|
||||
# For local development (run.sh without Docker), this provides a safety net.
|
||||
# @REJECTED Removing init_db() entirely was rejected — local development without
|
||||
# Docker would require manual Alembic setup, increasing friction for developers.
|
||||
# @REJECTED _ensure_*() inline additive migrations removed — they were duplicating
|
||||
# Alembic logic, were not versioned, and created hidden schema drift.
|
||||
# All schema changes must now go through Alembic.
|
||||
|
||||
114
backend/src/scripts/check_migration_chain.py
Normal file
114
backend/src/scripts/check_migration_chain.py
Normal file
@@ -0,0 +1,114 @@
|
||||
# #region check_migration_chain [C:3] [TYPE Module] [SEMANTICS alembic,migration,validation,ci]
|
||||
# @BRIEF Validates Alembic migration chain integrity — checks for multiple heads,
|
||||
# broken chains, and missing down_revision references.
|
||||
# @RATIONALE Merge migrations (e.g., f0e9d8c7b6a5) are fragile — a new migration
|
||||
# with wrong down_revision can create multiple heads and break `alembic upgrade head`.
|
||||
# This script catches the issue before deployment.
|
||||
# @REJECTED Relying on `alembic upgrade head` to fail at deploy time was rejected —
|
||||
# it's better to catch migration chain issues in CI or pre-commit.
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Ensure backend/src is importable
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
|
||||
# #region check_migration_heads [C:2] [TYPE Function]
|
||||
# @BRIEF Checks that there is exactly one migration head.
|
||||
# @POST Prints error and exits 1 if multiple heads found.
|
||||
def check_migration_heads() -> None:
|
||||
from alembic.config import Config as AlembicConfig
|
||||
from alembic.script import ScriptDirectory
|
||||
|
||||
alembic_cfg = AlembicConfig("alembic.ini")
|
||||
alembic_cfg.set_main_option("script_location", "alembic")
|
||||
|
||||
script_dir = ScriptDirectory.from_config(alembic_cfg)
|
||||
heads = list(script_dir.get_heads())
|
||||
|
||||
if len(heads) == 0:
|
||||
print("[migration-check] ERROR: No migration heads found — chain is empty")
|
||||
sys.exit(1)
|
||||
|
||||
if len(heads) > 1:
|
||||
print(f"[migration-check] ERROR: Multiple migration heads detected ({len(heads)}):")
|
||||
for h in heads:
|
||||
rev = script_dir.get_revision(h)
|
||||
print(f" - {h}: {rev.doc}")
|
||||
print("[migration-check] Fix: ensure new migrations reference the correct parent")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"[migration-check] OK: Single migration head — {heads[0]}")
|
||||
|
||||
|
||||
# #endregion check_migration_heads
|
||||
|
||||
# #region check_migration_chain_integrity [C:2] [TYPE Function]
|
||||
# @BRIEF Walks the migration chain from head to root, checking all down_revision links exist.
|
||||
# @POST Prints error and exits 1 if a broken link is found.
|
||||
def check_migration_chain_integrity() -> None:
|
||||
from alembic.config import Config as AlembicConfig
|
||||
from alembic.script import ScriptDirectory
|
||||
|
||||
alembic_cfg = AlembicConfig("alembic.ini")
|
||||
alembic_cfg.set_main_option("script_location", "alembic")
|
||||
|
||||
script_dir = ScriptDirectory.from_config(alembic_cfg)
|
||||
heads = list(script_dir.get_heads())
|
||||
|
||||
if not heads:
|
||||
return # already checked above
|
||||
|
||||
visited = set()
|
||||
current_revs = heads
|
||||
|
||||
while current_revs:
|
||||
next_revs = []
|
||||
for rev_id in current_revs:
|
||||
if rev_id in visited:
|
||||
continue
|
||||
visited.add(rev_id)
|
||||
rev = script_dir.get_revision(rev_id)
|
||||
|
||||
down = rev.down_revision
|
||||
if down is None:
|
||||
continue # root migration
|
||||
|
||||
if isinstance(down, str):
|
||||
try:
|
||||
script_dir.get_revision(down)
|
||||
except Exception:
|
||||
print(f"[migration-check] ERROR: Broken link — {rev_id} -> {down} (not found)")
|
||||
sys.exit(1)
|
||||
next_revs.append(down)
|
||||
elif isinstance(down, tuple):
|
||||
# Merge migration — check all parents exist
|
||||
for parent in down:
|
||||
try:
|
||||
script_dir.get_revision(parent)
|
||||
except Exception:
|
||||
print(f"[migration-check] ERROR: Broken merge link — {rev_id} -> {parent} (not found)")
|
||||
sys.exit(1)
|
||||
next_revs.append(parent)
|
||||
|
||||
current_revs = next_revs
|
||||
|
||||
print(f"[migration-check] OK: Chain integrity verified — {len(visited)} migrations traversed")
|
||||
|
||||
|
||||
# #endregion check_migration_chain_integrity
|
||||
|
||||
# #region check_migration_main [C:2] [TYPE Function]
|
||||
# @BRIEF Entry point — runs all migration chain checks.
|
||||
def main() -> None:
|
||||
print("[migration-check] === Alembic Migration Chain Validation ===")
|
||||
check_migration_heads()
|
||||
check_migration_chain_integrity()
|
||||
print("[migration-check] === All checks passed ===")
|
||||
|
||||
|
||||
# #endregion check_migration_main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,16 +1,17 @@
|
||||
# #region TestAlembicMigrations [C:3] [TYPE Module] [SEMANTICS tests, alembic, migration, schema, idempotent]
|
||||
# @BRIEF Verifies Alembic migration chain: fresh upgrade and legacy stamp.
|
||||
# @BRIEF Verifies Alembic migration chain: fresh upgrade and legacy upgrade.
|
||||
# @RELATION BINDS_TO -> [ed310b33f02c]
|
||||
# @INVARIANT Running `alembic upgrade head` twice is idempotent (no-op on second run) on PostgreSQL.
|
||||
# @INVARIANT Legacy database with existing tables can be stamped after adding boot-critical columns.
|
||||
# @INVARIANT Legacy database with existing tables can be upgraded via `alembic upgrade head`.
|
||||
# @PRE Alembic is installed, migration files exist in alembic/versions/.
|
||||
# @POST Schema matches expected state after upgrade.
|
||||
# @TEST_INVARIANT: alembic_fresh_upgrade -> VERIFIED_BY: [test_fresh_upgrade_creates_tables]
|
||||
# @TEST_INVARIANT: alembic_stamp_legacy -> VERIFIED_BY: [test_legacy_database_stamp]
|
||||
# @TEST_INVARIANT: alembic_upgrade_legacy -> VERIFIED_BY: [test_legacy_database_upgrade]
|
||||
# @REJECTED Full migration chain idempotency on SQLite — initial migration uses
|
||||
# op.create_index() which conflicts with SQLite auto-created indexes
|
||||
# for UNIQUE constraints. Migrations are designed for PostgreSQL.
|
||||
# Legacy databases use stamp + selective raw SQL in entrypoint instead.
|
||||
# @REJECTED Legacy stamp + raw SQL rejected — entrypoint now runs `alembic upgrade head`
|
||||
# for legacy databases, ensuring all missing tables/columns are created.
|
||||
|
||||
import os
|
||||
import sys
|
||||
@@ -65,51 +66,43 @@ def test_fresh_upgrade_creates_tables() -> None:
|
||||
# #endregion test_fresh_upgrade_creates_tables
|
||||
|
||||
|
||||
# #region test_legacy_database_stamp [C:2] [TYPE Function]
|
||||
# @BRIEF Pre-populate tables via create_all(), then verify entrypoint-style stamp works.
|
||||
# @POST alembic_version created, is_admin column present after stamp + raw SQL.
|
||||
# #region test_legacy_database_upgrade [C:2] [TYPE Function]
|
||||
# @BRIEF Simulate legacy DB (tables exist, no alembic_version), verify upgrade head works.
|
||||
# @POST alembic_version created, all migration columns present after upgrade.
|
||||
# @TEST_INVARIANT: alembic_upgrade_legacy -> VERIFIED_BY: [test_legacy_database_upgrade]
|
||||
@_REQUIRES_PG
|
||||
def test_legacy_database_stamp() -> None:
|
||||
"""Alembic stamp works on a database that already has tables (PostgreSQL only)."""
|
||||
def test_legacy_database_upgrade() -> None:
|
||||
"""Alembic upgrade head works on a database that already has tables but no alembic_version."""
|
||||
from sqlalchemy import create_engine, inspect
|
||||
from sqlalchemy import text as sa_text
|
||||
from src.models.mapping import Base
|
||||
from src.models import auth as _auth # noqa: F401
|
||||
from src.models import task as _task # noqa: F401
|
||||
from alembic.config import Config as AlembicConfig
|
||||
from alembic import command as alembic_command
|
||||
|
||||
engine = create_engine(os.environ["DATABASE_URL"])
|
||||
# Drop alembic_version if exists to simulate legacy state
|
||||
with engine.begin() as conn:
|
||||
conn.execute(sa_text("DROP TABLE IF EXISTS alembic_version CASCADE"))
|
||||
engine.dispose()
|
||||
# Tables already exist from previous runs (create_all)
|
||||
# Add is_admin column (simulating entrypoint raw SQL)
|
||||
engine = create_engine(os.environ["DATABASE_URL"])
|
||||
with engine.begin() as conn:
|
||||
inspector = inspect(conn)
|
||||
roles_cols = {c["name"] for c in inspector.get_columns("roles")}
|
||||
if "is_admin" not in roles_cols:
|
||||
conn.execute(
|
||||
sa_text("ALTER TABLE roles ADD COLUMN is_admin BOOLEAN NOT NULL DEFAULT FALSE")
|
||||
)
|
||||
conn.execute(sa_text("UPDATE roles SET is_admin = TRUE WHERE name = 'Admin'"))
|
||||
engine.dispose()
|
||||
# Stamp head (entrypoint behavior for legacy DB)
|
||||
backend_dir = Path(__file__).parent.parent
|
||||
alembic_cfg = AlembicConfig(str(backend_dir / "alembic.ini"))
|
||||
alembic_cfg.set_main_option("script_location", str(backend_dir / "alembic"))
|
||||
alembic_command.stamp(alembic_cfg, "head")
|
||||
|
||||
# Tables already exist from previous runs (create_all) — this is the legacy state.
|
||||
# Run upgrade head (new entrypoint behavior for legacy DB)
|
||||
_run_alembic_upgrade()
|
||||
|
||||
# Verify
|
||||
engine = create_engine(os.environ["DATABASE_URL"])
|
||||
inspector = inspect(engine)
|
||||
tables = set(inspector.get_table_names())
|
||||
assert "alembic_version" in tables, "alembic_version table not created by stamp"
|
||||
assert "alembic_version" in tables, "alembic_version table not created by upgrade"
|
||||
roles_cols = {c["name"] for c in inspector.get_columns("roles")}
|
||||
assert "is_admin" in roles_cols
|
||||
assert "is_admin" in roles_cols, "roles.is_admin column missing after legacy upgrade"
|
||||
|
||||
# Verify version_num matches head
|
||||
result = engine.connect().execute(sa_text("SELECT version_num FROM alembic_version"))
|
||||
version = result.scalar_one()
|
||||
assert version is not None and len(version) > 0
|
||||
engine.dispose()
|
||||
# #endregion test_legacy_database_stamp
|
||||
# #endregion test_legacy_database_upgrade
|
||||
|
||||
|
||||
# #region _run_alembic_upgrade [C:1] [TYPE Function]
|
||||
|
||||
@@ -15,6 +15,10 @@
|
||||
# @RELATION DEPENDS_ON -> [docker/backend.entrypoint.sh]
|
||||
# @DATA_CONTRACT Input: .env.enterprise-clean + ./certs/*.crt -> Output: запущенные контейнеры
|
||||
# @INVARIANT Внешний Postgres НЕ требует healthcheck внутри compose — ответственность оператора.
|
||||
# Backend entrypoint использует wait_for_db() (retry до 30 раз) для проверки доступности БД
|
||||
# перед запуском миграций. Если БД недоступна — контейнер exit(1) после таймаута.
|
||||
# @RATIONALE Нет depends_on для backend → внешний PostgreSQL: оператор гарантирует доступность.
|
||||
# wait_for_db() в entrypoint.sh обеспечивает graceful retry вместо мгновенного crash.
|
||||
# @INVARIANT Если CERTS_PATH пуст или не содержит .crt файлов, entrypoint не падает.
|
||||
# @INVARIANT nginx слушает порт 80 всегда. Порт 443 включается только если есть server.crt + server.key.
|
||||
# @RATIONALE Отказ от встроенного postgres-container: корпоративный стандарт — внешний PostgreSQL
|
||||
|
||||
@@ -2,14 +2,17 @@
|
||||
set -euo pipefail
|
||||
|
||||
# #region docker.backend.entrypoint [C:4] [TYPE Module] [SEMANTICS docker,entrypoint,enterprise,certs]
|
||||
# @BRIEF Container entrypoint — установка корп. сертификатов, bootstrap admin,
|
||||
# затем запуск backend (exec "$@").
|
||||
# @BRIEF Container entrypoint — установка корп. сертификатов, wait-for-DB,
|
||||
# Alembic миграции, bootstrap admin, затем запуск backend (exec "$@").
|
||||
# @PRE Python runtime и backend source доступны в /app/backend.
|
||||
# Переменная CERTS_PATH указывает на директорию с .crt файлами (или пусто).
|
||||
# DATABASE_URL установлен и указывает на доступный PostgreSQL.
|
||||
# @POST Сертификаты установлены в системное хранилище и NSS базу Chromium.
|
||||
# БД подтверждена как доступная. Alembic миграции применены.
|
||||
# Admin создан (если INITIAL_ADMIN_CREATE=true). Backend запущен через exec "$@".
|
||||
# @SIDE_EFFECT Модифицирует /usr/local/share/ca-certificates/ и вызывает update-ca-certificates.
|
||||
# Импортирует CA в NSS DB (~/.pki/nssdb) для Chromium.
|
||||
# Выполняет Alembic upgrade head.
|
||||
# @LAYER Infrastructure
|
||||
# @RELATION DEPENDS_ON -> [backend/src/scripts/create_admin.py]
|
||||
# @RELATION DEPENDS_ON -> [backend/src/scripts/init_auth_db.py]
|
||||
@@ -18,6 +21,9 @@ set -euo pipefail
|
||||
# (create_admin.py идемпотентен).
|
||||
# @INVARIANT Повторный запуск install_llm_ca_certs не дублирует сертификаты в bundle.
|
||||
# @INVARIANT Повторный запуск install_ca_to_nss не дублирует сертификаты в NSS DB.
|
||||
# @INVARIANT wait_for_db exit(1) если БД недоступна после DB_WAIT_ATTEMPTS попыток —
|
||||
# контейнер не продолжит запуск с недоступной БД.
|
||||
# @INVARIANT Alembic миграции выполняются ТОЛЬКО в entrypoint, НЕ в app.py lifespan.
|
||||
# #endregion docker.backend.entrypoint
|
||||
|
||||
# ======================================================================
|
||||
@@ -420,6 +426,73 @@ install_ca_to_nss() {
|
||||
}
|
||||
# #endregion docker.backend.entrypoint.install_ca_to_nss
|
||||
|
||||
# ======================================================================
|
||||
# wait_for_db — проверка доступности БД с retry
|
||||
# ======================================================================
|
||||
# #region docker.backend.entrypoint.wait_for_db [C:3] [TYPE Function]
|
||||
# @BRIEF Проверяет доступность PostgreSQL с retry-логикой перед запуском миграций.
|
||||
# @PRE DATABASE_URL установлен и указывает на валидный PostgreSQL.
|
||||
# @POST Соединение с БД подтверждено (SELECT 1) или скрипт exit(1) после таймаута.
|
||||
# @RATIONALE Без wait-for-db enterprise-деплой с внешним PostgreSQL падает
|
||||
# мгновенно — entrypoint интерпретирует connection refused как "пустая БД".
|
||||
# @REJECTED Использование pg_isready напрямую отвергнуто — клиент psql может
|
||||
# отсутствовать в python:3.11-slim образе. SQLAlchemy-проверка универсальна.
|
||||
wait_for_db() {
|
||||
local max_attempts="${DB_WAIT_ATTEMPTS:-30}"
|
||||
local delay="${DB_WAIT_DELAY:-2}"
|
||||
local attempt=0
|
||||
|
||||
echo "[entrypoint] Waiting for database (max ${max_attempts} attempts, ${delay}s delay)..."
|
||||
while [ $attempt -lt $max_attempts ]; do
|
||||
attempt=$((attempt + 1))
|
||||
# Capture stderr to detect permanent errors (KeyError, ArgumentError)
|
||||
local _err_output
|
||||
_err_output="$(python3 -c "
|
||||
import os, sys
|
||||
from sqlalchemy import create_engine, text
|
||||
try:
|
||||
url = os.environ['DATABASE_URL']
|
||||
except KeyError:
|
||||
print('FATAL: DATABASE_URL environment variable is not set', file=sys.stderr)
|
||||
sys.exit(99)
|
||||
try:
|
||||
e = create_engine(url)
|
||||
with e.connect() as c:
|
||||
c.execute(text('SELECT 1'))
|
||||
except Exception as exc:
|
||||
# Permanent errors — no point retrying
|
||||
err_str = str(exc).lower()
|
||||
if 'database_url' in err_str or 'could not parse' in err_str or 'invalid' in err_str:
|
||||
print(f'FATAL: {exc}', file=sys.stderr)
|
||||
sys.exit(99)
|
||||
sys.exit(1)
|
||||
" 2>&1)"
|
||||
local _rc=$?
|
||||
|
||||
if [ $_rc -eq 0 ]; then
|
||||
echo "[entrypoint] ✅ Database is reachable"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Permanent error — exit immediately, don't waste retries
|
||||
if [ $_rc -eq 99 ]; then
|
||||
echo "[entrypoint] ❌ $_err_output" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "[entrypoint] Attempt ${attempt}/${max_attempts} — retrying in ${delay}s..."
|
||||
sleep "$delay"
|
||||
done
|
||||
|
||||
# P0: Mask credentials — never print full DATABASE_URL to logs
|
||||
local _masked_url
|
||||
_masked_url="$(echo "$DATABASE_URL" | sed -E 's|://([^:]+):([^@]+)@|://\1:***@|')"
|
||||
echo "[entrypoint] ❌ Database not reachable after ${max_attempts} attempts" >&2
|
||||
echo "[entrypoint] DATABASE_URL=${_masked_url}" >&2
|
||||
return 1
|
||||
}
|
||||
# #endregion docker.backend.entrypoint.wait_for_db
|
||||
|
||||
# ======================================================================
|
||||
# MAIN
|
||||
# ======================================================================
|
||||
@@ -428,8 +501,10 @@ install_certificates
|
||||
install_llm_ca_certs
|
||||
install_ca_to_nss
|
||||
|
||||
# Check DB state: empty, legacy (tables exist but no alembic), or managed
|
||||
# || prevents set -e from killing the script on non-zero exit from detection script
|
||||
# ── C1: Wait for database before any migration logic ──
|
||||
wait_for_db
|
||||
|
||||
# ── C2: DB state detection (now safe — DB is confirmed reachable) ──
|
||||
_db_state=0
|
||||
python3 -c "
|
||||
import os
|
||||
@@ -452,25 +527,11 @@ elif [ $_db_state -eq 1 ]; then
|
||||
alembic upgrade head 2>&1 | sed 's/^/[entrypoint] /'
|
||||
echo "[entrypoint] ✅ Alembic initial migration applied"
|
||||
else
|
||||
echo "[entrypoint] Legacy database — adding critical migration columns..."
|
||||
# Add boot-critical columns that Alembic migrations would add
|
||||
python3 -c "
|
||||
import os
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
e = create_engine(os.environ['DATABASE_URL'])
|
||||
with e.begin() as conn:
|
||||
inspector = inspect(conn)
|
||||
roles_cols = {c['name'] for c in inspector.get_columns('roles')}
|
||||
if 'is_admin' not in roles_cols:
|
||||
conn.execute(text('ALTER TABLE roles ADD COLUMN is_admin BOOLEAN NOT NULL DEFAULT FALSE'))
|
||||
conn.execute(text(\"UPDATE roles SET is_admin = TRUE WHERE name = 'Admin'\"))
|
||||
print(' roles.is_admin column added')
|
||||
else:
|
||||
print(' roles.is_admin already exists')
|
||||
"
|
||||
echo "[entrypoint] Stamping Alembic head (tables already exist)..."
|
||||
alembic stamp head 2>&1 | sed 's/^/[entrypoint] /'
|
||||
echo "[entrypoint] ✅ Alembic stamped head, schema ready"
|
||||
echo "[entrypoint] Legacy database — running Alembic upgrade (not stamp)..."
|
||||
# H2: Запускаем полноценный upgrade вместо stamp head — legacy-БД может
|
||||
# отсутствовать таблицы, добавленные миграциями после написания entrypoint.
|
||||
alembic upgrade head 2>&1 | sed 's/^/[entrypoint] /'
|
||||
echo "[entrypoint] ✅ Alembic upgrade complete for legacy database"
|
||||
fi
|
||||
|
||||
bootstrap_admin
|
||||
|
||||
@@ -234,19 +234,16 @@ stage1_enterprise_clean() {
|
||||
docker build -f docker/frontend.Dockerfile -t "ss-tools-frontend:${RELEASE_TAG}" .
|
||||
fi
|
||||
|
||||
# ── 1.3. START POSTGRESQL ────────────────────────────────
|
||||
# ── 1.3. START POSTGRESQL (on default bridge first) ─────
|
||||
log_info "Step 1.3: Starting new PostgreSQL container..."
|
||||
docker rm -f ss-tools-e2e-pg 2>/dev/null || true
|
||||
|
||||
# Create the network that docker compose will use (ss-tools-e2e-ec_default)
|
||||
# so PostgreSQL is on the same network and resolvable by container name.
|
||||
if ! docker network inspect ss-tools-e2e-ec_default >/dev/null 2>&1; then
|
||||
docker network create ss-tools-e2e-ec_default
|
||||
fi
|
||||
# Remove any stale compose network so docker compose creates it fresh
|
||||
docker network rm ss-tools-e2e-ec_default 2>/dev/null || true
|
||||
|
||||
# Start PG on default bridge initially — compose will create its own network
|
||||
docker run -d \
|
||||
--name ss-tools-e2e-pg \
|
||||
--network ss-tools-e2e-ec_default \
|
||||
-e POSTGRES_DB=ss_tools \
|
||||
-e POSTGRES_USER=postgres \
|
||||
-e POSTGRES_PASSWORD=postgres \
|
||||
@@ -267,7 +264,6 @@ stage1_enterprise_clean() {
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Get PostgreSQL container IP on the enterprise network
|
||||
EC_PG_HOST="ss-tools-e2e-pg"
|
||||
EC_PG_PORT="5432"
|
||||
|
||||
@@ -308,11 +304,20 @@ EOF
|
||||
export FRONTEND_TAG="${RELEASE_TAG}"
|
||||
fi
|
||||
|
||||
# Start compose FIRST so it creates the network with proper compose labels.
|
||||
# The backend will fail to connect to PG on first start (PG not on compose network
|
||||
# yet), but Docker restart policy (unless-stopped) will retry.
|
||||
docker compose -p ss-tools-e2e-ec \
|
||||
-f "$EC_COMPOSE_FILE" \
|
||||
--env-file "/tmp/.env.e2e-enterprise-clean" \
|
||||
up -d
|
||||
|
||||
# Connect PG to the compose-created network so backend can resolve ss-tools-e2e-pg
|
||||
docker network connect ss-tools-e2e-ec_default ss-tools-e2e-pg
|
||||
|
||||
# Restart backend so entrypoint reruns with PG accessible
|
||||
docker compose -p ss-tools-e2e-ec restart backend
|
||||
|
||||
# ── 1.5. WAIT FOR BACKEND + FRONTEND ────────────────────
|
||||
log_info "Step 1.5: Waiting for services..."
|
||||
|
||||
|
||||
Reference in New Issue
Block a user