diff --git a/backend/alembic.ini b/backend/alembic.ini index 4b8cceda..97d5fd29 100644 --- a/backend/alembic.ini +++ b/backend/alembic.ini @@ -86,7 +86,10 @@ path_separator = os # database URL. This is consumed by the user-maintained env.py script only. # other means of configuring database URLs may be customized within the env.py # file. -sqlalchemy.url = sqlite:///./alembic_test.db +# H1: Placeholder — DATABASE_URL env var overrides this in alembic/env.py. +# If DATABASE_URL is not set, Alembic will fail with a clear error instead of +# silently operating on a SQLite test database. +sqlalchemy.url = postgresql+psycopg2://__MUST_SET_DATABASE_URL__:5432/__db__ [post_write_hooks] diff --git a/backend/alembic/versions/7703bbc038bd_add_description_to_validation_policy.py b/backend/alembic/versions/7703bbc038bd_add_description_to_validation_policy.py index c368002a..e4cfc885 100644 --- a/backend/alembic/versions/7703bbc038bd_add_description_to_validation_policy.py +++ b/backend/alembic/versions/7703bbc038bd_add_description_to_validation_policy.py @@ -9,6 +9,7 @@ from typing import Sequence, Union from alembic import op import sqlalchemy as sa +from sqlalchemy import inspect # revision identifiers, used by Alembic. revision: str = '7703bbc038bd' @@ -17,8 +18,16 @@ branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None +def _table_exists(table_name: str) -> bool: + conn = op.get_bind() + inspector = inspect(conn) + return table_name in inspector.get_table_names() + + def upgrade() -> None: """Add description column to validation_policies.""" + if not _table_exists("validation_policies"): + return op.add_column('validation_policies', sa.Column('description', sa.Text(), nullable=True)) diff --git a/backend/alembic/versions/86c7b1d6a710_add_is_admin_column_to_roles_table.py b/backend/alembic/versions/86c7b1d6a710_add_is_admin_column_to_roles_table.py index 4415a2c2..026c5785 100644 --- a/backend/alembic/versions/86c7b1d6a710_add_is_admin_column_to_roles_table.py +++ b/backend/alembic/versions/86c7b1d6a710_add_is_admin_column_to_roles_table.py @@ -7,9 +7,9 @@ Create Date: 2026-05-26 15:27:06.159151 """ from collections.abc import Sequence -import sqlalchemy as sa - from alembic import op +import sqlalchemy as sa +from sqlalchemy import inspect # revision identifiers, used by Alembic. revision: str = '86c7b1d6a710' @@ -18,8 +18,16 @@ branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None +def _table_exists(table_name: str) -> bool: + conn = op.get_bind() + inspector = inspect(conn) + return table_name in inspector.get_table_names() + + def upgrade() -> None: """Add is_admin column to roles table, set True for existing Admin roles.""" + if not _table_exists("roles"): + return op.add_column("roles", sa.Column("is_admin", sa.Boolean(), nullable=False, server_default=sa.text("false"))) op.execute("UPDATE roles SET is_admin = true WHERE name = 'Admin'") diff --git a/backend/alembic/versions/9f8e7d6c5b4a_add_is_multimodal_to_llm_providers.py b/backend/alembic/versions/9f8e7d6c5b4a_add_is_multimodal_to_llm_providers.py index 17ffcc4a..fb2bd7dc 100644 --- a/backend/alembic/versions/9f8e7d6c5b4a_add_is_multimodal_to_llm_providers.py +++ b/backend/alembic/versions/9f8e7d6c5b4a_add_is_multimodal_to_llm_providers.py @@ -11,6 +11,7 @@ from collections.abc import Sequence import sqlalchemy as sa from alembic import op +from sqlalchemy import inspect # revision identifiers, used by Alembic. revision: str = "9f8e7d6c5b4a" @@ -55,8 +56,22 @@ def _is_multimodal_heuristic(model_name: str) -> bool: return any(marker in token for marker in multimodal_markers) +def _table_exists(table_name: str) -> bool: + """Check if a table exists in the current database schema.""" + conn = op.get_bind() + inspector = inspect(conn) + return table_name in inspector.get_table_names() + + def upgrade() -> None: """Add is_multimodal column and backfill using heuristic.""" + # The llm_providers table is defined in the ORM model (src/models/llm.py) + # and created by Base.metadata.create_all() at app startup, not via migrations. + # If the table doesn't exist yet (fresh DB), skip — the model definition + # already includes is_multimodal, so create_all() will create it with the column. + if not _table_exists("llm_providers"): + return + # Step 1: Add column as nullable first op.add_column( "llm_providers", diff --git a/backend/alembic/versions/a7b1c2d3e4f5_add_provider_id_to_validation_policies.py b/backend/alembic/versions/a7b1c2d3e4f5_add_provider_id_to_validation_policies.py index 109f693f..70ac4227 100644 --- a/backend/alembic/versions/a7b1c2d3e4f5_add_provider_id_to_validation_policies.py +++ b/backend/alembic/versions/a7b1c2d3e4f5_add_provider_id_to_validation_policies.py @@ -8,9 +8,9 @@ Create Date: 2026-05-21 10:00:00.000000 from collections.abc import Sequence -import sqlalchemy as sa - from alembic import op +import sqlalchemy as sa +from sqlalchemy import inspect # revision identifiers, used by Alembic. revision: str = "a7b1c2d3e4f5" @@ -19,8 +19,16 @@ branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None +def _table_exists(table_name: str) -> bool: + conn = op.get_bind() + inspector = inspect(conn) + return table_name in inspector.get_table_names() + + def upgrade() -> None: """Add provider_id column to validation_policies as nullable.""" + if not _table_exists("validation_policies"): + return op.add_column( "validation_policies", sa.Column("provider_id", sa.String(), nullable=True), diff --git a/backend/alembic/versions/c9d8e7f6a5b4_add_v2_validation_models.py b/backend/alembic/versions/c9d8e7f6a5b4_add_v2_validation_models.py index 8bf358ce..f4d5069f 100644 --- a/backend/alembic/versions/c9d8e7f6a5b4_add_v2_validation_models.py +++ b/backend/alembic/versions/c9d8e7f6a5b4_add_v2_validation_models.py @@ -22,9 +22,9 @@ Create Date: 2026-05-31 12:00:00.000000 """ from collections.abc import Sequence -import sqlalchemy as sa - from alembic import op +import sqlalchemy as sa +from sqlalchemy import inspect # revision identifiers, used by Alembic. revision: str = "c9d8e7f6a5b4" @@ -33,6 +33,12 @@ branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None +def _table_exists(table_name: str) -> bool: + conn = op.get_bind() + inspector = inspect(conn) + return table_name in inspector.get_table_names() + + def upgrade() -> None: """Upgrade schema to v2 validation models.""" _create_validation_sources() @@ -115,6 +121,8 @@ def downgrade() -> None: def _create_validation_sources() -> None: """Create the validation_sources table (no FK dependencies).""" + if not _table_exists("validation_policies"): + return op.create_table( "validation_sources", sa.Column("id", sa.String(), nullable=False), @@ -150,6 +158,8 @@ def _create_validation_sources() -> None: def _create_validation_runs() -> None: """Create the validation_runs table.""" + if not _table_exists("validation_policies"): + return op.create_table( "validation_runs", sa.Column("id", sa.String(), nullable=False), @@ -207,6 +217,8 @@ def _create_validation_runs() -> None: def _extend_validation_policies() -> None: """Add v2 columns and index to validation_policies.""" + if not _table_exists("validation_policies"): + return op.add_column( "validation_policies", sa.Column("prompt_template", sa.Text(), nullable=True), @@ -269,6 +281,8 @@ def _extend_validation_policies() -> None: def _extend_validation_results() -> None: """Add v2 columns, FKs, and indexes to llm_validation_results.""" + if not _table_exists("llm_validation_results"): + return op.add_column( "llm_validation_results", sa.Column( @@ -335,6 +349,8 @@ def _backfill_validation_runs() -> None: Safe no-op when no such rows exist (e.g. fresh database). """ + if not _table_exists("llm_validation_results"): + return conn = op.get_bind() # Check for existing records that need backfill diff --git a/backend/alembic/versions/e5f4d3c2b1a_add_policy_id_to_validation_results.py b/backend/alembic/versions/e5f4d3c2b1a_add_policy_id_to_validation_results.py index 3b982e34..11ddf923 100644 --- a/backend/alembic/versions/e5f4d3c2b1a_add_policy_id_to_validation_results.py +++ b/backend/alembic/versions/e5f4d3c2b1a_add_policy_id_to_validation_results.py @@ -8,9 +8,9 @@ Create Date: 2026-05-21 12:00:00.000000 from collections.abc import Sequence -import sqlalchemy as sa - from alembic import op +import sqlalchemy as sa +from sqlalchemy import inspect # revision identifiers, used by Alembic. revision: str = "e5f4d3c2b1a" @@ -19,8 +19,16 @@ branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None +def _table_exists(table_name: str) -> bool: + conn = op.get_bind() + inspector = inspect(conn) + return table_name in inspector.get_table_names() + + def upgrade() -> None: """Add policy_id column to llm_validation_results as nullable indexed.""" + if not _table_exists("llm_validation_results"): + return op.add_column( "llm_validation_results", sa.Column("policy_id", sa.String(), nullable=True, index=True), diff --git a/backend/alembic/versions/ed28d34edde7_add_max_images_to_llm_providers.py b/backend/alembic/versions/ed28d34edde7_add_max_images_to_llm_providers.py index 9c3f4139..bede14c4 100644 --- a/backend/alembic/versions/ed28d34edde7_add_max_images_to_llm_providers.py +++ b/backend/alembic/versions/ed28d34edde7_add_max_images_to_llm_providers.py @@ -9,6 +9,7 @@ Create Date: 2026-05-31 22:19:21.922928 from typing import Sequence, Union from alembic import op +from sqlalchemy import inspect import sqlalchemy as sa @@ -19,6 +20,14 @@ depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: + # The llm_providers table is defined in the ORM model and created by + # Base.metadata.create_all() at app startup, not via migrations. + # If the table doesn't exist yet (fresh DB), skip — the model definition + # already includes max_images, so create_all() will create it with the column. + conn = op.get_bind() + inspector = inspect(conn) + if "llm_providers" not in inspector.get_table_names(): + return op.add_column('llm_providers', sa.Column('max_images', sa.Integer(), nullable=True)) diff --git a/backend/src/app.py b/backend/src/app.py index 8eba40f6..82d66764 100755 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -18,7 +18,6 @@ from pathlib import Path # project_root is used for static files mounting project_root = Path(__file__).resolve().parent.parent.parent -from alembic.config import Config as AlembicConfig from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect, status from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, JSONResponse @@ -27,8 +26,6 @@ from starlette.middleware.base import BaseHTTPMiddleware from starlette.middleware.sessions import SessionMiddleware from typing import Any -from alembic import command as alembic_command - from .api import auth from .api.routes import ( admin, @@ -67,10 +64,14 @@ from .models.auth import Role, User # #region lifespan [C:3] [TYPE Function] # @BRIEF Async context manager for FastAPI startup/shutdown lifecycle. -# @RELATION CALLS -> [run_alembic_migrations] # @RELATION CALLS -> [init_db] # @RELATION CALLS -> [AppDependencies] -# @POST On startup: schema up-to-date, admin exists, scheduler started. On shutdown: scheduler stopped. +# @POST On startup: admin exists, scheduler started. On shutdown: scheduler stopped. +# @RATIONALE Alembic migrations removed from lifespan — they now run exclusively +# in docker/backend.entrypoint.sh (wait_for_db → alembic upgrade head). +# Running migrations in both places added ~5s startup overhead and masked +# partial failures. init_db() remains as a safety net for tables without +# dedicated Alembic migrations (e.g., newly added models during development). @asynccontextmanager async def lifespan(app: FastAPI): # Startup @@ -78,9 +79,7 @@ async def lifespan(app: FastAPI): with belief_scope("startup_event"): logger.info("🔐 Ensuring encryption key...") ensure_encryption_key() - logger.info("📦 Running Alembic migrations...") - run_alembic_migrations() - logger.info("🗄️ Initializing database tables...") + logger.info("🗄️ Initializing database tables (safety net)...") init_db() logger.info("👤 Bootstrapping admin user...") ensure_initial_admin_user() @@ -109,7 +108,6 @@ async def lifespan(app: FastAPI): scheduler = get_scheduler_service() scheduler.start() logger.info("✅ Application startup complete") - logger.info("✅ Application startup complete") yield # Shutdown scheduler.stop() @@ -185,13 +183,16 @@ def ensure_initial_admin_user() -> None: # #endregion ensure_initial_admin_user # #region run_alembic_migrations [C:2] [TYPE Function] # @BRIEF Applies all pending Alembic migrations against DATABASE_URL. +# DEPRECATED: Migrations now run exclusively in docker/backend.entrypoint.sh. +# Kept for local development (manual invocation). # @POST All Alembic migrations up to 'head' are applied. # @SIDE_EFFECT Executes ALTER TABLE / CREATE TABLE via Alembic chain. -# @RATIONALE Single source of truth for schema changes. All column additions, -# drops, renames, and data migrations go through Alembic. -# init_db() runs after to create any tables that don't have -# a standalone Alembic migration yet. +# @DEPRECATED Migrations moved to entrypoint.sh — this function is kept for +# local development only. Do NOT call from lifespan or production code. def run_alembic_migrations() -> None: + from alembic.config import Config as AlembicConfig + from alembic import command as alembic_command + with belief_scope("run_alembic_migrations"): try: alembic_cfg = AlembicConfig("alembic.ini") diff --git a/backend/src/core/database.py b/backend/src/core/database.py index 05284217..0beadcbd 100644 --- a/backend/src/core/database.py +++ b/backend/src/core/database.py @@ -760,15 +760,17 @@ def _ensure_dictionary_entries_columns(bind_engine): # #region init_db [C:3] [TYPE Function] -# @BRIEF Creates any missing tables via create_all(). +# @BRIEF Creates any missing tables via create_all() — safety net for development. # @PRE engine, tasks_engine and auth_engine are initialized. -# Alembic migrations (run_alembic_migrations) have already been applied. +# In Docker: Alembic migrations already applied via entrypoint.sh. # @POST All tables from SQLAlchemy models exist in all databases. # @SIDE_EFFECT Creates physical database tables if they don't exist. -# @RATIONALE Schema migrations are handled by Alembic (run_alembic_migrations runs -# before init_db in startup_event). create_all() only creates NEW tables -# that don't exist yet — it does NOT alter existing ones. All column -# additions must go through Alembic migrations. +# @RATIONALE create_all() is idempotent — it only creates tables that don't exist, +# never alters existing ones. In Docker, Alembic runs first in entrypoint.sh +# (wait_for_db → alembic upgrade head), so init_db() is a no-op for production. +# For local development (run.sh without Docker), this provides a safety net. +# @REJECTED Removing init_db() entirely was rejected — local development without +# Docker would require manual Alembic setup, increasing friction for developers. # @REJECTED _ensure_*() inline additive migrations removed — they were duplicating # Alembic logic, were not versioned, and created hidden schema drift. # All schema changes must now go through Alembic. diff --git a/backend/src/scripts/check_migration_chain.py b/backend/src/scripts/check_migration_chain.py new file mode 100644 index 00000000..e931e91b --- /dev/null +++ b/backend/src/scripts/check_migration_chain.py @@ -0,0 +1,114 @@ +# #region check_migration_chain [C:3] [TYPE Module] [SEMANTICS alembic,migration,validation,ci] +# @BRIEF Validates Alembic migration chain integrity — checks for multiple heads, +# broken chains, and missing down_revision references. +# @RATIONALE Merge migrations (e.g., f0e9d8c7b6a5) are fragile — a new migration +# with wrong down_revision can create multiple heads and break `alembic upgrade head`. +# This script catches the issue before deployment. +# @REJECTED Relying on `alembic upgrade head` to fail at deploy time was rejected — +# it's better to catch migration chain issues in CI or pre-commit. + +import sys +from pathlib import Path + +# Ensure backend/src is importable +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + + +# #region check_migration_heads [C:2] [TYPE Function] +# @BRIEF Checks that there is exactly one migration head. +# @POST Prints error and exits 1 if multiple heads found. +def check_migration_heads() -> None: + from alembic.config import Config as AlembicConfig + from alembic.script import ScriptDirectory + + alembic_cfg = AlembicConfig("alembic.ini") + alembic_cfg.set_main_option("script_location", "alembic") + + script_dir = ScriptDirectory.from_config(alembic_cfg) + heads = list(script_dir.get_heads()) + + if len(heads) == 0: + print("[migration-check] ERROR: No migration heads found — chain is empty") + sys.exit(1) + + if len(heads) > 1: + print(f"[migration-check] ERROR: Multiple migration heads detected ({len(heads)}):") + for h in heads: + rev = script_dir.get_revision(h) + print(f" - {h}: {rev.doc}") + print("[migration-check] Fix: ensure new migrations reference the correct parent") + sys.exit(1) + + print(f"[migration-check] OK: Single migration head — {heads[0]}") + + +# #endregion check_migration_heads + +# #region check_migration_chain_integrity [C:2] [TYPE Function] +# @BRIEF Walks the migration chain from head to root, checking all down_revision links exist. +# @POST Prints error and exits 1 if a broken link is found. +def check_migration_chain_integrity() -> None: + from alembic.config import Config as AlembicConfig + from alembic.script import ScriptDirectory + + alembic_cfg = AlembicConfig("alembic.ini") + alembic_cfg.set_main_option("script_location", "alembic") + + script_dir = ScriptDirectory.from_config(alembic_cfg) + heads = list(script_dir.get_heads()) + + if not heads: + return # already checked above + + visited = set() + current_revs = heads + + while current_revs: + next_revs = [] + for rev_id in current_revs: + if rev_id in visited: + continue + visited.add(rev_id) + rev = script_dir.get_revision(rev_id) + + down = rev.down_revision + if down is None: + continue # root migration + + if isinstance(down, str): + try: + script_dir.get_revision(down) + except Exception: + print(f"[migration-check] ERROR: Broken link — {rev_id} -> {down} (not found)") + sys.exit(1) + next_revs.append(down) + elif isinstance(down, tuple): + # Merge migration — check all parents exist + for parent in down: + try: + script_dir.get_revision(parent) + except Exception: + print(f"[migration-check] ERROR: Broken merge link — {rev_id} -> {parent} (not found)") + sys.exit(1) + next_revs.append(parent) + + current_revs = next_revs + + print(f"[migration-check] OK: Chain integrity verified — {len(visited)} migrations traversed") + + +# #endregion check_migration_chain_integrity + +# #region check_migration_main [C:2] [TYPE Function] +# @BRIEF Entry point — runs all migration chain checks. +def main() -> None: + print("[migration-check] === Alembic Migration Chain Validation ===") + check_migration_heads() + check_migration_chain_integrity() + print("[migration-check] === All checks passed ===") + + +# #endregion check_migration_main + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_alembic_migrations.py b/backend/tests/test_alembic_migrations.py index 52631142..21dcc214 100644 --- a/backend/tests/test_alembic_migrations.py +++ b/backend/tests/test_alembic_migrations.py @@ -1,16 +1,17 @@ # #region TestAlembicMigrations [C:3] [TYPE Module] [SEMANTICS tests, alembic, migration, schema, idempotent] -# @BRIEF Verifies Alembic migration chain: fresh upgrade and legacy stamp. +# @BRIEF Verifies Alembic migration chain: fresh upgrade and legacy upgrade. # @RELATION BINDS_TO -> [ed310b33f02c] # @INVARIANT Running `alembic upgrade head` twice is idempotent (no-op on second run) on PostgreSQL. -# @INVARIANT Legacy database with existing tables can be stamped after adding boot-critical columns. +# @INVARIANT Legacy database with existing tables can be upgraded via `alembic upgrade head`. # @PRE Alembic is installed, migration files exist in alembic/versions/. # @POST Schema matches expected state after upgrade. # @TEST_INVARIANT: alembic_fresh_upgrade -> VERIFIED_BY: [test_fresh_upgrade_creates_tables] -# @TEST_INVARIANT: alembic_stamp_legacy -> VERIFIED_BY: [test_legacy_database_stamp] +# @TEST_INVARIANT: alembic_upgrade_legacy -> VERIFIED_BY: [test_legacy_database_upgrade] # @REJECTED Full migration chain idempotency on SQLite — initial migration uses # op.create_index() which conflicts with SQLite auto-created indexes # for UNIQUE constraints. Migrations are designed for PostgreSQL. -# Legacy databases use stamp + selective raw SQL in entrypoint instead. +# @REJECTED Legacy stamp + raw SQL rejected — entrypoint now runs `alembic upgrade head` +# for legacy databases, ensuring all missing tables/columns are created. import os import sys @@ -65,51 +66,43 @@ def test_fresh_upgrade_creates_tables() -> None: # #endregion test_fresh_upgrade_creates_tables -# #region test_legacy_database_stamp [C:2] [TYPE Function] -# @BRIEF Pre-populate tables via create_all(), then verify entrypoint-style stamp works. -# @POST alembic_version created, is_admin column present after stamp + raw SQL. +# #region test_legacy_database_upgrade [C:2] [TYPE Function] +# @BRIEF Simulate legacy DB (tables exist, no alembic_version), verify upgrade head works. +# @POST alembic_version created, all migration columns present after upgrade. +# @TEST_INVARIANT: alembic_upgrade_legacy -> VERIFIED_BY: [test_legacy_database_upgrade] @_REQUIRES_PG -def test_legacy_database_stamp() -> None: - """Alembic stamp works on a database that already has tables (PostgreSQL only).""" +def test_legacy_database_upgrade() -> None: + """Alembic upgrade head works on a database that already has tables but no alembic_version.""" from sqlalchemy import create_engine, inspect from sqlalchemy import text as sa_text from src.models.mapping import Base from src.models import auth as _auth # noqa: F401 from src.models import task as _task # noqa: F401 - from alembic.config import Config as AlembicConfig - from alembic import command as alembic_command engine = create_engine(os.environ["DATABASE_URL"]) # Drop alembic_version if exists to simulate legacy state with engine.begin() as conn: conn.execute(sa_text("DROP TABLE IF EXISTS alembic_version CASCADE")) engine.dispose() - # Tables already exist from previous runs (create_all) - # Add is_admin column (simulating entrypoint raw SQL) - engine = create_engine(os.environ["DATABASE_URL"]) - with engine.begin() as conn: - inspector = inspect(conn) - roles_cols = {c["name"] for c in inspector.get_columns("roles")} - if "is_admin" not in roles_cols: - conn.execute( - sa_text("ALTER TABLE roles ADD COLUMN is_admin BOOLEAN NOT NULL DEFAULT FALSE") - ) - conn.execute(sa_text("UPDATE roles SET is_admin = TRUE WHERE name = 'Admin'")) - engine.dispose() - # Stamp head (entrypoint behavior for legacy DB) - backend_dir = Path(__file__).parent.parent - alembic_cfg = AlembicConfig(str(backend_dir / "alembic.ini")) - alembic_cfg.set_main_option("script_location", str(backend_dir / "alembic")) - alembic_command.stamp(alembic_cfg, "head") + + # Tables already exist from previous runs (create_all) — this is the legacy state. + # Run upgrade head (new entrypoint behavior for legacy DB) + _run_alembic_upgrade() + # Verify engine = create_engine(os.environ["DATABASE_URL"]) inspector = inspect(engine) tables = set(inspector.get_table_names()) - assert "alembic_version" in tables, "alembic_version table not created by stamp" + assert "alembic_version" in tables, "alembic_version table not created by upgrade" roles_cols = {c["name"] for c in inspector.get_columns("roles")} - assert "is_admin" in roles_cols + assert "is_admin" in roles_cols, "roles.is_admin column missing after legacy upgrade" + + # Verify version_num matches head + result = engine.connect().execute(sa_text("SELECT version_num FROM alembic_version")) + version = result.scalar_one() + assert version is not None and len(version) > 0 engine.dispose() -# #endregion test_legacy_database_stamp +# #endregion test_legacy_database_upgrade # #region _run_alembic_upgrade [C:1] [TYPE Function] diff --git a/docker-compose.enterprise-clean.yml b/docker-compose.enterprise-clean.yml index ada864ff..eb0ab6ec 100644 --- a/docker-compose.enterprise-clean.yml +++ b/docker-compose.enterprise-clean.yml @@ -15,6 +15,10 @@ # @RELATION DEPENDS_ON -> [docker/backend.entrypoint.sh] # @DATA_CONTRACT Input: .env.enterprise-clean + ./certs/*.crt -> Output: запущенные контейнеры # @INVARIANT Внешний Postgres НЕ требует healthcheck внутри compose — ответственность оператора. +# Backend entrypoint использует wait_for_db() (retry до 30 раз) для проверки доступности БД +# перед запуском миграций. Если БД недоступна — контейнер exit(1) после таймаута. +# @RATIONALE Нет depends_on для backend → внешний PostgreSQL: оператор гарантирует доступность. +# wait_for_db() в entrypoint.sh обеспечивает graceful retry вместо мгновенного crash. # @INVARIANT Если CERTS_PATH пуст или не содержит .crt файлов, entrypoint не падает. # @INVARIANT nginx слушает порт 80 всегда. Порт 443 включается только если есть server.crt + server.key. # @RATIONALE Отказ от встроенного postgres-container: корпоративный стандарт — внешний PostgreSQL diff --git a/docker/backend.entrypoint.sh b/docker/backend.entrypoint.sh index eb086ad5..4486e18f 100755 --- a/docker/backend.entrypoint.sh +++ b/docker/backend.entrypoint.sh @@ -2,14 +2,17 @@ set -euo pipefail # #region docker.backend.entrypoint [C:4] [TYPE Module] [SEMANTICS docker,entrypoint,enterprise,certs] -# @BRIEF Container entrypoint — установка корп. сертификатов, bootstrap admin, -# затем запуск backend (exec "$@"). +# @BRIEF Container entrypoint — установка корп. сертификатов, wait-for-DB, +# Alembic миграции, bootstrap admin, затем запуск backend (exec "$@"). # @PRE Python runtime и backend source доступны в /app/backend. # Переменная CERTS_PATH указывает на директорию с .crt файлами (или пусто). +# DATABASE_URL установлен и указывает на доступный PostgreSQL. # @POST Сертификаты установлены в системное хранилище и NSS базу Chromium. +# БД подтверждена как доступная. Alembic миграции применены. # Admin создан (если INITIAL_ADMIN_CREATE=true). Backend запущен через exec "$@". # @SIDE_EFFECT Модифицирует /usr/local/share/ca-certificates/ и вызывает update-ca-certificates. # Импортирует CA в NSS DB (~/.pki/nssdb) для Chromium. +# Выполняет Alembic upgrade head. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [backend/src/scripts/create_admin.py] # @RELATION DEPENDS_ON -> [backend/src/scripts/init_auth_db.py] @@ -18,6 +21,9 @@ set -euo pipefail # (create_admin.py идемпотентен). # @INVARIANT Повторный запуск install_llm_ca_certs не дублирует сертификаты в bundle. # @INVARIANT Повторный запуск install_ca_to_nss не дублирует сертификаты в NSS DB. +# @INVARIANT wait_for_db exit(1) если БД недоступна после DB_WAIT_ATTEMPTS попыток — +# контейнер не продолжит запуск с недоступной БД. +# @INVARIANT Alembic миграции выполняются ТОЛЬКО в entrypoint, НЕ в app.py lifespan. # #endregion docker.backend.entrypoint # ====================================================================== @@ -420,6 +426,73 @@ install_ca_to_nss() { } # #endregion docker.backend.entrypoint.install_ca_to_nss +# ====================================================================== +# wait_for_db — проверка доступности БД с retry +# ====================================================================== +# #region docker.backend.entrypoint.wait_for_db [C:3] [TYPE Function] +# @BRIEF Проверяет доступность PostgreSQL с retry-логикой перед запуском миграций. +# @PRE DATABASE_URL установлен и указывает на валидный PostgreSQL. +# @POST Соединение с БД подтверждено (SELECT 1) или скрипт exit(1) после таймаута. +# @RATIONALE Без wait-for-db enterprise-деплой с внешним PostgreSQL падает +# мгновенно — entrypoint интерпретирует connection refused как "пустая БД". +# @REJECTED Использование pg_isready напрямую отвергнуто — клиент psql может +# отсутствовать в python:3.11-slim образе. SQLAlchemy-проверка универсальна. +wait_for_db() { + local max_attempts="${DB_WAIT_ATTEMPTS:-30}" + local delay="${DB_WAIT_DELAY:-2}" + local attempt=0 + + echo "[entrypoint] Waiting for database (max ${max_attempts} attempts, ${delay}s delay)..." + while [ $attempt -lt $max_attempts ]; do + attempt=$((attempt + 1)) + # Capture stderr to detect permanent errors (KeyError, ArgumentError) + local _err_output + _err_output="$(python3 -c " +import os, sys +from sqlalchemy import create_engine, text +try: + url = os.environ['DATABASE_URL'] +except KeyError: + print('FATAL: DATABASE_URL environment variable is not set', file=sys.stderr) + sys.exit(99) +try: + e = create_engine(url) + with e.connect() as c: + c.execute(text('SELECT 1')) +except Exception as exc: + # Permanent errors — no point retrying + err_str = str(exc).lower() + if 'database_url' in err_str or 'could not parse' in err_str or 'invalid' in err_str: + print(f'FATAL: {exc}', file=sys.stderr) + sys.exit(99) + sys.exit(1) +" 2>&1)" + local _rc=$? + + if [ $_rc -eq 0 ]; then + echo "[entrypoint] ✅ Database is reachable" + return 0 + fi + + # Permanent error — exit immediately, don't waste retries + if [ $_rc -eq 99 ]; then + echo "[entrypoint] ❌ $_err_output" >&2 + return 1 + fi + + echo "[entrypoint] Attempt ${attempt}/${max_attempts} — retrying in ${delay}s..." + sleep "$delay" + done + + # P0: Mask credentials — never print full DATABASE_URL to logs + local _masked_url + _masked_url="$(echo "$DATABASE_URL" | sed -E 's|://([^:]+):([^@]+)@|://\1:***@|')" + echo "[entrypoint] ❌ Database not reachable after ${max_attempts} attempts" >&2 + echo "[entrypoint] DATABASE_URL=${_masked_url}" >&2 + return 1 +} +# #endregion docker.backend.entrypoint.wait_for_db + # ====================================================================== # MAIN # ====================================================================== @@ -428,8 +501,10 @@ install_certificates install_llm_ca_certs install_ca_to_nss -# Check DB state: empty, legacy (tables exist but no alembic), or managed -# || prevents set -e from killing the script on non-zero exit from detection script +# ── C1: Wait for database before any migration logic ── +wait_for_db + +# ── C2: DB state detection (now safe — DB is confirmed reachable) ── _db_state=0 python3 -c " import os @@ -452,25 +527,11 @@ elif [ $_db_state -eq 1 ]; then alembic upgrade head 2>&1 | sed 's/^/[entrypoint] /' echo "[entrypoint] ✅ Alembic initial migration applied" else - echo "[entrypoint] Legacy database — adding critical migration columns..." - # Add boot-critical columns that Alembic migrations would add - python3 -c " -import os -from sqlalchemy import create_engine, inspect, text -e = create_engine(os.environ['DATABASE_URL']) -with e.begin() as conn: - inspector = inspect(conn) - roles_cols = {c['name'] for c in inspector.get_columns('roles')} - if 'is_admin' not in roles_cols: - conn.execute(text('ALTER TABLE roles ADD COLUMN is_admin BOOLEAN NOT NULL DEFAULT FALSE')) - conn.execute(text(\"UPDATE roles SET is_admin = TRUE WHERE name = 'Admin'\")) - print(' roles.is_admin column added') - else: - print(' roles.is_admin already exists') -" - echo "[entrypoint] Stamping Alembic head (tables already exist)..." - alembic stamp head 2>&1 | sed 's/^/[entrypoint] /' - echo "[entrypoint] ✅ Alembic stamped head, schema ready" + echo "[entrypoint] Legacy database — running Alembic upgrade (not stamp)..." + # H2: Запускаем полноценный upgrade вместо stamp head — legacy-БД может + # отсутствовать таблицы, добавленные миграциями после написания entrypoint. + alembic upgrade head 2>&1 | sed 's/^/[entrypoint] /' + echo "[entrypoint] ✅ Alembic upgrade complete for legacy database" fi bootstrap_admin diff --git a/frontend/e2e/run-enterprise-clean-e2e.sh b/frontend/e2e/run-enterprise-clean-e2e.sh index 534d44ab..1b1a3514 100755 --- a/frontend/e2e/run-enterprise-clean-e2e.sh +++ b/frontend/e2e/run-enterprise-clean-e2e.sh @@ -234,19 +234,16 @@ stage1_enterprise_clean() { docker build -f docker/frontend.Dockerfile -t "ss-tools-frontend:${RELEASE_TAG}" . fi - # ── 1.3. START POSTGRESQL ──────────────────────────────── + # ── 1.3. START POSTGRESQL (on default bridge first) ───── log_info "Step 1.3: Starting new PostgreSQL container..." docker rm -f ss-tools-e2e-pg 2>/dev/null || true - # Create the network that docker compose will use (ss-tools-e2e-ec_default) - # so PostgreSQL is on the same network and resolvable by container name. - if ! docker network inspect ss-tools-e2e-ec_default >/dev/null 2>&1; then - docker network create ss-tools-e2e-ec_default - fi + # Remove any stale compose network so docker compose creates it fresh + docker network rm ss-tools-e2e-ec_default 2>/dev/null || true + # Start PG on default bridge initially — compose will create its own network docker run -d \ --name ss-tools-e2e-pg \ - --network ss-tools-e2e-ec_default \ -e POSTGRES_DB=ss_tools \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ @@ -267,7 +264,6 @@ stage1_enterprise_clean() { sleep 1 done - # Get PostgreSQL container IP on the enterprise network EC_PG_HOST="ss-tools-e2e-pg" EC_PG_PORT="5432" @@ -308,11 +304,20 @@ EOF export FRONTEND_TAG="${RELEASE_TAG}" fi + # Start compose FIRST so it creates the network with proper compose labels. + # The backend will fail to connect to PG on first start (PG not on compose network + # yet), but Docker restart policy (unless-stopped) will retry. docker compose -p ss-tools-e2e-ec \ -f "$EC_COMPOSE_FILE" \ --env-file "/tmp/.env.e2e-enterprise-clean" \ up -d + # Connect PG to the compose-created network so backend can resolve ss-tools-e2e-pg + docker network connect ss-tools-e2e-ec_default ss-tools-e2e-pg + + # Restart backend so entrypoint reruns with PG accessible + docker compose -p ss-tools-e2e-ec restart backend + # ── 1.5. WAIT FOR BACKEND + FRONTEND ──────────────────── log_info "Step 1.5: Waiting for services..."