# #region DatabaseModule [C:3] [TYPE Module] [SEMANTICS sqlalchemy, connection, session] # # @BRIEF Configures database connection and session management (PostgreSQL-first). # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [MappingModels] # @RELATION DEPENDS_ON -> [auth_config] # # @INVARIANT A single engine instance is used for the entire application. import os from pathlib import Path from sqlalchemy import create_engine, inspect, text from sqlalchemy.orm import sessionmaker # Import models to ensure they're registered with Base from ..models import ( api_key as _api_key_models, # noqa: F401 assistant as _assistant_models, # noqa: F401 auth as _auth_models, # noqa: F401 clean_release as _clean_release_models, # noqa: F401 config as _config_models, # noqa: F401 dataset_review as _dataset_review_models, # noqa: F401 llm as _llm_models, # noqa: F401 maintenance as _maintenance_models, # noqa: F401 profile as _profile_models, # noqa: F401 task as _task_models, # noqa: F401 ) from ..models.mapping import Base from .auth.config import auth_config from .logger import belief_scope, logger # #region BASE_DIR [C:1] [TYPE Variable] # @BRIEF Base directory for the backend. BASE_DIR = Path(__file__).resolve().parent.parent.parent # #endregion BASE_DIR # #region DATABASE_URL [C:1] [TYPE Constant] # @BRIEF URL for the main application database. Read from env; crashes if unset. # @RATIONALE DATABASE_URL reads from env (DATABASE_URL or POSTGRES_URL). # Crashes at import if unset. DEV_MODE fallback removed in [SEC:C-4]. # @REJECTED Hardcoded postgres:postgres@localhost in source code rejected — exposes # database credentials in version control (Class 1 security violation). # DEV_MODE fallback removed — same violation via env toggle. DATABASE_URL = os.getenv("DATABASE_URL") or os.getenv("POSTGRES_URL") if not DATABASE_URL: raise RuntimeError( "DATABASE_URL (or POSTGRES_URL) environment variable is required. " "Set it before starting the server. " "For local development, create a .env file or use docker-compose.yml." ) # #endregion DATABASE_URL # #region TASKS_DATABASE_URL [C:1] [TYPE Constant] # @BRIEF URL for the tasks execution database. # Defaults to DATABASE_URL to keep task logs in the same PostgreSQL instance. TASKS_DATABASE_URL = os.getenv("TASKS_DATABASE_URL", DATABASE_URL) # #endregion TASKS_DATABASE_URL # #region AUTH_DATABASE_URL [C:1] [TYPE Constant] # @BRIEF URL for the authentication database. AUTH_DATABASE_URL = os.getenv("AUTH_DATABASE_URL", auth_config.AUTH_DATABASE_URL) # #endregion AUTH_DATABASE_URL # #region engine [C:1] [TYPE Variable] # @BRIEF SQLAlchemy engine for mappings database. # @SIDE_EFFECT Creates database engine and manages connection pool. def _build_engine(db_url: str): with belief_scope("_build_engine"): if db_url.startswith("sqlite"): return create_engine(db_url, connect_args={"check_same_thread": False}) return create_engine(db_url, pool_pre_ping=True) engine = _build_engine(DATABASE_URL) # #endregion engine # #region tasks_engine [C:1] [TYPE Variable] # @BRIEF SQLAlchemy engine for tasks database. tasks_engine = _build_engine(TASKS_DATABASE_URL) # #endregion tasks_engine # #region auth_engine [C:1] [TYPE Variable] # @BRIEF SQLAlchemy engine for authentication database. auth_engine = _build_engine(AUTH_DATABASE_URL) # #endregion auth_engine # #region SessionLocal [C:1] [TYPE Class] # @BRIEF A session factory for the main mappings database. # @PRE engine is initialized. SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) # #endregion SessionLocal # #region TasksSessionLocal [C:1] [TYPE Class] # @BRIEF A session factory for the tasks execution database. # @PRE tasks_engine is initialized. TasksSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=tasks_engine) # #endregion TasksSessionLocal # #region AuthSessionLocal [C:1] [TYPE Class] # @BRIEF A session factory for the authentication database. # @PRE auth_engine is initialized. AuthSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=auth_engine) # #endregion AuthSessionLocal # #region _ensure_user_dashboard_preferences_columns [C:3] [TYPE Function] # @BRIEF Applies additive schema upgrades for user_dashboard_preferences table. # @PRE bind_engine points to application database where profile table is stored. # @POST Missing columns are added without data loss. # @RELATION DEPENDS_ON -> [engine] def _ensure_user_dashboard_preferences_columns(bind_engine): with belief_scope("_ensure_user_dashboard_preferences_columns"): table_name = "user_dashboard_preferences" inspector = inspect(bind_engine) if table_name not in inspector.get_table_names(): return existing_columns = { str(column.get("name") or "").strip() for column in inspector.get_columns(table_name) } alter_statements = [] if "git_username" not in existing_columns: alter_statements.append( "ALTER TABLE user_dashboard_preferences ADD COLUMN git_username VARCHAR" ) if "git_email" not in existing_columns: alter_statements.append( "ALTER TABLE user_dashboard_preferences ADD COLUMN git_email VARCHAR" ) if "git_personal_access_token_encrypted" not in existing_columns: alter_statements.append( "ALTER TABLE user_dashboard_preferences " "ADD COLUMN git_personal_access_token_encrypted VARCHAR" ) if "start_page" not in existing_columns: alter_statements.append( "ALTER TABLE user_dashboard_preferences " "ADD COLUMN start_page VARCHAR NOT NULL DEFAULT 'dashboards'" ) if "auto_open_task_drawer" not in existing_columns: alter_statements.append( "ALTER TABLE user_dashboard_preferences " "ADD COLUMN auto_open_task_drawer BOOLEAN NOT NULL DEFAULT TRUE" ) if "dashboards_table_density" not in existing_columns: alter_statements.append( "ALTER TABLE user_dashboard_preferences " "ADD COLUMN dashboards_table_density VARCHAR NOT NULL DEFAULT 'comfortable'" ) if "show_only_slug_dashboards" not in existing_columns: alter_statements.append( "ALTER TABLE user_dashboard_preferences " "ADD COLUMN show_only_slug_dashboards BOOLEAN NOT NULL DEFAULT TRUE" ) if not alter_statements: return try: with bind_engine.begin() as connection: for statement in alter_statements: connection.execute(text(statement)) except Exception as migration_error: logger.explore( "Profile preference additive migration failed", extra={"src": "database", "error": str(migration_error)}, ) # #endregion _ensure_user_dashboard_preferences_columns # #region _ensure_user_dashboard_preferences_health_columns [C:3] [TYPE Function] # @BRIEF Applies additive schema upgrades for user_dashboard_preferences table (health fields). # @RELATION DEPENDS_ON -> [engine] def _ensure_user_dashboard_preferences_health_columns(bind_engine): with belief_scope("_ensure_user_dashboard_preferences_health_columns"): table_name = "user_dashboard_preferences" inspector = inspect(bind_engine) if table_name not in inspector.get_table_names(): return existing_columns = { str(column.get("name") or "").strip() for column in inspector.get_columns(table_name) } alter_statements = [] if "telegram_id" not in existing_columns: alter_statements.append( "ALTER TABLE user_dashboard_preferences ADD COLUMN telegram_id VARCHAR" ) if "email_address" not in existing_columns: alter_statements.append( "ALTER TABLE user_dashboard_preferences ADD COLUMN email_address VARCHAR" ) if "notify_on_fail" not in existing_columns: alter_statements.append( "ALTER TABLE user_dashboard_preferences ADD COLUMN notify_on_fail BOOLEAN NOT NULL DEFAULT TRUE" ) if not alter_statements: return try: with bind_engine.begin() as connection: for statement in alter_statements: connection.execute(text(statement)) except Exception as migration_error: logger.explore( "Profile health preference additive migration failed", extra={"src": "database", "error": str(migration_error)}, ) # #endregion _ensure_user_dashboard_preferences_health_columns # #region _ensure_llm_validation_results_columns [C:3] [TYPE Function] # @BRIEF Applies additive schema upgrades for llm_validation_results table. # @RELATION DEPENDS_ON -> [engine] def _ensure_llm_validation_results_columns(bind_engine): with belief_scope("_ensure_llm_validation_results_columns"): table_name = "llm_validation_results" inspector = inspect(bind_engine) if table_name not in inspector.get_table_names(): return existing_columns = { str(column.get("name") or "").strip() for column in inspector.get_columns(table_name) } alter_statements = [] if "task_id" not in existing_columns: alter_statements.append( "ALTER TABLE llm_validation_results ADD COLUMN task_id VARCHAR" ) if "environment_id" not in existing_columns: alter_statements.append( "ALTER TABLE llm_validation_results ADD COLUMN environment_id VARCHAR" ) if not alter_statements: return try: with bind_engine.begin() as connection: for statement in alter_statements: connection.execute(text(statement)) except Exception as migration_error: logger.explore( "ValidationRecord additive migration failed", extra={"src": "database", "error": str(migration_error)}, ) # #endregion _ensure_llm_validation_results_columns # #region _ensure_git_server_configs_columns [C:3] [TYPE Function] # @BRIEF Applies additive schema upgrades for git_server_configs table. # @PRE bind_engine points to application database. # @POST Missing columns are added without data loss. # @RELATION DEPENDS_ON -> [engine] def _ensure_git_server_configs_columns(bind_engine): with belief_scope("_ensure_git_server_configs_columns"): table_name = "git_server_configs" inspector = inspect(bind_engine) if table_name not in inspector.get_table_names(): return existing_columns = { str(column.get("name") or "").strip() for column in inspector.get_columns(table_name) } alter_statements = [] if "default_branch" not in existing_columns: alter_statements.append( "ALTER TABLE git_server_configs ADD COLUMN default_branch VARCHAR NOT NULL DEFAULT 'main'" ) if not alter_statements: return try: with bind_engine.begin() as connection: for statement in alter_statements: connection.execute(text(statement)) except Exception as migration_error: logger.explore( "GitServerConfig preference additive migration failed", extra={"src": "database", "error": str(migration_error)}, ) # #endregion _ensure_git_server_configs_columns # #region _ensure_auth_users_columns [C:3] [TYPE Function] # @BRIEF Applies additive schema upgrades for auth users table. # @PRE bind_engine points to authentication database. # @POST Missing columns are added without data loss. # @RELATION DEPENDS_ON -> [auth_engine] def _ensure_auth_users_columns(bind_engine): with belief_scope("_ensure_auth_users_columns"): table_name = "users" inspector = inspect(bind_engine) if table_name not in inspector.get_table_names(): return existing_columns = { str(column.get("name") or "").strip() for column in inspector.get_columns(table_name) } alter_statements = [] if "full_name" not in existing_columns: alter_statements.append("ALTER TABLE users ADD COLUMN full_name VARCHAR") if "is_ad_user" not in existing_columns: alter_statements.append( "ALTER TABLE users ADD COLUMN is_ad_user BOOLEAN NOT NULL DEFAULT FALSE" ) if not alter_statements: logger.reason( "Auth users schema already up to date", extra={"table": table_name, "columns": sorted(existing_columns)}, ) return logger.reason( "Applying additive auth users schema migration", extra={"table": table_name, "statements": alter_statements}, ) try: with bind_engine.begin() as connection: for statement in alter_statements: connection.execute(text(statement)) logger.reason( "Auth users schema migration completed", extra={ "table": table_name, "added_columns": [ stmt.split(" ADD COLUMN ", 1)[1].split()[0] for stmt in alter_statements ], }, ) except Exception as migration_error: logger.explore( "Auth users additive migration failed", extra={"src": "database", "error": str(migration_error)}, ) raise # #endregion _ensure_auth_users_columns # #region _ensure_roles_is_admin_column [C:3] [TYPE Function] # @BRIEF Add is_admin column to roles table if missing (additive migration). # @PRE Database connection is active. # @POST roles.is_admin column exists (BOOLEAN, default FALSE). # @SIDE_EFFECT Executes ALTER TABLE on the auth database. def _ensure_roles_is_admin_column(bind_engine): with belief_scope("_ensure_roles_is_admin_column"): table_name = "roles" inspector = inspect(bind_engine) if table_name not in inspector.get_table_names(): return existing_columns = { str(column.get("name") or "").strip() for column in inspector.get_columns(table_name) } if "is_admin" in existing_columns: logger.reason("roles.is_admin column already exists") return alter = "ALTER TABLE roles ADD COLUMN is_admin BOOLEAN NOT NULL DEFAULT FALSE" try: with bind_engine.begin() as connection: connection.execute(text(alter)) connection.execute(text("UPDATE roles SET is_admin = true WHERE name = 'Admin'")) logger.reason("Added roles.is_admin column, updated Admin roles", extra={"statement": alter}) except Exception as migration_error: logger.explore( "roles.is_admin additive migration failed", extra={"error": str(migration_error)}, ) raise # #endregion _ensure_roles_is_admin_column # #region _ensure_filter_source_enum_values [C:3] [TYPE Function] # @BRIEF Adds missing FilterSource enum values to the PostgreSQL native filtersource type. # @PRE bind_engine points to application database with imported_filters table. # @POST New enum values are available without data loss. # @RELATION DEPENDS_ON -> [engine] def _ensure_filter_source_enum_values(bind_engine): with belief_scope("_ensure_filter_source_enum_values"): try: with bind_engine.connect() as connection: # Check if the native enum type exists result = connection.execute( text( "SELECT t.typname FROM pg_type t " "JOIN pg_namespace n ON t.typnamespace = n.oid " "WHERE t.typname = 'filtersource' AND n.nspname = 'public'" ) ) if result.fetchone() is None: logger.reason( "filtersource enum type does not exist yet; skipping migration" ) return # Get existing enum values result = connection.execute( text( "SELECT e.enumlabel FROM pg_enum e " "JOIN pg_type t ON e.enumtypid = t.oid " "WHERE t.typname = 'filtersource' " "ORDER BY e.enumsortorder" ) ) existing_values = {row[0] for row in result.fetchall()} required_values = ["SUPERSET_PERMALINK", "SUPERSET_NATIVE_FILTERS_KEY"] missing_values = [ v for v in required_values if v not in existing_values ] if not missing_values: logger.reason( "filtersource enum already up to date", extra={"existing": sorted(existing_values)}, ) return logger.reason( "Adding missing values to filtersource enum", extra={"missing": missing_values}, ) for value in missing_values: connection.execute( text( f"ALTER TYPE filtersource ADD VALUE IF NOT EXISTS '{value}'" ) ) connection.commit() logger.reason( "filtersource enum migration completed", extra={"added": missing_values}, ) except Exception as migration_error: logger.explore( "FilterSource enum additive migration failed", extra={"src": "database", "error": str(migration_error)}, ) # #endregion _ensure_filter_source_enum_values # #region _ensure_dataset_review_session_columns [C:4] [TYPE Function] # @BRIEF Apply additive schema upgrades for dataset review persistence required by optimistic-lock and recovery metadata semantics. # @PRE bind_engine points to the application database where dataset review tables are stored. # @POST Missing additive columns across legacy dataset review tables are created without removing existing data. # @SIDE_EFFECT Executes ALTER TABLE statements against dataset review tables in the application database. # @RELATION DEPENDS_ON -> [DatasetReviewSession] # @RELATION DEPENDS_ON -> [ImportedFilter] def _ensure_translation_jobs_columns(bind_engine): with belief_scope("_ensure_translation_jobs_columns"): table_name = "translation_jobs" inspector = inspect(bind_engine) if table_name not in inspector.get_table_names(): return existing_columns = { str(column.get("name") or "").strip() for column in inspector.get_columns(table_name) } if "environment_id" not in existing_columns: try: with bind_engine.begin() as connection: connection.execute( text( "ALTER TABLE translation_jobs " "ADD COLUMN environment_id VARCHAR" ) ) logger.reflect( "Added environment_id column to translation_jobs", ) except Exception as migration_error: logger.explore( "Failed to add environment_id to translation_jobs", extra={"error": str(migration_error)}, ) raise if "target_database_id" not in existing_columns: try: with bind_engine.begin() as connection: connection.execute( text( "ALTER TABLE translation_jobs " "ADD COLUMN target_database_id VARCHAR" ) ) logger.reflect( "Added target_database_id column to translation_jobs", ) except Exception as migration_error: logger.explore( "Failed to add target_database_id to translation_jobs", extra={"error": str(migration_error)}, ) # Columns added to model AFTER initial Alembic migration — no Alembic migration exists if "target_language_column" not in existing_columns: try: with bind_engine.begin() as connection: connection.execute( text( "ALTER TABLE translation_jobs " "ADD COLUMN target_language_column VARCHAR" ) ) logger.reflect("Added target_language_column to translation_jobs") except Exception as migration_error: logger.explore( "Failed to add target_language_column to translation_jobs", extra={"error": str(migration_error)}, ) if "target_source_column" not in existing_columns: try: with bind_engine.begin() as connection: connection.execute( text( "ALTER TABLE translation_jobs " "ADD COLUMN target_source_column VARCHAR" ) ) logger.reflect("Added target_source_column to translation_jobs") except Exception as migration_error: logger.explore( "Failed to add target_source_column to translation_jobs", extra={"error": str(migration_error)}, ) if "target_source_language_column" not in existing_columns: try: with bind_engine.begin() as connection: connection.execute( text( "ALTER TABLE translation_jobs " "ADD COLUMN target_source_language_column VARCHAR" ) ) logger.reflect("Added target_source_language_column to translation_jobs") except Exception as migration_error: logger.explore( "Failed to add target_source_language_column to translation_jobs", extra={"error": str(migration_error)}, ) if "disable_reasoning" not in existing_columns: try: with bind_engine.begin() as connection: connection.execute( text( "ALTER TABLE translation_jobs " "ADD COLUMN disable_reasoning BOOLEAN NOT NULL DEFAULT FALSE" ) ) logger.reflect("Added disable_reasoning column to translation_jobs") except Exception as migration_error: logger.explore( "Failed to add disable_reasoning to translation_jobs", extra={"error": str(migration_error)}, ) def _ensure_dataset_review_session_columns(bind_engine): with belief_scope("_ensure_dataset_review_session_columns"): inspector = inspect(bind_engine) existing_tables = set(inspector.get_table_names()) migration_plan = { "dataset_review_sessions": [ ( "version", "ALTER TABLE dataset_review_sessions " "ADD COLUMN version INTEGER NOT NULL DEFAULT 0", ) ], "imported_filters": [ ( "raw_value_masked", "ALTER TABLE imported_filters " "ADD COLUMN raw_value_masked BOOLEAN NOT NULL DEFAULT FALSE", ) ], } for table_name, planned_columns in migration_plan.items(): if table_name not in existing_tables: logger.reason( "Dataset review table does not exist yet; skipping additive schema migration", extra={"table": table_name}, ) continue existing_columns = { str(column.get("name") or "").strip() for column in inspector.get_columns(table_name) } alter_statements = [ statement for column_name, statement in planned_columns if column_name not in existing_columns ] if not alter_statements: logger.reason( "Dataset review table schema already up to date", extra={"table": table_name, "columns": sorted(existing_columns)}, ) continue logger.reason( "Applying additive dataset review schema migration", extra={"table": table_name, "statements": alter_statements}, ) try: with bind_engine.begin() as connection: for statement in alter_statements: connection.execute(text(statement)) logger.reflect( "Dataset review additive schema migration completed", extra={ "table": table_name, "added_columns": [ stmt.split(" ADD COLUMN ", 1)[1].split()[0] for stmt in alter_statements ], }, ) except Exception as migration_error: logger.explore( "Dataset review additive schema migration failed", extra={"table": table_name, "error": str(migration_error)}, ) raise # #endregion _ensure_dataset_review_session_columns # #region _ensure_translation_schedules_columns [C:3] [TYPE Function] # @BRIEF Applies additive schema upgrades for translation_schedules table. # @PRE bind_engine points to application database. # @POST Missing columns are added without data loss. # @RELATION DEPENDS_ON -> [engine] def _ensure_translation_schedules_columns(bind_engine): with belief_scope("_ensure_translation_schedules_columns"): table_name = "translation_schedules" inspector = inspect(bind_engine) if table_name not in inspector.get_table_names(): return existing_columns = { str(column.get("name") or "").strip() for column in inspector.get_columns(table_name) } alter_statements = [] if "execution_mode" not in existing_columns: alter_statements.append( "ALTER TABLE translation_schedules " "ADD COLUMN execution_mode VARCHAR NOT NULL DEFAULT 'full'" ) if not alter_statements: return try: with bind_engine.begin() as connection: for statement in alter_statements: connection.execute(text(statement)) except Exception as migration_error: logger.explore( "TranslationSchedule additive migration failed", extra={"src": "database", "error": str(migration_error)}, ) # #endregion _ensure_translation_schedules_columns # #region _ensure_dictionary_entries_columns [C:3] [TYPE Function] # @BRIEF Additive migration for dictionary_entries origin tracking columns. # @RELATION DEPENDS_ON -> [engine] def _ensure_dictionary_entries_columns(bind_engine): with belief_scope("_ensure_dictionary_entries_columns"): table_name = "dictionary_entries" inspector = inspect(bind_engine) if table_name not in inspector.get_table_names(): return existing_columns = { str(column.get("name") or "").strip() for column in inspector.get_columns(table_name) } alter_statements = [] if "origin_run_id" not in existing_columns: alter_statements.append( "ALTER TABLE dictionary_entries " "ADD COLUMN origin_run_id VARCHAR" ) if "origin_row_key" not in existing_columns: alter_statements.append( "ALTER TABLE dictionary_entries " "ADD COLUMN origin_row_key VARCHAR" ) if "origin_user_id" not in existing_columns: alter_statements.append( "ALTER TABLE dictionary_entries " "ADD COLUMN origin_user_id VARCHAR" ) if not alter_statements: return try: with bind_engine.begin() as connection: for statement in alter_statements: connection.execute(text(statement)) except Exception as migration_error: logger.explore( "DictionaryEntry additive migration failed", extra={"src": "database", "error": str(migration_error)}, ) # #endregion _ensure_dictionary_entries_columns # #region init_db [C:3] [TYPE Function] # @BRIEF Initializes the database by creating all tables. # @PRE engine, tasks_engine and auth_engine are initialized. # @POST Database tables created in all databases. # @SIDE_EFFECT Creates physical database files if they don't exist. # @RELATION CALLS -> [_ensure_filter_source_enum_values] # @RELATION CALLS -> [_ensure_dataset_review_session_columns] # @RELATION CALLS -> [_ensure_dictionary_entries_columns] def init_db(): with belief_scope("init_db"): Base.metadata.create_all(bind=engine) Base.metadata.create_all(bind=tasks_engine) Base.metadata.create_all(bind=auth_engine) _ensure_user_dashboard_preferences_columns(engine) _ensure_llm_validation_results_columns(engine) _ensure_user_dashboard_preferences_health_columns(engine) _ensure_git_server_configs_columns(engine) _ensure_auth_users_columns(auth_engine) _ensure_roles_is_admin_column(auth_engine) _ensure_filter_source_enum_values(engine) _ensure_dataset_review_session_columns(engine) _ensure_translation_jobs_columns(engine) _ensure_translation_schedules_columns(engine) _ensure_dictionary_entries_columns(engine) # #endregion init_db # #region get_db [C:3] [TYPE Function] # @BRIEF Dependency for getting a database session. # @PRE SessionLocal is initialized. # @POST Session is closed after use. # @RELATION DEPENDS_ON -> [SessionLocal] def get_db(): with belief_scope("get_db"): db = SessionLocal() try: yield db finally: db.close() # #endregion get_db # #region get_tasks_db [C:3] [TYPE Function] # @BRIEF Dependency for getting a tasks database session. # @PRE TasksSessionLocal is initialized. # @POST Session is closed after use. # @RELATION DEPENDS_ON -> [TasksSessionLocal] def get_tasks_db(): with belief_scope("get_tasks_db"): db = TasksSessionLocal() try: yield db finally: db.close() # #endregion get_tasks_db # #region get_auth_db [C:3] [TYPE Function] # @BRIEF Dependency for getting an authentication database session. # @PRE AuthSessionLocal is initialized. # @POST Session is closed after use. # @DATA_CONTRACT None -> Output[EXT:Library:sqlalchemy.orm.Session] # @RELATION DEPENDS_ON -> [AuthSessionLocal] def get_auth_db(): with belief_scope("get_auth_db"): db = AuthSessionLocal() try: yield db finally: db.close() # #endregion get_auth_db # #endregion DatabaseModule