busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

semantic

Browse Source
This commit is contained in:
busya
2026-05-20 23:54:53 +03:00
parent a43886106e
commit 68740cd9ed
167 changed files with 10620 additions and 29041 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/core/__tests__/test_superset_preview_pipeline.py
View File

@@ -3,10 +3,10 @@
# @LAYER: Domain
# @RELATION [BINDS_TO] ->[AsyncNetworkModule]
import json
import pytest
from unittest.mock import MagicMock
import httpx
import pytest
import requests
from src.core.config_models import Environment

5
backend/src/core/__tests__/test_superset_profile_lookup.py
View File

@@ -4,11 +4,10 @@
# @LAYER: Domain
# [SECTION: IMPORTS]
import json
import sys
from pathlib import Path
from typing import Any
import pytest
import sys
from typing import Any
backend_dir = str(Path(__file__).parent.parent.parent.parent.resolve())
if backend_dir not in sys.path:

11
backend/src/core/async_superset_client.py
View File

@@ -7,8 +7,7 @@ import re
from typing import Any, cast
from .config_models import Environment
from .logger import belief_scope
from .logger import logger as app_logger
from .logger import belief_scope, logger as app_logger
from .superset_client import SupersetClient
from .utils.async_network import AsyncAPIClient
@@ -263,7 +262,7 @@ class AsyncSupersetClient(SupersetClient):
self._extract_chart_ids_from_layout(parsed_position)
)
except Exception:
pass
app_logger.debug("Could not parse position JSON for dashboard %s", dashboard_ref)
elif isinstance(raw_position_json, dict):
chart_ids_from_position.update(
self._extract_chart_ids_from_layout(raw_position_json)
@@ -276,7 +275,7 @@ class AsyncSupersetClient(SupersetClient):
self._extract_chart_ids_from_layout(parsed_metadata)
)
except Exception:
pass
app_logger.debug("Could not parse json_metadata for dashboard %s", dashboard_ref)
elif isinstance(raw_json_metadata, dict):
chart_ids_from_position.update(
self._extract_chart_ids_from_layout(raw_json_metadata)
@@ -548,7 +547,7 @@ class AsyncSupersetClient(SupersetClient):
result["filters"] = filter_data
return result
except ValueError:
pass
app_logger.debug("Permalink key not found in URL for dashboard %s", dashboard_ref)
# Check for native_filters_key in query params
native_filters_key = query_params.get("native_filters_key", [None])[0]
if native_filters_key:
@@ -561,7 +560,7 @@ class AsyncSupersetClient(SupersetClient):
if potential_id not in ("p", "list", "new"):
dashboard_ref = potential_id
except ValueError:
pass
app_logger.debug("Dashboard not found in path_parts for native_filters_key")
if dashboard_ref:
# Resolve slug to numeric ID — the filter_state API requires a numeric ID
resolved_id = None

3
backend/src/core/auth/__tests__/test_auth.py
View File

@@ -2,12 +2,13 @@
# @BRIEF Unit tests for authentication module
# @LAYER: Domain
# @RELATION VERIFIES -> AuthPackage
import sys
from pathlib import Path
import sys
# Add src to path
sys.path.append(str(Path(__file__).parent.parent.parent.parent / "src"))
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

41
backend/src/core/auth/config.py
View File

@@ -4,9 +4,16 @@
# @LAYER: Core
# @RELATION DEPENDS_ON -> pydantic
#
# @INVARIANT: All sensitive configuration must have defaults or be loaded from environment.
# @INVARIANT: All sensitive configuration must be loaded from environment; no hardcoded secrets.
# @RATIONALE SECRET_KEY and AUTH_DATABASE_URL now crash-early if env vars are missing.
# Dev fallback for AUTH_DATABASE_URL only when DEV_MODE=true.
# @REJECTED Default secrets in source code rejected — Class 1 security violation:
# "super-secret-key-change-in-production" and "postgres:postgres" exposed
# secrets in version control.
from pydantic import Field
import os
from pydantic import Field, field_validator
from pydantic_settings import BaseSettings
@@ -17,22 +24,42 @@ from pydantic_settings import BaseSettings
# @RELATION INHERITS -> pydantic_settings.BaseSettings
class AuthConfig(BaseSettings):
# JWT Settings
SECRET_KEY: str = Field(default="super-secret-key-change-in-production", env="AUTH_SECRET_KEY")
SECRET_KEY: str = Field(default="", env="AUTH_SECRET_KEY")
ALGORITHM: str = "HS256"
ACCESS_TOKEN_EXPIRE_MINUTES: int = 480
REFRESH_TOKEN_EXPIRE_DAYS: int = 7
# Database Settings
AUTH_DATABASE_URL: str = Field(
default="postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools",
env="AUTH_DATABASE_URL",
)
AUTH_DATABASE_URL: str = Field(default="", env="AUTH_DATABASE_URL")
# ADFS Settings
ADFS_CLIENT_ID: str = Field(default="", env="ADFS_CLIENT_ID")
ADFS_CLIENT_SECRET: str = Field(default="", env="ADFS_CLIENT_SECRET")
ADFS_METADATA_URL: str = Field(default="", env="ADFS_METADATA_URL")
@field_validator("SECRET_KEY", mode="after")
@classmethod
def validate_secret_key(cls, v: str) -> str:
if v:
return v
raise ValueError(
"AUTH_SECRET_KEY environment variable is required. "
"Set it in .env or export it before starting the server."
)
@field_validator("AUTH_DATABASE_URL", mode="after")
@classmethod
def validate_auth_db_url(cls, v: str) -> str:
if v:
return v
is_dev = os.getenv("DEV_MODE", "").strip().lower() in {"1", "true", "yes"}
if is_dev:
return "postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools"
raise ValueError(
"AUTH_DATABASE_URL environment variable is required. "
"Set it in .env or export it before starting the server."
)
class Config:
env_file = ".env"
extra = "ignore"

4
backend/src/core/cot_logger.py
View File

@@ -10,10 +10,10 @@
# @SIDE_EFFECT: Writes structured JSON to the 'cot' Python logger.
# @DATA_CONTRACT: Log call -> Single-line JSON to logging.StreamHandler/file.
import logging
import uuid
from contextvars import ContextVar
import logging
from typing import Any
import uuid
# #region cot_trace_context [C:1] [TYPE Data] [SEMANTICS contextvar, trace_id, span_id, propagation]
# @BRIEF ContextVars for trace ID and span ID propagation across async boundaries.

42
backend/src/core/database.py
View File

@@ -14,17 +14,18 @@ from pathlib import Path
from sqlalchemy import create_engine, inspect, text
from sqlalchemy.orm import sessionmaker
from ..models import assistant as _assistant_models # noqa: F401
from ..models import auth as _auth_models # noqa: F401
from ..models import clean_release as _clean_release_models # noqa: F401
from ..models import config as _config_models # noqa: F401
from ..models import connection as _connection_models # noqa: F401
from ..models import dataset_review as _dataset_review_models # noqa: F401
from ..models import llm as _llm_models # noqa: F401
from ..models import profile as _profile_models # noqa: F401
# Import models to ensure they're registered with Base
from ..models import task as _task_models # noqa: F401
from ..models import (
assistant as _assistant_models, # noqa: F401
auth as _auth_models, # noqa: F401
clean_release as _clean_release_models, # noqa: F401
config as _config_models, # noqa: F401
connection as _connection_models, # noqa: F401
dataset_review as _dataset_review_models, # noqa: F401
llm as _llm_models, # noqa: F401
profile as _profile_models, # noqa: F401
task as _task_models, # noqa: F401
)
from ..models.connection import ConnectionConfig
from ..models.mapping import Base
from .auth.config import auth_config
@@ -36,12 +37,21 @@ BASE_DIR = Path(__file__).resolve().parent.parent.parent
# #endregion BASE_DIR
# #region DATABASE_URL [C:1] [TYPE Constant]
# @BRIEF URL for the main application database.
DEFAULT_POSTGRES_URL = os.getenv(
"POSTGRES_URL",
"postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools",
)
DATABASE_URL = os.getenv("DATABASE_URL", DEFAULT_POSTGRES_URL)
# @BRIEF URL for the main application database. Read from env; dev fallback only.
# @RATIONALE DATABASE_URL reads from env (DATABASE_URL or POSTGRES_URL).
# Crashes at import if unset in production; provides localhost fallback in dev.
# @REJECTED Hardcoded postgres:postgres@localhost in source code rejected — exposes
# database credentials in version control (Class 1 security violation).
_DATABASE_URL = os.getenv("DATABASE_URL") or os.getenv("POSTGRES_URL")
if _DATABASE_URL:
DATABASE_URL = _DATABASE_URL
elif os.getenv("DEV_MODE", "").strip().lower() in {"1", "true", "yes"}:
DATABASE_URL = "postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools"
else:
raise RuntimeError(
"DATABASE_URL (or POSTGRES_URL) environment variable is required. "
"Set it before starting the server."
)
# #endregion DATABASE_URL
# #region TASKS_DATABASE_URL [C:1] [TYPE Constant]

737
backend/src/core/database.py.bak
View File

@@ -1,737 +0,0 @@
# [DEF:DatabaseModule:Module]
#
# @COMPLEXITY: 3
# @SEMANTICS: database, postgresql, sqlalchemy, session, persistence
# @PURPOSE: Configures database connection and session management (PostgreSQL-first).
# @LAYER: Core
# @RELATION: [DEPENDS_ON] ->[MappingModels]
# @RELATION: [DEPENDS_ON] ->[auth_config]
# @RELATION: [DEPENDS_ON] ->[ConnectionConfig]
#
# @INVARIANT: A single engine instance is used for the entire application.
# [SECTION: IMPORTS]
from sqlalchemy import create_engine, inspect, text
from sqlalchemy.orm import sessionmaker
from ..models.mapping import Base
from ..models.connection import ConnectionConfig
# Import models to ensure they're registered with Base
from ..models import task as _task_models # noqa: F401
from ..models import auth as _auth_models # noqa: F401
from ..models import config as _config_models # noqa: F401
from ..models import llm as _llm_models # noqa: F401
from ..models import assistant as _assistant_models # noqa: F401
from ..models import profile as _profile_models # noqa: F401
from ..models import clean_release as _clean_release_models # noqa: F401
from ..models import connection as _connection_models # noqa: F401
from ..models import dataset_review as _dataset_review_models # noqa: F401
from .logger import belief_scope, logger
from .auth.config import auth_config
import os
from pathlib import Path
# [/SECTION]
# [DEF:BASE_DIR:Variable]
# @COMPLEXITY: 1
# @PURPOSE: Base directory for the backend.
BASE_DIR = Path(__file__).resolve().parent.parent.parent
# [/DEF:BASE_DIR:Variable]
# [DEF:DATABASE_URL:Constant]
# @COMPLEXITY: 1
# @PURPOSE: URL for the main application database.
DEFAULT_POSTGRES_URL = os.getenv(
"POSTGRES_URL",
"postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools",
)
DATABASE_URL = os.getenv("DATABASE_URL", DEFAULT_POSTGRES_URL)
# [/DEF:DATABASE_URL:Constant]
# [DEF:TASKS_DATABASE_URL:Constant]
# @COMPLEXITY: 1
# @PURPOSE: URL for the tasks execution database.
# Defaults to DATABASE_URL to keep task logs in the same PostgreSQL instance.
TASKS_DATABASE_URL = os.getenv("TASKS_DATABASE_URL", DATABASE_URL)
# [/DEF:TASKS_DATABASE_URL:Constant]
# [DEF:AUTH_DATABASE_URL:Constant]
# @COMPLEXITY: 1
# @PURPOSE: URL for the authentication database.
AUTH_DATABASE_URL = os.getenv("AUTH_DATABASE_URL", auth_config.AUTH_DATABASE_URL)
# [/DEF:AUTH_DATABASE_URL:Constant]
# [DEF:engine:Variable]
# @COMPLEXITY: 1
# @PURPOSE: SQLAlchemy engine for mappings database.
# @SIDE_EFFECT: Creates database engine and manages connection pool.
def _build_engine(db_url: str):
with belief_scope("_build_engine"):
if db_url.startswith("sqlite"):
return create_engine(db_url, connect_args={"check_same_thread": False})
return create_engine(db_url, pool_pre_ping=True)
engine = _build_engine(DATABASE_URL)
# [/DEF:engine:Variable]
# [DEF:tasks_engine:Variable]
# @COMPLEXITY: 1
# @PURPOSE: SQLAlchemy engine for tasks database.
tasks_engine = _build_engine(TASKS_DATABASE_URL)
# [/DEF:tasks_engine:Variable]
# [DEF:auth_engine:Variable]
# @COMPLEXITY: 1
# @PURPOSE: SQLAlchemy engine for authentication database.
auth_engine = _build_engine(AUTH_DATABASE_URL)
# [/DEF:auth_engine:Variable]
# [DEF:SessionLocal:Class]
# @COMPLEXITY: 1
# @PURPOSE: A session factory for the main mappings database.
# @PRE: engine is initialized.
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
# [/DEF:SessionLocal:Class]
# [DEF:TasksSessionLocal:Class]
# @COMPLEXITY: 1
# @PURPOSE: A session factory for the tasks execution database.
# @PRE: tasks_engine is initialized.
TasksSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=tasks_engine)
# [/DEF:TasksSessionLocal:Class]
# [DEF:AuthSessionLocal:Class]
# @COMPLEXITY: 1
# @PURPOSE: A session factory for the authentication database.
# @PRE: auth_engine is initialized.
AuthSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=auth_engine)
# [/DEF:AuthSessionLocal:Class]
# [DEF:_ensure_user_dashboard_preferences_columns:Function]
# @COMPLEXITY: 3
# @PURPOSE: Applies additive schema upgrades for user_dashboard_preferences table.
# @PRE: bind_engine points to application database where profile table is stored.
# @POST: Missing columns are added without data loss.
# @RELATION: [DEPENDS_ON] ->[engine]
def _ensure_user_dashboard_preferences_columns(bind_engine):
with belief_scope("_ensure_user_dashboard_preferences_columns"):
table_name = "user_dashboard_preferences"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {
str(column.get("name") or "").strip()
for column in inspector.get_columns(table_name)
}
alter_statements = []
if "git_username" not in existing_columns:
alter_statements.append(
"ALTER TABLE user_dashboard_preferences ADD COLUMN git_username VARCHAR"
)
if "git_email" not in existing_columns:
alter_statements.append(
"ALTER TABLE user_dashboard_preferences ADD COLUMN git_email VARCHAR"
)
if "git_personal_access_token_encrypted" not in existing_columns:
alter_statements.append(
"ALTER TABLE user_dashboard_preferences "
"ADD COLUMN git_personal_access_token_encrypted VARCHAR"
)
if "start_page" not in existing_columns:
alter_statements.append(
"ALTER TABLE user_dashboard_preferences "
"ADD COLUMN start_page VARCHAR NOT NULL DEFAULT 'dashboards'"
)
if "auto_open_task_drawer" not in existing_columns:
alter_statements.append(
"ALTER TABLE user_dashboard_preferences "
"ADD COLUMN auto_open_task_drawer BOOLEAN NOT NULL DEFAULT TRUE"
)
if "dashboards_table_density" not in existing_columns:
alter_statements.append(
"ALTER TABLE user_dashboard_preferences "
"ADD COLUMN dashboards_table_density VARCHAR NOT NULL DEFAULT 'comfortable'"
)
if "show_only_slug_dashboards" not in existing_columns:
alter_statements.append(
"ALTER TABLE user_dashboard_preferences "
"ADD COLUMN show_only_slug_dashboards BOOLEAN NOT NULL DEFAULT TRUE"
)
if not alter_statements:
return
try:
with bind_engine.begin() as connection:
for statement in alter_statements:
connection.execute(text(statement))
except Exception as migration_error:
logger.warning(
"[database][EXPLORE] Profile preference additive migration failed: %s",
migration_error,
)
# [/DEF:_ensure_user_dashboard_preferences_columns:Function]
# [DEF:_ensure_user_dashboard_preferences_health_columns:Function]
# @COMPLEXITY: 3
# @PURPOSE: Applies additive schema upgrades for user_dashboard_preferences table (health fields).
# @RELATION: [DEPENDS_ON] ->[engine]
def _ensure_user_dashboard_preferences_health_columns(bind_engine):
with belief_scope("_ensure_user_dashboard_preferences_health_columns"):
table_name = "user_dashboard_preferences"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {
str(column.get("name") or "").strip()
for column in inspector.get_columns(table_name)
}
alter_statements = []
if "telegram_id" not in existing_columns:
alter_statements.append(
"ALTER TABLE user_dashboard_preferences ADD COLUMN telegram_id VARCHAR"
)
if "email_address" not in existing_columns:
alter_statements.append(
"ALTER TABLE user_dashboard_preferences ADD COLUMN email_address VARCHAR"
)
if "notify_on_fail" not in existing_columns:
alter_statements.append(
"ALTER TABLE user_dashboard_preferences ADD COLUMN notify_on_fail BOOLEAN NOT NULL DEFAULT TRUE"
)
if not alter_statements:
return
try:
with bind_engine.begin() as connection:
for statement in alter_statements:
connection.execute(text(statement))
except Exception as migration_error:
logger.warning(
"[database][EXPLORE] Profile health preference additive migration failed: %s",
migration_error,
)
# [/DEF:_ensure_user_dashboard_preferences_health_columns:Function]
# [DEF:_ensure_llm_validation_results_columns:Function]
# @COMPLEXITY: 3
# @PURPOSE: Applies additive schema upgrades for llm_validation_results table.
# @RELATION: [DEPENDS_ON] ->[engine]
def _ensure_llm_validation_results_columns(bind_engine):
with belief_scope("_ensure_llm_validation_results_columns"):
table_name = "llm_validation_results"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {
str(column.get("name") or "").strip()
for column in inspector.get_columns(table_name)
}
alter_statements = []
if "task_id" not in existing_columns:
alter_statements.append(
"ALTER TABLE llm_validation_results ADD COLUMN task_id VARCHAR"
)
if "environment_id" not in existing_columns:
alter_statements.append(
"ALTER TABLE llm_validation_results ADD COLUMN environment_id VARCHAR"
)
if not alter_statements:
return
try:
with bind_engine.begin() as connection:
for statement in alter_statements:
connection.execute(text(statement))
except Exception as migration_error:
logger.warning(
"[database][EXPLORE] ValidationRecord additive migration failed: %s",
migration_error,
)
# [/DEF:_ensure_llm_validation_results_columns:Function]
# [DEF:_ensure_git_server_configs_columns:Function]
# @COMPLEXITY: 3
# @PURPOSE: Applies additive schema upgrades for git_server_configs table.
# @PRE: bind_engine points to application database.
# @POST: Missing columns are added without data loss.
# @RELATION: [DEPENDS_ON] ->[engine]
def _ensure_git_server_configs_columns(bind_engine):
with belief_scope("_ensure_git_server_configs_columns"):
table_name = "git_server_configs"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {
str(column.get("name") or "").strip()
for column in inspector.get_columns(table_name)
}
alter_statements = []
if "default_branch" not in existing_columns:
alter_statements.append(
"ALTER TABLE git_server_configs ADD COLUMN default_branch VARCHAR NOT NULL DEFAULT 'main'"
)
if not alter_statements:
return
try:
with bind_engine.begin() as connection:
for statement in alter_statements:
connection.execute(text(statement))
except Exception as migration_error:
logger.warning(
"[database][EXPLORE] GitServerConfig preference additive migration failed: %s",
migration_error,
)
# [/DEF:_ensure_git_server_configs_columns:Function]
# [DEF:_ensure_auth_users_columns:Function]
# @COMPLEXITY: 3
# @PURPOSE: Applies additive schema upgrades for auth users table.
# @PRE: bind_engine points to authentication database.
# @POST: Missing columns are added without data loss.
# @RELATION: [DEPENDS_ON] ->[auth_engine]
def _ensure_auth_users_columns(bind_engine):
with belief_scope("_ensure_auth_users_columns"):
table_name = "users"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {
str(column.get("name") or "").strip()
for column in inspector.get_columns(table_name)
}
alter_statements = []
if "full_name" not in existing_columns:
alter_statements.append("ALTER TABLE users ADD COLUMN full_name VARCHAR")
if "is_ad_user" not in existing_columns:
alter_statements.append(
"ALTER TABLE users ADD COLUMN is_ad_user BOOLEAN NOT NULL DEFAULT FALSE"
)
if not alter_statements:
logger.reason(
"Auth users schema already up to date",
extra={"table": table_name, "columns": sorted(existing_columns)},
)
return
logger.reason(
"Applying additive auth users schema migration",
extra={"table": table_name, "statements": alter_statements},
)
try:
with bind_engine.begin() as connection:
for statement in alter_statements:
connection.execute(text(statement))
logger.reason(
"Auth users schema migration completed",
extra={
"table": table_name,
"added_columns": [
stmt.split(" ADD COLUMN ", 1)[1].split()[0]
for stmt in alter_statements
],
},
)
except Exception as migration_error:
logger.warning(
"[database][EXPLORE] Auth users additive migration failed: %s",
migration_error,
)
raise
# [/DEF:_ensure_auth_users_columns:Function]
# [DEF:ensure_connection_configs_table:Function]
# @COMPLEXITY: 3
# @PURPOSE: Ensures the external connection registry table exists in the main database.
# @PRE: bind_engine points to the application database.
# @POST: connection_configs table exists without dropping existing data.
# @RELATION: [DEPENDS_ON] ->[ConnectionConfig]
def ensure_connection_configs_table(bind_engine):
with belief_scope("ensure_connection_configs_table"):
try:
ConnectionConfig.__table__.create(bind=bind_engine, checkfirst=True)
except Exception as migration_error:
logger.warning(
"[database][EXPLORE] ConnectionConfig table ensure failed: %s",
migration_error,
)
raise
# [/DEF:ensure_connection_configs_table:Function]
# [DEF:_ensure_filter_source_enum_values:Function]
# @COMPLEXITY: 3
# @PURPOSE: Adds missing FilterSource enum values to the PostgreSQL native filtersource type.
# @PRE: bind_engine points to application database with imported_filters table.
# @POST: New enum values are available without data loss.
# @RELATION: [DEPENDS_ON] ->[engine]
def _ensure_filter_source_enum_values(bind_engine):
with belief_scope("_ensure_filter_source_enum_values"):
try:
with bind_engine.connect() as connection:
# Check if the native enum type exists
result = connection.execute(
text(
"SELECT t.typname FROM pg_type t "
"JOIN pg_namespace n ON t.typnamespace = n.oid "
"WHERE t.typname = 'filtersource' AND n.nspname = 'public'"
)
)
if result.fetchone() is None:
logger.reason(
"filtersource enum type does not exist yet; skipping migration"
)
return
# Get existing enum values
result = connection.execute(
text(
"SELECT e.enumlabel FROM pg_enum e "
"JOIN pg_type t ON e.enumtypid = t.oid "
"WHERE t.typname = 'filtersource' "
"ORDER BY e.enumsortorder"
)
)
existing_values = {row[0] for row in result.fetchall()}
required_values = ["SUPERSET_PERMALINK", "SUPERSET_NATIVE_FILTERS_KEY"]
missing_values = [
v for v in required_values if v not in existing_values
]
if not missing_values:
logger.reason(
"filtersource enum already up to date",
extra={"existing": sorted(existing_values)},
)
return
logger.reason(
"Adding missing values to filtersource enum",
extra={"missing": missing_values},
)
for value in missing_values:
connection.execute(
text(
f"ALTER TYPE filtersource ADD VALUE IF NOT EXISTS '{value}'"
)
)
connection.commit()
logger.reason(
"filtersource enum migration completed",
extra={"added": missing_values},
)
except Exception as migration_error:
logger.warning(
"[database][EXPLORE] FilterSource enum additive migration failed: %s",
migration_error,
)
# [/DEF:_ensure_filter_source_enum_values:Function]
# [DEF:_ensure_translation_jobs_columns:Function]
# @COMPLEXITY: 3
# @PURPOSE: Apply additive schema upgrades for translation_jobs table.
# @PRE: bind_engine points to the application database where translation_jobs table is stored.
# @POST: Missing additive columns (environment_id, target_database_id) across translation_jobs are created without removing existing data.
# @SIDE_EFFECT: Executes ALTER TABLE statements against translation_jobs table.
# @RELATION: [DEPENDS_ON] ->[engine]
def _ensure_translation_jobs_columns(bind_engine):
with belief_scope("_ensure_translation_jobs_columns"):
table_name = "translation_jobs"
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
return
existing_columns = {
str(column.get("name") or "").strip()
for column in inspector.get_columns(table_name)
}
if "environment_id" not in existing_columns:
try:
with bind_engine.begin() as connection:
connection.execute(
text(
"ALTER TABLE translation_jobs "
"ADD COLUMN environment_id VARCHAR"
)
)
logger.reflect(
"Added environment_id column to translation_jobs",
)
except Exception as migration_error:
logger.explore(
"Failed to add environment_id to translation_jobs",
extra={"error": str(migration_error)},
)
raise
if "target_database_id" not in existing_columns:
try:
with bind_engine.begin() as connection:
connection.execute(
text(
"ALTER TABLE translation_jobs "
"ADD COLUMN target_database_id VARCHAR"
)
)
logger.reflect(
"Added target_database_id column to translation_jobs",
)
except Exception as migration_error:
logger.explore(
"Failed to add target_database_id to translation_jobs",
extra={"error": str(migration_error)},
)
raise
# [/DEF:_ensure_translation_jobs_columns:Function]
# [DEF:_ensure_translation_records_columns:Function]
# @COMPLEXITY: 3
# @PURPOSE: Add source_data JSON column to translation_records and translation_preview_records.
# @PRE: bind_engine points to the application database.
# @POST: source_data column exists on both tables.
# @SIDE_EFFECT: Executes ALTER TABLE statements.
# @RELATION: [DEPENDS_ON] ->[engine]
def _ensure_translation_records_columns(bind_engine):
with belief_scope("_ensure_translation_records_columns"):
migrations = [
("translation_records", "source_data",
"ALTER TABLE translation_records ADD COLUMN source_data JSON"),
("translation_preview_records", "source_data",
"ALTER TABLE translation_preview_records ADD COLUMN source_data JSON"),
]
for table_name, column_name, alter_sql in migrations:
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
continue
existing_columns = {
str(c.get("name") or "").strip()
for c in inspector.get_columns(table_name)
}
if column_name not in existing_columns:
try:
with bind_engine.begin() as connection:
connection.execute(text(alter_sql))
logger.reflect(
f"Added {column_name} column to {table_name}",
)
except Exception as migration_error:
logger.explore(
f"Failed to add {column_name} to {table_name}",
extra={"error": str(migration_error)},
)
raise
# [/DEF:_ensure_translation_records_columns:Function]
# [DEF:_ensure_dataset_review_session_columns:Function]
# @COMPLEXITY: 3
# @PURPOSE: Apply additive schema upgrades for dataset review persistence required by optimistic-lock and recovery metadata semantics.
# @PRE: bind_engine points to the application database where dataset review tables are stored.
# @POST: Missing additive columns across legacy dataset review tables are created without removing existing data.
# @SIDE_EFFECT: Executes ALTER TABLE statements against dataset review tables in the application database.
# @RELATION: [DEPENDS_ON] ->[DatasetReviewSession]
# @RELATION: [DEPENDS_ON] ->[ImportedFilter]
def _ensure_dataset_review_session_columns(bind_engine):
with belief_scope("_ensure_dataset_review_session_columns"):
inspector = inspect(bind_engine)
existing_tables = set(inspector.get_table_names())
migration_plan = {
"dataset_review_sessions": [
(
"version",
"ALTER TABLE dataset_review_sessions "
"ADD COLUMN version INTEGER NOT NULL DEFAULT 0",
)
],
"imported_filters": [
(
"raw_value_masked",
"ALTER TABLE imported_filters "
"ADD COLUMN raw_value_masked BOOLEAN NOT NULL DEFAULT FALSE",
)
],
}
for table_name, planned_columns in migration_plan.items():
if table_name not in existing_tables:
logger.reason(
"Dataset review table does not exist yet; skipping additive schema migration",
extra={"table": table_name},
)
continue
existing_columns = {
str(column.get("name") or "").strip()
for column in inspector.get_columns(table_name)
}
alter_statements = [
statement
for column_name, statement in planned_columns
if column_name not in existing_columns
]
if not alter_statements:
logger.reason(
"Dataset review table schema already up to date",
extra={"table": table_name, "columns": sorted(existing_columns)},
)
continue
logger.reason(
"Applying additive dataset review schema migration",
extra={"table": table_name, "statements": alter_statements},
)
try:
with bind_engine.begin() as connection:
for statement in alter_statements:
connection.execute(text(statement))
logger.reflect(
"Dataset review additive schema migration completed",
extra={
"table": table_name,
"added_columns": [
stmt.split(" ADD COLUMN ", 1)[1].split()[0]
for stmt in alter_statements
],
},
)
except Exception as migration_error:
logger.explore(
"Dataset review additive schema migration failed",
extra={"table": table_name, "error": str(migration_error)},
)
raise
# [/DEF:_ensure_dataset_review_session_columns:Function]
# [DEF:init_db:Function]
# @COMPLEXITY: 3
# @PURPOSE: Initializes the database by creating all tables.
# @PRE: engine, tasks_engine and auth_engine are initialized.
# @POST: Database tables created in all databases.
# @SIDE_EFFECT: Creates physical database files if they don't exist.
# @RELATION: [CALLS] ->[ensure_connection_configs_table]
# @RELATION: [CALLS] ->[_ensure_filter_source_enum_values]
# @RELATION: [CALLS] ->[_ensure_dataset_review_session_columns]
def init_db():
with belief_scope("init_db"):
Base.metadata.create_all(bind=engine)
Base.metadata.create_all(bind=tasks_engine)
Base.metadata.create_all(bind=auth_engine)
_ensure_user_dashboard_preferences_columns(engine)
_ensure_llm_validation_results_columns(engine)
_ensure_user_dashboard_preferences_health_columns(engine)
_ensure_git_server_configs_columns(engine)
_ensure_auth_users_columns(auth_engine)
ensure_connection_configs_table(engine)
_ensure_filter_source_enum_values(engine)
_ensure_dataset_review_session_columns(engine)
_ensure_translation_jobs_columns(engine)
_ensure_translation_records_columns(engine)
# [/DEF:init_db:Function]
# [DEF:get_db:Function]
# @COMPLEXITY: 3
# @PURPOSE: Dependency for getting a database session.
# @PRE: SessionLocal is initialized.
# @POST: Session is closed after use.
# @RETURN: Generator[Session, None, None]
# @RELATION: [DEPENDS_ON] ->[SessionLocal]
def get_db():
with belief_scope("get_db"):
db = SessionLocal()
try:
yield db
finally:
db.close()
# [/DEF:get_db:Function]
# [DEF:get_tasks_db:Function]
# @COMPLEXITY: 3
# @PURPOSE: Dependency for getting a tasks database session.
# @PRE: TasksSessionLocal is initialized.
# @POST: Session is closed after use.
# @RETURN: Generator[Session, None, None]
# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]
def get_tasks_db():
with belief_scope("get_tasks_db"):
db = TasksSessionLocal()
try:
yield db
finally:
db.close()
# [/DEF:get_tasks_db:Function]
# [DEF:get_auth_db:Function]
# @COMPLEXITY: 3
# @PURPOSE: Dependency for getting an authentication database session.
# @PRE: AuthSessionLocal is initialized.
# @POST: Session is closed after use.
# @DATA_CONTRACT: None -> Output[sqlalchemy.orm.Session]
# @RETURN: Generator[Session, None, None]
# @RELATION: [DEPENDS_ON] ->[AuthSessionLocal]
def get_auth_db():
with belief_scope("get_auth_db"):
db = AuthSessionLocal()
try:
yield db
finally:
db.close()
# [/DEF:get_auth_db:Function]
# [/DEF:DatabaseModule:Module]

20
backend/src/core/encryption_key.py
View File

@@ -1,12 +1,13 @@
# #region EncryptionKeyModule [C:5] [TYPE Module] [SEMANTICS encryption, fernet, key, env, secret]
# @BRIEF Resolve and persist the Fernet encryption key required by runtime services.
# @LAYER: Infra
# @LAYER Infra
# @RELATION DEPENDS_ON -> [LoggerModule]
# @INVARIANT: Runtime key resolution never falls back to an ephemeral secret.
# @PRE: Runtime environment can read process variables and target .env path is writable when key generation is required.
# @POST: A valid Fernet key is available to runtime services via ENCRYPTION_KEY.
# @SIDE_EFFECT: May append ENCRYPTION_KEY entry into backend .env file and set process environment variable.
# @DATA_CONTRACT: Input[env_file_path] -> Output[encryption_key]
# @INVARIANT Runtime key resolution never falls back to an ephemeral secret.
# @PRE Runtime environment can read process variables and target .env path is writable when key generation is required.
# @POST A valid Fernet key is available to runtime services via ENCRYPTION_KEY.
# @SIDE_EFFECT May append ENCRYPTION_KEY entry into backend .env file and set process environment variable.
# @DATA_CONTRACT Input[env_file_path] -> Output[encryption_key]
# @RATIONALE Replaced Path(__file__).parents[2] with BASE_DIR import from database.py — same result, avoids recomputing path traversal.
from __future__ import annotations
@@ -22,9 +23,9 @@ DEFAULT_ENV_FILE_PATH = Path(__file__).resolve().parents[2] / ".env"
# #region ensure_encryption_key [TYPE Function]
# @BRIEF Ensure backend runtime has a persistent valid Fernet key.
# @PRE: env_file_path points to a writable backend .env file or ENCRYPTION_KEY exists in process environment.
# @POST: Returns a valid Fernet key and guarantees it is present in process environment.
# @SIDE_EFFECT: May create or append backend/.env when key is missing.
# @PRE env_file_path points to a writable backend .env file or ENCRYPTION_KEY exists in process environment.
# @POST Returns a valid Fernet key and guarantees it is present in process environment.
# @SIDE_EFFECT May create or append backend/.env when key is missing.
def ensure_encryption_key(env_file_path: Path = DEFAULT_ENV_FILE_PATH) -> str:
with belief_scope("ensure_encryption_key", f"env_file_path={env_file_path}"):
existing_key = os.getenv("ENCRYPTION_KEY", "").strip()
@@ -58,3 +59,4 @@ def ensure_encryption_key(env_file_path: Path = DEFAULT_ENV_FILE_PATH) -> str:
# #endregion ensure_encryption_key
# #endregion EncryptionKeyModule
# #endregion EncryptionKeyModule

11
backend/src/core/logger.py
View File

@@ -11,19 +11,18 @@
# @REJECTED Keeping both formatters side-by-side was rejected (two inconsistent output formats would confuse log consumers).
# @DATA_CONTRACT All log records conform to molecular CoT JSON schema: {ts, level, trace_id, src, marker, intent, span_id?, payload?, error?}
# @INVARIANT CotJsonFormatter.format() always returns valid single-line JSON string.
import json
import logging
import threading
import types
from contextlib import contextmanager
from datetime import UTC, datetime
import json
import logging
from logging.handlers import RotatingFileHandler
import threading
import types
from typing import Any
from pydantic import BaseModel, Field
from .cot_logger import _span_id, _trace_id
from .cot_logger import log as cot_log
from .cot_logger import _span_id, _trace_id, log as cot_log
from .ws_log_handler import WebSocketLogHandler
# Thread-local storage for belief state

5
backend/src/core/logger/__tests__/test_logger.py
View File

@@ -2,13 +2,12 @@
# @BRIEF Unit tests for logger module
# @LAYER: Infra
# @RELATION VERIFIES -> src.core.logger
import sys
from pathlib import Path
import sys
# Add src to path
sys.path.append(str(Path(__file__).parent.parent.parent.parent / "src"))
import logging
import pytest
from src.core.config_models import LoggingConfig
@@ -218,8 +217,8 @@ def test_configure_logger_post_conditions(tmp_path):
import logging
from logging.handlers import RotatingFileHandler
import src.core.logger as logger_module
from src.core.config_models import LoggingConfig
import src.core.logger as logger_module
from src.core.logger import BeliefFormatter, configure_logger, get_task_log_level, logger
log_file = tmp_path / "test.log"
config = LoggingConfig(

2
backend/src/core/mapping_service.py
View File

@@ -271,7 +271,7 @@ class IdMappingService:
try:
result[m.uuid] = int(m.remote_integer_id)
except ValueError:
pass
logger.debug("Could not parse remote_integer_id for mapping %s (uuid=%s)", m.id, m.uuid)
return result
# #endregion get_remote_ids_batch
# #endregion IdMappingService

4
backend/src/core/migration/archive_parser.py
View File

@@ -8,10 +8,10 @@
# @SIDE_EFFECT: Reads archive file (read-only)
# @DATA_CONTRACT: ArchivePath -> ParsedMigration
import json
import tempfile
import zipfile
from pathlib import Path
import tempfile
from typing import Any
import zipfile
import yaml

2
backend/src/core/migration/dry_run_orchestrator.py
View File

@@ -10,8 +10,8 @@
# @POST: Dry-run diff returned without mutation
# @SIDE_EFFECT: Reads source environment (read-only)
# @DATA_CONTRACT: EnvironmentConfig -> MigrationDiffReport
import json
from datetime import UTC, datetime
import json
from typing import Any
from sqlalchemy.orm import Session

4
backend/src/core/migration_engine.py
View File

@@ -13,10 +13,10 @@
# @INVARIANT: ZIP structure and non-targeted metadata must remain valid after transformation.
import json
import os
from pathlib import Path
import re
import tempfile
import zipfile
from pathlib import Path
import yaml
@@ -179,7 +179,7 @@ class MigrationEngine:
if cdata and "id" in cdata and "uuid" in cdata:
mapping[cdata["id"]] = cdata["uuid"]
except Exception:
pass
logger.debug("Could not parse chart YAML file %s", cf)
return mapping
# #endregion _extract_chart_uuids_from_archive
# #region _patch_dashboard_metadata [TYPE Function]

19
backend/src/core/plugin_loader.py
View File

@@ -2,7 +2,7 @@ import importlib.util
import os
import sys # Added this line
from .logger import belief_scope
from .logger import belief_scope, logger as _logger
from .plugin_base import PluginBase, PluginConfig
@@ -10,6 +10,8 @@ from .plugin_base import PluginBase, PluginConfig
# @BRIEF Scans a specified directory for Python modules, dynamically loads them, and registers any classes that are valid implementations of the PluginBase interface.
# @LAYER: Core
# @RELATION Depends on PluginBase. It is used by the main application to discover and manage available plugins.
# @RATIONALE Replaced print() with _logger calls to eliminate silent failures. Added top-level logger import, removed late imports.
# @REJECTED Keeping print() statements with "Replace with proper logging" comments was rejected — they produce no structured output and cannot be filtered by log level.
class PluginLoader:
"""
Scans a directory for Python modules, loads them, and identifies classes
@@ -72,16 +74,15 @@ class PluginLoader:
# All runtime code is imported through the canonical `src` package root.
package_name = f"src.plugins.{module_name}"
# print(f"DEBUG: Loading plugin {module_name} as {package_name}")
spec = importlib.util.spec_from_file_location(package_name, file_path)
if spec is None or spec.loader is None:
print(f"Could not load module spec for {package_name}") # Replace with proper logging
_logger.error(f"Could not load module spec for {package_name}")
return
module = importlib.util.module_from_spec(spec)
try:
spec.loader.exec_module(module)
except Exception as e:
print(f"Error loading plugin module {module_name}: {e}") # Replace with proper logging
_logger.error(f"Error loading plugin module {module_name}: {e}")
return
for attribute_name in dir(module):
attribute = getattr(module, attribute_name)
@@ -94,7 +95,7 @@ class PluginLoader:
plugin_instance = attribute()
self._register_plugin(plugin_instance)
except Exception as e:
print(f"Error instantiating plugin {attribute_name} in {module_name}: {e}") # Replace with proper logging
_logger.error(f"Error instantiating plugin {attribute_name} in {module_name}: {e}")
# #endregion _load_module
# #region _register_plugin [TYPE Function]
# @PURPOSE: Registers a PluginBase instance and its configuration.
@@ -108,7 +109,7 @@ class PluginLoader:
"""
plugin_id = plugin_instance.id
if plugin_id in self._plugins:
print(f"Warning: Duplicate plugin ID '{plugin_id}' found. Skipping.") # Replace with proper logging
_logger.warning(f"Warning: Duplicate plugin ID '{plugin_id}' found. Skipping.")
return
try:
schema = plugin_instance.get_schema()
@@ -129,11 +130,9 @@ class PluginLoader:
# validate(instance={}, schema=schema)
self._plugins[plugin_id] = plugin_instance
self._plugin_configs[plugin_id] = plugin_config
from ..core.logger import logger
logger.info(f"Plugin '{plugin_instance.name}' (ID: {plugin_id}) loaded successfully.")
_logger.info(f"Plugin '{plugin_instance.name}' (ID: {plugin_id}) loaded successfully.")
except Exception as e:
from ..core.logger import logger
logger.error(f"Error validating plugin '{plugin_instance.name}' (ID: {plugin_id}): {e}")
_logger.error(f"Error validating plugin '{plugin_instance.name}' (ID: {plugin_id}): {e}")
# #endregion _register_plugin
# #region get_plugin [TYPE Function]
# @PURPOSE: Retrieves a loaded plugin instance by its ID.

215
backend/src/core/scheduler.py.bak
View File

@@ -1,215 +0,0 @@
# [DEF:SchedulerModule:Module]
# @COMPLEXITY: 3
# @SEMANTICS: scheduler, apscheduler, cron, backup
# @PURPOSE: Manages scheduled tasks using APScheduler.
# @LAYER: Core
# @RELATION: [DEPENDS_ON] ->[TaskManager]
# [SECTION: IMPORTS]
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
from .logger import logger, belief_scope
from .config_manager import ConfigManager
from .cot_logger import MarkerLogger, get_trace_id, set_trace_id
log = MarkerLogger("SchedulerService")
log_tsc = MarkerLogger("ThrottledSchedulerConfigurator")
import asyncio
from datetime import datetime, time, timedelta, date
# [/SECTION]
# [DEF:SchedulerService:Class]
# @COMPLEXITY: 3
# @SEMANTICS: scheduler, service, apscheduler
# @PURPOSE: Provides a service to manage scheduled backup tasks.
# @RELATION: DEPENDS_ON -> [ThrottledSchedulerConfigurator]
class SchedulerService:
# [DEF:__init__:Function]
# @PURPOSE: Initializes the scheduler service with task and config managers.
# @PRE: task_manager and config_manager must be provided.
# @POST: Scheduler instance is created but not started.
def __init__(self, task_manager, config_manager: ConfigManager):
with belief_scope("SchedulerService.__init__"):
self.task_manager = task_manager
self.config_manager = config_manager
self.scheduler = BackgroundScheduler()
self.loop = asyncio.get_event_loop()
# [/DEF:__init__:Function]
# [DEF:start:Function]
# @PURPOSE: Starts the background scheduler and loads initial schedules.
# @PRE: Scheduler should be initialized.
# @POST: Scheduler is running and schedules are loaded.
def start(self):
with belief_scope("SchedulerService.start"):
if not self.scheduler.running:
self.scheduler.start()
log.reflect("Scheduler started")
self.load_schedules()
# [/DEF:start:Function]
# [DEF:stop:Function]
# @PURPOSE: Stops the background scheduler.
# @PRE: Scheduler should be running.
# @POST: Scheduler is shut down.
def stop(self):
with belief_scope("SchedulerService.stop"):
if self.scheduler.running:
self.scheduler.shutdown()
log.reflect("Scheduler stopped")
# [/DEF:stop:Function]
# [DEF:load_schedules:Function]
# @PURPOSE: Loads backup schedules from configuration and registers them.
# @PRE: config_manager must have valid configuration.
# @POST: All enabled backup jobs are added to the scheduler.
def load_schedules(self):
with belief_scope("SchedulerService.load_schedules"):
# Clear existing jobs
self.scheduler.remove_all_jobs()
config = self.config_manager.get_config()
for env in config.environments:
if env.backup_schedule and env.backup_schedule.enabled:
self.add_backup_job(env.id, env.backup_schedule.cron_expression)
# [/DEF:load_schedules:Function]
# [DEF:add_backup_job:Function]
# @PURPOSE: Adds a scheduled backup job for an environment.
# @PRE: env_id and cron_expression must be valid strings.
# @POST: A new job is added to the scheduler or replaced if it already exists.
# @PARAM: env_id (str) - The ID of the environment.
# @PARAM: cron_expression (str) - The cron expression for the schedule.
def add_backup_job(self, env_id: str, cron_expression: str):
with belief_scope(
"SchedulerService.add_backup_job",
f"env_id={env_id}, cron={cron_expression}",
):
job_id = f"backup_{env_id}"
try:
self.scheduler.add_job(
self._trigger_backup,
CronTrigger.from_crontab(cron_expression),
id=job_id,
args=[env_id],
replace_existing=True,
)
log.reflect(
"Scheduled backup job added",
payload={"env_id": env_id, "cron": cron_expression},
)
except Exception as e:
log.explore("Failed to add backup job", error=str(e), payload={"env_id": env_id, "cron": cron_expression})
logger.error(f"Failed to add backup job for environment {env_id}: {e}")
# [/DEF:add_backup_job:Function]
# [DEF:_trigger_backup:Function]
# @PURPOSE: Triggered by the scheduler to start a backup task.
# @PRE: env_id must be a valid environment ID.
# @POST: A new backup task is created in the task manager if not already running.
# @PARAM: env_id (str) - The ID of the environment.
def _trigger_backup(self, env_id: str):
with belief_scope("SchedulerService._trigger_backup", f"env_id={env_id}"):
log.reason("Triggering scheduled backup", payload={"env_id": env_id})
# Check if a backup is already running for this environment
active_tasks = self.task_manager.get_tasks(limit=100)
for task in active_tasks:
if (
task.plugin_id == "superset-backup"
and task.status in ["PENDING", "RUNNING"]
and task.params.get("environment_id") == env_id
):
log.explore("Backup already running, skipping scheduled run", error="Duplicate backup prevented", payload={"env_id": env_id})
return
# Run the backup task
# We need to run this in the event loop since create_task is async
trace_id = get_trace_id()
async def _backup_task():
if trace_id:
set_trace_id(trace_id)
await self.task_manager.create_task(
"superset-backup", {"environment_id": env_id}
)
asyncio.run_coroutine_threadsafe(_backup_task(), self.loop)
# [/DEF:_trigger_backup:Function]
# [/DEF:SchedulerService:Class]
# [DEF:ThrottledSchedulerConfigurator:Class]
# @COMPLEXITY: 5
# @SEMANTICS: scheduler, throttling, distribution
# @PURPOSE: Distributes validation tasks evenly within an execution window.
# @PRE: Validation policies provide a finite dashboard list and a valid execution window.
# @POST: Produces deterministic per-dashboard run timestamps within the configured window.
# @RELATION: [DEPENDS_ON] ->[SchedulerModule]
# @INVARIANT: Returned schedule size always matches number of dashboard IDs.
# @SIDE_EFFECT: Emits warning logs for degenerate or near-zero scheduling windows.
# @DATA_CONTRACT: Input[window_start, window_end, dashboard_ids, current_date] -> Output[List[datetime]]
class ThrottledSchedulerConfigurator:
# [DEF:calculate_schedule:Function]
# @PURPOSE: Calculates execution times for N tasks within a window.
# @PRE: window_start, window_end (time), dashboard_ids (List), current_date (date).
# @POST: Returns List[datetime] of scheduled times.
# @INVARIANT: Tasks are distributed with near-even spacing.
@staticmethod
def calculate_schedule(
window_start: time, window_end: time, dashboard_ids: list, current_date: date
) -> list:
with belief_scope("ThrottledSchedulerConfigurator.calculate_schedule"):
n = len(dashboard_ids)
if n == 0:
return []
start_dt = datetime.combine(current_date, window_start)
end_dt = datetime.combine(current_date, window_end)
# Handle window crossing midnight
if end_dt < start_dt:
end_dt += timedelta(days=1)
total_seconds = (end_dt - start_dt).total_seconds()
# Minimum interval of 1 second to avoid division by zero or negative
if total_seconds <= 0:
log_tsc.explore("Window size is zero or negative, falling back to start time", error=f"total_seconds={total_seconds}", payload={"n": n})
return [start_dt] * n
# If window is too small for even distribution (e.g. 10 tasks in 5 seconds),
# we still distribute them but they might be very close.
# The requirement says "near-even spacing".
if n == 1:
return [start_dt]
interval = total_seconds / (n - 1) if n > 1 else 0
# If interval is too small (e.g. < 1s), we might want a fallback,
# but the spec says "handle too-small windows with explicit fallback/warning".
if interval < 1:
log_tsc.explore("Window too small for task distribution", error=f"interval={interval:.2f}s", payload={"n": n})
scheduled_times = []
for i in range(n):
scheduled_times.append(start_dt + timedelta(seconds=i * interval))
return scheduled_times
# [/DEF:calculate_schedule:Function]
# [/DEF:ThrottledSchedulerConfigurator:Class]
# [/DEF:SchedulerModule:Module]

59
backend/src/core/superset_client/_base.py
View File

@@ -1,19 +1,22 @@
# #region SupersetClientBase [C:3] [TYPE Module] [SEMANTICS superset, client, base, auth, pagination]
# @LAYER: Infra
# @LAYER Infra
# @BRIEF Base class for SupersetClient providing initialization, authentication, pagination, and import/export helpers.
# @RELATION DEPENDS_ON -> [ConfigModels]
# @RELATION DEPENDS_ON -> [APIClient]
# @RELATION DEPENDS_ON -> [SupersetAPIError]
# @RELATION DEPENDS_ON -> [get_filename_from_headers]
import zipfile
# @RELATION DEPENDS_ON -> [get_filename_from_headers]
# @RELATION DEPENDS_ON -> [get_filename_from_headers]
# @RELATION DEPENDS_ON -> [get_filename_from_headers]
# @RATIONALE Extracted magic number 100 into DEFAULT_PAGE_SIZE constant at module level. Value unchanged — Superset API caps page_size at 100.
from pathlib import Path
from typing import Any, Optional, cast
import zipfile
from requests import Response
DEFAULT_PAGE_SIZE = 100
from ..config_models import Environment
from ..logger import belief_scope
from ..logger import logger as app_logger
from ..logger import belief_scope, logger as app_logger
from ..utils.fileio import get_filename_from_headers
from ..utils.network import APIClient, SupersetAPIError
@@ -25,9 +28,9 @@ app_logger = cast(Any, app_logger)
# @RELATION DEPENDS_ON -> [SupersetAPIError]
class SupersetClientBase:
# #region SupersetClientInit [TYPE Function] [C:3]
# @PURPOSE: Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент.
# @RELATION: DEPENDS_ON -> [Environment]
# @RELATION: DEPENDS_ON -> [APIClient]
# @PURPOSE Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент.
# @RELATION DEPENDS_ON -> [Environment]
# @RELATION DEPENDS_ON -> [APIClient]
def __init__(self, env: Environment):
with belief_scope("SupersetClientInit"):
app_logger.reason(
@@ -54,8 +57,8 @@ class SupersetClientBase:
)
# #endregion SupersetClientInit
# #region SupersetClientAuthenticate [TYPE Function] [C:3]
# @PURPOSE: Authenticates the client using the configured credentials.
# @RELATION: CALLS -> [APIClient]
# @PURPOSE Authenticates the client using the configured credentials.
# @RELATION CALLS -> [APIClient]
def authenticate(self) -> dict[str, str]:
with belief_scope("SupersetClientAuthenticate"):
app_logger.reason(
@@ -74,24 +77,24 @@ class SupersetClientBase:
# #endregion SupersetClientAuthenticate
@property
# #region SupersetClientHeaders [TYPE Function] [C:1]
# @PURPOSE: Возвращает базовые HTTP-заголовки, используемые сетевым клиентом.
# @PURPOSE Возвращает базовые HTTP-заголовки, используемые сетевым клиентом.
def headers(self) -> dict:
with belief_scope("headers"):
return self.network.headers
# #endregion SupersetClientHeaders
# --- Pagination helpers ---
# #region SupersetClientValidateQueryParams [TYPE Function] [C:1]
# @PURPOSE: Ensures query parameters have default page and page_size.
# @PURPOSE Ensures query parameters have default page and page_size.
def _validate_query_params(self, query: dict | None) -> dict:
with belief_scope("_validate_query_params"):
# Superset list endpoints commonly cap page_size at 100.
# Using 100 avoids partial fetches when larger values are silently truncated.
base_query = {"page": 0, "page_size": 100}
base_query = {"page": 0, "page_size": DEFAULT_PAGE_SIZE}
return {**base_query, **(query or {})}
# #endregion SupersetClientValidateQueryParams
# #region SupersetClientFetchTotalObjectCount [TYPE Function] [C:1]
# @PURPOSE: Fetches the total number of items for a given endpoint.
# @RELATION: CALLS -> [APIClient]
# @PURPOSE Fetches the total number of items for a given endpoint.
# @RELATION CALLS -> [APIClient]
def _fetch_total_object_count(self, endpoint: str) -> int:
with belief_scope("_fetch_total_object_count"):
return self.network.fetch_paginated_count(
@@ -101,8 +104,8 @@ class SupersetClientBase:
)
# #endregion SupersetClientFetchTotalObjectCount
# #region SupersetClientFetchAllPages [TYPE Function] [C:1]
# @PURPOSE: Iterates through all pages to collect all data items.
# @RELATION: CALLS -> [APIClient]
# @PURPOSE Iterates through all pages to collect all data items.
# @RELATION CALLS -> [APIClient]
def _fetch_all_pages(self, endpoint: str, pagination_options: dict) -> list[dict]:
with belief_scope("_fetch_all_pages"):
return self.network.fetch_paginated_data(
@@ -111,8 +114,8 @@ class SupersetClientBase:
# #endregion SupersetClientFetchAllPages
# --- Import/Export helpers ---
# #region SupersetClientDoImport [TYPE Function] [C:1]
# @PURPOSE: Performs the actual multipart upload for import.
# @RELATION: CALLS -> [APIClient]
# @PURPOSE Performs the actual multipart upload for import.
# @RELATION CALLS -> [APIClient]
def _do_import(self, file_name: str | Path) -> dict:
with belief_scope("_do_import"):
app_logger.debug(f"[_do_import][State] Uploading file: {file_name}")
@@ -134,7 +137,7 @@ class SupersetClientBase:
)
# #endregion SupersetClientDoImport
# #region SupersetClientValidateExportResponse [TYPE Function] [C:1]
# @PURPOSE: Validates that the export response is a non-empty ZIP archive.
# @PURPOSE Validates that the export response is a non-empty ZIP archive.
def _validate_export_response(self, response: Response, dashboard_id: int) -> None:
with belief_scope("_validate_export_response"):
content_type = response.headers.get("Content-Type", "")
@@ -146,7 +149,7 @@ class SupersetClientBase:
raise SupersetAPIError("Получены пустые данные при экспорте")
# #endregion SupersetClientValidateExportResponse
# #region SupersetClientResolveExportFilename [TYPE Function] [C:1]
# @PURPOSE: Determines the filename for an exported dashboard.
# @PURPOSE Determines the filename for an exported dashboard.
def _resolve_export_filename(self, response: Response, dashboard_id: int) -> str:
with belief_scope("_resolve_export_filename"):
filename = get_filename_from_headers(dict(response.headers))
@@ -161,7 +164,7 @@ class SupersetClientBase:
return filename
# #endregion SupersetClientResolveExportFilename
# #region SupersetClientValidateImportFile [TYPE Function] [C:1]
# @PURPOSE: Validates that the file to be imported is a valid ZIP with metadata.yaml.
# @PURPOSE Validates that the file to be imported is a valid ZIP with metadata.yaml.
def _validate_import_file(self, zip_path: str | Path) -> None:
with belief_scope("_validate_import_file"):
path = Path(zip_path)
@@ -176,8 +179,8 @@ class SupersetClientBase:
)
# #endregion SupersetClientValidateImportFile
# #region SupersetClientResolveTargetIdForDelete [TYPE Function] [C:1]
# @PURPOSE: Resolves a dashboard ID from either an ID or a slug.
# @RELATION: CALLS -> [SupersetClientGetDashboards]
# @PURPOSE Resolves a dashboard ID from either an ID or a slug.
# @RELATION CALLS -> [SupersetClientGetDashboards]
def _resolve_target_id_for_delete(
self, dash_id: int | None, dash_slug: str | None
) -> int | None:
@@ -211,8 +214,8 @@ class SupersetClientBase:
return None
# #endregion SupersetClientResolveTargetIdForDelete
# #region SupersetClientGetAllResources [TYPE Function] [C:3]
# @PURPOSE: Fetches all resources of a given type with id, uuid, and name columns.
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
# @PURPOSE Fetches all resources of a given type with id, uuid, and name columns.
# @RELATION CALLS -> [SupersetClientFetchAllPages]
def get_all_resources(
self, resource_type: str, since_dttm: Optional["datetime"] = None
) -> list[dict]:

7
backend/src/core/superset_client/_charts.py
View File

@@ -5,8 +5,7 @@
import re
from typing import Any, cast
from ..logger import belief_scope
from ..logger import logger as app_logger
from ..logger import belief_scope, logger as app_logger
app_logger = cast(Any, app_logger)
# #region SupersetChartsMixin [C:3] [TYPE Class]
@@ -52,14 +51,14 @@ class SupersetChartsMixin:
try:
found.add(int(value))
except (TypeError, ValueError):
pass
app_logger.debug("Could not parse chart id value: %s", value)
if key == "id" and isinstance(value, str):
match = re.match(r"^CHART-(\d+)$", value)
if match:
try:
found.add(int(match.group(1)))
except ValueError:
pass
app_logger.debug("Could not parse CHART-N id: %s", match.group(1))
walk(value)
elif isinstance(node, list):
for item in node:

9
backend/src/core/superset_client/_dashboards_crud.py
View File

@@ -5,14 +5,13 @@
# @RELATION DEPENDS_ON -> [SupersetDashboardsFiltersMixin]
# @RELATION DEPENDS_ON -> [SupersetChartsMixin]
import json
import re
from pathlib import Path
import re
from typing import Any, cast
from requests import Response
from ..logger import belief_scope
from ..logger import logger as app_logger
from ..logger import belief_scope, logger as app_logger
app_logger = cast(Any, app_logger)
# #region SupersetDashboardsCrudMixin [C:3] [TYPE Class]
@@ -151,7 +150,7 @@ class SupersetDashboardsCrudMixin:
self._extract_chart_ids_from_layout(parsed_position)
)
except Exception:
pass
app_logger.debug("Could not parse position JSON for dashboard %s", dashboard_ref)
elif isinstance(raw_position_json, dict):
chart_ids_from_position.update(
self._extract_chart_ids_from_layout(raw_position_json)
@@ -164,7 +163,7 @@ class SupersetDashboardsCrudMixin:
self._extract_chart_ids_from_layout(parsed_metadata)
)
except Exception:
pass
app_logger.debug("Could not parse json_metadata for dashboard %s", dashboard_ref)
elif isinstance(raw_json_metadata, dict):
chart_ids_from_position.update(
self._extract_chart_ids_from_layout(raw_json_metadata)

7
backend/src/core/superset_client/_dashboards_filters.py
View File

@@ -5,8 +5,7 @@
import json
from typing import Any, cast
from ..logger import belief_scope
from ..logger import logger as app_logger
from ..logger import belief_scope, logger as app_logger
app_logger = cast(Any, app_logger)
# #region SupersetDashboardsFiltersMixin [C:3] [TYPE Class]
@@ -162,7 +161,7 @@ class SupersetDashboardsFiltersMixin:
result["filters"] = filter_data
return result
except ValueError:
pass
app_logger.debug("Permalink key %s not found in URL", permalink_key)
# Check for native_filters_key in query params
native_filters_key = query_params.get("native_filters_key", [None])[0]
if native_filters_key:
@@ -175,7 +174,7 @@ class SupersetDashboardsFiltersMixin:
if potential_id not in ("p", "list", "new"):
dashboard_ref = potential_id
except ValueError:
pass
app_logger.debug("Dashboard not found in path_parts for native_filters_key")
if dashboard_ref:
resolved_id = None
try:

3
backend/src/core/superset_client/_dashboards_list.py
View File

@@ -6,8 +6,7 @@
import json
from typing import Any, cast
from ..logger import belief_scope
from ..logger import logger as app_logger
from ..logger import belief_scope, logger as app_logger
app_logger = cast(Any, app_logger)
# #region SupersetDashboardsListMixin [C:3] [TYPE Class]

3
backend/src/core/superset_client/_databases.py
View File

@@ -4,8 +4,7 @@
# @RELATION DEPENDS_ON -> [SupersetClientBase]
from typing import Any, cast
from ..logger import belief_scope
from ..logger import logger as app_logger
from ..logger import belief_scope, logger as app_logger
app_logger = cast(Any, app_logger)
# #region SupersetDatabasesMixin [C:3] [TYPE Class]

3
backend/src/core/superset_client/_datasets.py
View File

@@ -5,8 +5,7 @@
import json
from typing import Any, cast
from ..logger import belief_scope
from ..logger import logger as app_logger
from ..logger import belief_scope, logger as app_logger
app_logger = cast(Any, app_logger)
# #region SupersetDatasetsMixin [C:3] [TYPE Class]

5
backend/src/core/superset_client/_datasets_preview.py
View File

@@ -3,12 +3,11 @@
# @BRIEF Dataset preview compilation mixin for SupersetClient — build query context, compile SQL.
# @RELATION DEPENDS_ON -> [SupersetClientBase]
# @RELATION DEPENDS_ON -> [SupersetDatasetsMixin]
import json
from copy import deepcopy
import json
from typing import Any
from ..logger import belief_scope
from ..logger import logger as app_logger
from ..logger import belief_scope, logger as app_logger
from ..utils.network import SupersetAPIError

3
backend/src/core/superset_client/_datasets_preview_filters.py
View File

@@ -6,8 +6,7 @@
from copy import deepcopy
from typing import Any
from ..logger import belief_scope
from ..logger import logger as app_logger
from ..logger import belief_scope, logger as app_logger
from ..utils.network import SupersetAPIError

3
backend/src/core/task_manager/__tests__/test_task_logger.py
View File

@@ -2,9 +2,8 @@
# @RELATION VERIFIES -> ../task_logger.py
# @BRIEF Contract testing for TaskLogger
# #endregion __tests__/test_task_logger
from unittest.mock import MagicMock
import pytest
from unittest.mock import MagicMock
from src.core.task_manager.task_logger import TaskLogger

2
backend/src/core/task_manager/lifecycle.py
View File

@@ -23,9 +23,9 @@
# a new wait future is created.
import asyncio
import inspect
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
import inspect
from typing import Any
from ..cot_logger import seed_trace_id

2
backend/src/core/task_manager/manager.py
View File

@@ -33,8 +33,8 @@
# with async execution, and created an unmaintainable god class.
import asyncio
import threading
from concurrent.futures import ThreadPoolExecutor
import threading
from typing import Any
from ..logger import belief_scope, logger

2
backend/src/core/task_manager/models.py
View File

@@ -9,10 +9,10 @@
# @POST: Task models exported with immutable IDs
# @SIDE_EFFECT: Defines task data schema
# @DATA_CONTRACT: TaskInput -> TaskModel
import uuid
from datetime import datetime
from enum import Enum
from typing import Any
import uuid
from pydantic import BaseModel, Field

2
backend/src/core/task_manager/persistence.py
View File

@@ -9,9 +9,9 @@
# @RELATION DEPENDS_ON -> [TaskGraph]
# @RELATION DEPENDS_ON -> [TasksSessionLocal]
# @INVARIANT: Database schema must match the TaskRecord model structure.
from datetime import datetime
import json
import re
from datetime import datetime
from sqlalchemy.orm import Session

3
backend/src/core/utils/async_network.py
View File

@@ -13,8 +13,7 @@ from typing import Any
import httpx
from ..logger import belief_scope
from ..logger import logger as app_logger
from ..logger import belief_scope, logger as app_logger
from .network import (
AuthenticationError,
DashboardNotFoundError,

3
backend/src/core/utils/dataset_mapper.py
View File

@@ -11,8 +11,7 @@ from typing import Any
import pandas as pd # type: ignore
import psycopg2 # type: ignore
from ..logger import belief_scope
from ..logger import logger as app_logger
from ..logger import belief_scope, logger as app_logger
# #region DatasetMapper [TYPE Class]

17
backend/src/core/utils/fileio.py
View File

@@ -5,21 +5,20 @@
# @LAYER: Infra
# @RELATION DEPENDS_ON -> [LoggerModule]
# @PUBLIC_API: create_temp_file, remove_empty_directories, read_dashboard_from_disk, calculate_crc32, RetentionPolicy, archive_exports, save_and_unpack_dashboard, update_yamls, create_dashboard_export, sanitize_filename, get_filename_from_headers, consolidate_archive_folders
import os
import re
import shutil
import tempfile
import zipfile
import zlib
from collections.abc import Generator
from contextlib import contextmanager
from dataclasses import dataclass
from datetime import date, datetime
import os
from pathlib import Path
import re
import shutil
import tempfile
from typing import Any, LiteralString
import zipfile
import zlib
from ..logger import belief_scope
from ..logger import logger as app_logger
from ..logger import belief_scope, logger as app_logger
# #region InvalidZipFormatError [TYPE Class]
@@ -174,7 +173,7 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
date_str = match.group(1)
file_date = datetime.strptime(date_str, "%Y%m%d").date()
except ValueError:
pass
app_logger.debug("Could not parse date from filename %s", file_path.name)
if not file_date:
# Fallback to modification time

7
backend/src/core/utils/network.py
View File

@@ -6,18 +6,17 @@
# @PUBLIC_API: APIClient
import io
import json
from pathlib import Path
import threading
import time
from pathlib import Path
from typing import Any, cast
import requests
import urllib3
from requests.adapters import HTTPAdapter
import urllib3
from urllib3.util.retry import Retry
from ..logger import belief_scope
from ..logger import logger as app_logger
from ..logger import belief_scope, logger as app_logger
# #region SupersetAPIError [C:1] [TYPE Class]

4
backend/src/core/utils/superset_context_extractor/_base.py
View File

@@ -12,8 +12,8 @@
# #region _base_imports [TYPE Block]
from __future__ import annotations
import json
from dataclasses import dataclass, field
import json
from typing import Any, cast
from urllib.parse import unquote
@@ -149,7 +149,7 @@ class SupersetContextExtractorBase:
if value is not None:
return int(value)
except (TypeError, ValueError):
pass
logger.debug("Could not parse numeric value from payload key %s", key)
found = self._search_nested_numeric_key(value, candidate_keys)
if found is not None:
return found

2
backend/src/core/utils/superset_context_extractor/_pii.py
View File

@@ -4,8 +4,8 @@
# #endregion SupersetContextExtractorPII
# #region _pii_imports [TYPE Block]
import re
from copy import deepcopy
import re
from typing import Any
# #endregion _pii_imports

2
backend/src/core/utils/superset_context_extractor/_recovery.py
View File

@@ -7,8 +7,8 @@
# #region _recovery_imports [TYPE Block]
from __future__ import annotations
import json
from copy import deepcopy
import json
from typing import Any, cast
from ...logger import belief_scope, logger

2
backend/src/core/ws_log_handler.py
View File

@@ -2,9 +2,9 @@
# @BRIEF WebSocket log handler module providing LogEntry DTO and WebSocketLogHandler for buffered log streaming.
# @RELATION DEPENDS_ON -> [LogEntry]
# @RELATION COMPOSES -> [CotJsonFormatter]
import logging
from collections import deque
from datetime import datetime
import logging
from typing import Any
from pydantic import BaseModel, Field