ss-tools/backend/src/core/config_models.py

# #region ConfigModels [C:3] [TYPE Module] [SEMANTICS pydantic, model, schedule]
# @defgroup Core Module group.
# @BRIEF Defines the data models for application configuration using Pydantic.
# @LAYER Core
# @RELATION IMPLEMENTS -> [CoreContracts]
# @RELATION IMPLEMENTS -> [ConnectionContracts]
#
# #region CoreContracts [C:2] [TYPE Interface]
# @ingroup Core
# @BRIEF Contract for core configuration models — Pydantic BaseModel subclasses with scheduler config.
# #endregion CoreContracts
#
# #region ConnectionContracts [C:2] [TYPE Interface]
# @ingroup Core
# @BRIEF Contract for database/environment connection configuration models.
# #endregion ConnectionContracts

import uuid
from datetime import datetime, timezone

from pydantic import BaseModel, Field, field_validator

from ..models.storage import StorageConfig
from ..services.llm_prompt_templates import (
    DEFAULT_LLM_ASSISTANT_SETTINGS,
    DEFAULT_LLM_PROMPTS,
    DEFAULT_LLM_PROVIDER_BINDINGS,
)


# #region Schedule [TYPE DataClass]
# @ingroup Core
# @BRIEF Represents a backup schedule configuration.
class Schedule(BaseModel):
    enabled: bool = False
    cron_expression: str = "0 0 * * *"  # Default: daily at midnight


# #endregion Schedule


# #region Environment [TYPE DataClass]
# @ingroup Core
# @BRIEF Represents a Superset environment configuration with async connection pool settings.
class Environment(BaseModel):
    id: str
    name: str
    url: str
    username: str
    password: str  # Will be masked in UI
    stage: str = Field(default="DEV", pattern="^(DEV|PREPROD|PROD)$")
    verify_ssl: bool = True
    timeout: int = 30
    is_default: bool = False
    is_production: bool = False
    backup_schedule: Schedule = Field(default_factory=Schedule)

    # Async connection pool settings (per-environment)
    connection_pool_size: int = Field(
        default=20, ge=1, le=100,
        description="Max concurrent connections to this Superset environment"
    )
    connection_pool_timeout: int = Field(
        default=30, ge=5, le=300,
        description="Seconds to wait for a connection pool slot before timeout"
    )
    request_timeout: int = Field(
        default=30, ge=5, le=300,
        description="Default HTTP request timeout for Superset API calls"
    )
    llm_timeout: int = Field(
        default=120, ge=30, le=600,
        description="Timeout for LLM API calls (seconds)"
    )


# #endregion Environment


# #region AppAsyncRuntimeConfig [TYPE DataClass]
# @ingroup Core
# @BRIEF Global application-level async runtime configuration.
# @RATIONALE Separated from EnvironmentConfig because these settings are not per-env:
#   executor workers, shutdown timeout, blocking queue timeout are app-wide.
class AppAsyncRuntimeConfig(BaseModel):
    db_executor_workers: int = Field(
        default=10, ge=1, le=50,
        description="Max worker threads for database blocking operations"
    )
    file_executor_workers: int = Field(
        default=10, ge=1, le=50,
        description="Max worker threads for file I/O blocking operations"
    )
    git_executor_workers: int = Field(
        default=5, ge=1, le=20,
        description="Max worker threads for Git blocking operations"
    )
    graceful_shutdown_timeout: int = Field(
        default=30, ge=10, le=120,
        description="Graceful shutdown timeout in seconds"
    )
    blocking_queue_timeout: int = Field(
        default=30, ge=5, le=120,
        description="Timeout for acquiring blocking executor queue slot"
    )
    event_bus_maxsize: int = Field(
        default=10000, ge=100, le=100000,
        description="Max pending events in EventBus per subscriber queue"
    )


# #endregion AppAsyncRuntimeConfig


# #region LoggingConfig [TYPE DataClass]
# @ingroup Core
# @BRIEF Defines the configuration for the application's logging system.
class LoggingConfig(BaseModel):
    level: str = "INFO"
    task_log_level: str = (
        "INFO"  # Minimum level for task-specific logs (DEBUG, INFO, WARNING, ERROR)
    )
    file_path: str | None = None
    max_bytes: int = 10 * 1024 * 1024
    backup_count: int = 5
    enable_belief_state: bool = True


# #endregion LoggingConfig


# #region CleanReleaseConfig [TYPE DataClass]
# @ingroup Core
# @BRIEF Configuration for clean release compliance subsystem.
class CleanReleaseConfig(BaseModel):
    active_policy_id: str | None = None
    active_registry_id: str | None = None


# #endregion CleanReleaseConfig


# #region FeaturesConfig [C:1] [TYPE DataClass]
# @BRIEF Top-level feature flags that toggle entire project features on/off.
# @RATIONALE  Features are read from environment variables on bootstrap and persisted in DB.
#            DB is source of truth after initial bootstrap; env vars only seed defaults.
# @INVARIANT Each feature flag controls visibility of plugin/service in UI and API access.
class FeaturesConfig(BaseModel):
    # ── Core Features ────────────────────────────────────────
    dataset_review: bool = True
    health_monitor: bool = True

    # ── Plugin Features ──────────────────────────────────────
    # Translation subsystem
    translate: bool = True                    # LLM Table Translation (main)
    translate_scheduling: bool = True          # Cron-based translation scheduling
    translate_dictionaries: bool = True        # Terminology dictionary management

    # Dashboard migration & version control
    migration: bool = True                    # Superset Dashboard Migration
    git_integration: bool = True              # Git Integration

    # System tools
    backup: bool = True                       # Superset Dashboard Backup
    debug: bool = True                        # System Diagnostics
    storage_manager: bool = True              # Storage Manager
    dataset_mapper: bool = True               # Dataset Column Mapper
    search_datasets: bool = True              # Search Datasets
    maintenance_banner: bool = True           # Maintenance Banner

    # LLM analysis tools
    llm_dashboard_validation: bool = True     # Dashboard LLM Validation
    llm_documentation: bool = True            # Dataset LLM Documentation


# #endregion FeaturesConfig


# #region DatabaseConnection [C:1] [TYPE DataClass] [SEMANTICS settings,connections,database]
# @ingroup Core
# @BRIEF A reusable database connection configuration stored in GlobalSettings.connections.
# @RATIONALE Typed model prevents config corruption from ad-hoc dicts. Replaces the existing
#           `connections: list[dict]` stub. connection_id references the UUID in this list — not a DB FK.
# @REJECTED Keeping `list[dict]` was rejected — would require per-field validation in every consumer.
#           A separate DB table was rejected — connections are configuration, not domain data.
class DatabaseConnection(BaseModel):
    id: str = Field(default_factory=lambda: uuid.uuid4().hex[:12])
    name: str
    host: str
    port: int = 5432
    database: str
    username: str
    password: str  # Fernet-encrypted at rest; masked as "********" in API responses
    dialect: str = "postgresql"
    extra_params: dict = Field(default_factory=dict)
    pool_size: int = Field(default=5, ge=1, le=100)
    created_at: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
    updated_at: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())

    @field_validator("dialect")
    @classmethod
    def validate_dialect(cls, v: str) -> str:
        allowed = {"postgresql", "clickhouse", "mysql"}
        if v.lower() not in allowed:
            raise ValueError(f"Unsupported dialect: {v}. Allowed: {', '.join(sorted(allowed))}")
        return v.lower()

    @field_validator("port")
    @classmethod
    def validate_port(cls, v: int) -> int:
        if v < 1 or v > 65535:
            raise ValueError(f"Port must be between 1 and 65535, got {v}")
        return v

    @field_validator("name")
    @classmethod
    def validate_name(cls, v: str) -> str:
        if not v or not v.strip():
            raise ValueError("Connection name must not be empty")
        return v.strip()


# #endregion DatabaseConnection


# #region GlobalSettings [TYPE DataClass]
# @ingroup Core
# @BRIEF Represents global application settings.
class GlobalSettings(BaseModel):
    storage: StorageConfig = Field(default_factory=StorageConfig)
    clean_release: CleanReleaseConfig = Field(default_factory=CleanReleaseConfig)
    default_environment_id: str | None = None
    logging: LoggingConfig = Field(default_factory=LoggingConfig)
    features: FeaturesConfig = Field(default_factory=FeaturesConfig)
    connections: list[DatabaseConnection] = []
    llm: dict = Field(
        default_factory=lambda: {
            "providers": [],
            "default_provider": "",
            "prompts": dict(DEFAULT_LLM_PROMPTS),
            "provider_bindings": dict(DEFAULT_LLM_PROVIDER_BINDINGS),
            **dict(DEFAULT_LLM_ASSISTANT_SETTINGS),
        }
    )

    # Application timezone
    app_timezone: str = "Europe/Moscow"

    @field_validator("app_timezone")
    @classmethod
    def validate_app_timezone(cls, v: str) -> str:
        from zoneinfo import ZoneInfo
        try:
            ZoneInfo(v)
        except (KeyError, TypeError):
            raise ValueError(f"Invalid IANA timezone: {v}") from None
        return v

    # Task retention settings
    task_retention_days: int = 30
    task_retention_limit: int = 100
    pagination_limit: int = 10

    # Migration sync settings
    migration_sync_cron: str = "0 2 * * *"

    # Dataset Review Feature Flags
    ff_dataset_auto_review: bool = True
    ff_dataset_clarification: bool = True
    ff_dataset_execution: bool = True

    # LLM validation retention (days)
    LLM_SCREENSHOT_RETENTION_DAYS: int = 30
    LLM_RAW_RESPONSE_RETENTION_DAYS: int = 30

    # Global worker limit for concurrent validation runs
    GLOBAL_VALIDATION_WORKER_LIMIT: int = 3

    # Allowed languages for translation (BCP-47 codes)
    allowed_languages: list[str] = Field(
        default_factory=lambda: [
            "ru", "en", "de", "fr", "es", "it", "pt", "zh", "ja", "ko",
            "ar", "tr", "nl", "pl", "sv", "da", "fi", "cs", "hu", "ro",
            "vi", "th", "he", "id", "ms",
        ]
    )

    @field_validator("allowed_languages")
    @classmethod
    def validate_allowed_languages(cls, v: list[str]) -> list[str]:
        import re
        for tag in v:
            if not tag or not tag.strip():
                raise ValueError("Empty language code is not allowed")
            if not re.match(r'^[a-zA-Z]{2,8}(-[a-zA-Z0-9]{1,8})*$', tag.strip()):
                raise ValueError(f"Invalid BCP-47 language code: {tag}")
        return v


# #endregion GlobalSettings


# #region AppConfig [TYPE DataClass]
# @ingroup Core
# @BRIEF The root configuration model containing all application settings.
class AppConfig(BaseModel):
    environments: list[Environment] = []
    settings: GlobalSettings


# #endregion AppConfig

# #endregion ConfigModels