fix(maintenance): auto-expiry + adaptive markdown height + tests

- Auto-expiry: expired events auto-end on GET /events via task manager
- Height: _estimate_markdown_height adapted to unit=8px scale with padding detection
- CRITICAL-1: removed dead import remove_chart_from_position (QA finding)
- CRITICAL-2: added chart alive check in ensure_banner_chart (QA finding)
- CRITICAL-3: fixed test assertions update_markdown_chart → update_banner_on_dashboard
- @POST contract: fixed return range [2,12] → [19,200]
- Tests: 8 new tests (auto-expiry + layout height)
This commit is contained in:
2026-05-25 08:36:33 +03:00
parent 2bf686674e
commit 31680a1bc9
100 changed files with 19635 additions and 7923 deletions

View File

@@ -28,6 +28,7 @@ __all__ = [
"git",
"health",
"llm",
"maintenance",
"mappings",
"migration",
"plugins",

View File

@@ -0,0 +1,10 @@
# #region MaintenanceRoutesPackage [C:1] [TYPE Package] [SEMANTICS maintenance, api, package]
# @BRIEF Maintenance Banner API route package.
# @LAYER API
# @RELATION DEPENDS_ON -> [MaintenanceRouter]
# @RELATION DEPENDS_ON -> [MaintenanceSchemasModule]
from ._router import router as maintenance_router
__all__ = ["maintenance_router"]
# #endregion MaintenanceRoutesPackage

View File

@@ -0,0 +1,14 @@
# #region MaintenanceRouter [C:3] [TYPE Module] [SEMANTICS fastapi, router, maintenance, api]
# @BRIEF FastAPI APIRouter for the Maintenance Banner endpoints.
# @LAYER API
# @RELATION DEPENDS_ON -> [MaintenanceRoutesModule]
# @RELATION DEPENDS_ON -> [AppDependencies]
# @RELATION DEPENDS_ON -> [MaintenanceSchemasModule]
from fastapi import APIRouter
router = APIRouter(prefix="/api/maintenance", tags=["Maintenance"])
# Import route handlers to register them with the router
from . import _routes # noqa: F401, E402
# #endregion MaintenanceRouter

View File

@@ -0,0 +1,562 @@
# #region MaintenanceRoutesModule [C:3] [TYPE Module] [SEMANTICS fastapi, routes, maintenance, api]
# @BRIEF Route handler stubs for all 7 Maintenance Banner API endpoints with RBAC guards per FR-015.
# @LAYER API
# @RELATION DEPENDS_ON -> [MaintenanceSchemasModule]
# @RELATION DEPENDS_ON -> [MaintenanceRouter]
# @RELATION DEPENDS_ON -> [AppDependencies]
# @RELATION DEPENDS_ON -> [MaintenanceServiceModule]
# @INVARIANT All mutation endpoints return 202 {task_id} per spec.
# @INVARIANT RBAC enforced per FR-015 matrix via has_permission() dependency.
from datetime import datetime, timezone
from typing import Any, cast
from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy.orm import Session
from ....core.logger import belief_scope, logger as app_logger
from ....core.task_manager import TaskManager
app_logger = cast(Any, app_logger)
from ....dependencies import get_current_user, get_db, get_task_manager, has_permission
from ....models.maintenance import (
MaintenanceDashboardBanner,
MaintenanceDashboardBannerStatus,
MaintenanceDashboardState,
MaintenanceDashboardStateStatus,
MaintenanceEvent,
MaintenanceEventStatus,
MaintenanceSettings,
)
from ._router import router
from ._schemas import (
MaintenanceAlreadyActiveResponse,
MaintenanceDashboardBannerState,
MaintenanceEndResponse,
MaintenanceEventItem,
MaintenanceEventListResponse,
MaintenanceSettingsResponse,
MaintenanceSettingsUpdate,
MaintenanceStartRequest,
MaintenanceStartResponse,
)
# ── GET /api/maintenance/dashboard-banners ───────────────────
# FR-015: viewer, operator, admin all have READ access
# #region list_dashboard_banners [C:2] [TYPE Function]
# @BRIEF Get per-dashboard banner state for the Dashboard Hub indicator.
# @RELATION DEPENDS_ON -> [has_permission("maintenance", "READ")]
@router.get("/dashboard-banners", response_model=list[MaintenanceDashboardBannerState])
async def list_dashboard_banners(
db: Session = Depends(get_db),
_=Depends(has_permission("maintenance", "READ")),
):
with belief_scope("list_dashboard_banners"):
# Query active banners with their linked active/pending states
active_banners = (
db.query(MaintenanceDashboardBanner)
.filter(MaintenanceDashboardBanner.status == MaintenanceDashboardBannerStatus.ACTIVE)
.all()
)
result: list[MaintenanceDashboardBannerState] = []
for banner in active_banners:
linked_states = (
db.query(MaintenanceDashboardState)
.filter(
MaintenanceDashboardState.banner_id == banner.id,
MaintenanceDashboardState.status.in_([
MaintenanceDashboardStateStatus.ACTIVE,
MaintenanceDashboardStateStatus.PENDING_APPLY,
]),
)
.all()
)
event_ids: list[str] = []
start_times: list[str] = []
end_times: list[str] = []
for state in linked_states:
event = (
db.query(MaintenanceEvent)
.filter(MaintenanceEvent.id == state.event_id)
.first()
)
if event:
event_ids.append(event.id)
if event.start_time:
start_times.append(event.start_time.isoformat())
if event.end_time:
end_times.append(event.end_time.isoformat())
result.append(
MaintenanceDashboardBannerState(
dashboard_id=banner.dashboard_id,
active=True,
events=event_ids,
start_time=min(start_times) if start_times else None,
end_time=max(end_times) if end_times else None,
)
)
return result
# #endregion list_dashboard_banners
# ── GET /api/maintenance/events ──────────────────────────────
# FR-015: operator, admin (NOT viewer)
# #region list_events [C:2] [TYPE Function]
# @BRIEF Get active and completed maintenance event lists with affected dashboard counts.
# @RELATION DEPENDS_ON -> [has_permission("maintenance", "READ")]
@router.get("/events", response_model=MaintenanceEventListResponse)
async def list_events(
db: Session = Depends(get_db),
_=Depends(has_permission("maintenance", "READ")),
task_manager: TaskManager = Depends(get_task_manager),
):
with belief_scope("list_events"):
# ── Auto-expiry: end events past their end_time ──
now = datetime.now(timezone.utc)
expired = (
db.query(MaintenanceEvent)
.filter(
MaintenanceEvent.status.in_([
MaintenanceEventStatus.ACTIVE,
MaintenanceEventStatus.PARTIAL,
MaintenanceEventStatus.ENDING,
]),
MaintenanceEvent.end_time.isnot(None),
MaintenanceEvent.end_time < now,
)
.all()
)
for ev in expired:
app_logger.reason(
"Auto-ending expired maintenance event",
extra={"event_id": ev.id, "end_time": ev.end_time.isoformat()},
)
# Schedule proper cleanup via task manager (removes banner from dashboard)
task = await task_manager.create_task(
plugin_id="maintenance_banner_apply",
params={
"operation": "end",
"event_id": ev.id,
"environment_id": ev.environment_id,
"user": "system",
},
)
app_logger.reflect(
"Scheduled cleanup for expired event",
extra={"event_id": ev.id, "task_id": task.id},
)
# Active events: ACTIVE, PARTIAL (per spec T040)
active_statuses = [
MaintenanceEventStatus.ACTIVE,
MaintenanceEventStatus.PARTIAL,
]
active_events = (
db.query(MaintenanceEvent)
.filter(MaintenanceEvent.status.in_(active_statuses))
.order_by(MaintenanceEvent.created_at.desc())
.all()
)
# Completed events: COMPLETED, FAILED
completed_events = (
db.query(MaintenanceEvent)
.filter(
MaintenanceEvent.status.in_([
MaintenanceEventStatus.COMPLETED,
MaintenanceEventStatus.FAILED,
])
)
.order_by(MaintenanceEvent.created_at.desc())
.limit(50)
.all()
)
def _to_item(event: MaintenanceEvent) -> MaintenanceEventItem:
affected_count = (
db.query(MaintenanceDashboardState)
.filter(
MaintenanceDashboardState.event_id == event.id,
MaintenanceDashboardState.status.in_([
MaintenanceDashboardStateStatus.ACTIVE,
MaintenanceDashboardStateStatus.PENDING_APPLY,
]),
)
.count()
)
return MaintenanceEventItem(
id=event.id,
tables=list(event.tables) if event.tables else [],
start_time=event.start_time.isoformat() if event.start_time else "",
end_time=event.end_time.isoformat() if event.end_time else None,
message=event.message or None,
status=event.status.value,
affected_count=affected_count,
created_at=event.created_at.isoformat() if event.created_at else "",
)
return MaintenanceEventListResponse(
active=[_to_item(e) for e in active_events],
completed=[_to_item(e) for e in completed_events],
)
# #endregion list_events
# ── POST /api/maintenance/start ──────────────────────────────
# FR-001: Always returns 202 {task_id} after validation
# FR-015: operator, admin (NOT viewer)
# #region start_maintenance [C:3] [TYPE Function]
# @BRIEF Start a maintenance event. Validates input, creates event, dispatches TaskManager task.
# Always returns 202 with task_id and maintenance_id.
# Idempotency: same (tables, start_time, end_time) returns already_active (409-like but 200).
# Tables sorted & lowered for idempotency key. Message NOT in key.
# @RELATION DEPENDS_ON -> [has_permission("maintenance", "WRITE")]
# @RELATION DEPENDS_ON -> [TaskManager]
@router.post(
"/start",
status_code=status.HTTP_202_ACCEPTED,
responses={
202: {"model": MaintenanceStartResponse},
409: {"model": MaintenanceAlreadyActiveResponse},
},
)
async def start_maintenance(
request: MaintenanceStartRequest,
db: Session = Depends(get_db),
user=Depends(get_current_user),
_=Depends(has_permission("maintenance", "WRITE")),
task_manager: TaskManager = Depends(get_task_manager),
):
with belief_scope("start_maintenance"):
# ── Validation ──
# Validate end_time > start_time
if request.end_time and request.start_time and request.end_time <= request.start_time:
raise HTTPException(
status_code=400,
detail="end_time must be after start_time",
)
# Validate times are within reasonable range (future ±1h grace period)
now = datetime.now(timezone.utc)
grace_threshold = 3600 # 1 hour in seconds
if request.start_time:
diff = (request.start_time - now).total_seconds() if request.start_time.tzinfo else \
(request.start_time.replace(tzinfo=timezone.utc) - now).total_seconds()
if diff < -grace_threshold:
raise HTTPException(
status_code=400,
detail="start_time is too far in the past (max 1h grace period)",
)
# Validate tables list length
if len(request.tables) > 100:
raise HTTPException(
status_code=400,
detail="Maximum 100 tables allowed",
)
# Sanitize message (HTML escape already done in rendering, but check for raw HTML)
message = request.message
if message:
import html as _html
sanitized = _html.escape(message)
if sanitized != message:
# Message contained HTML — accept it, it will be escaped at render time
pass
# ── Idempotency check ──
from ....services.maintenance import build_idempotency_key
sorted_tables = sorted(t.strip().lower() for t in request.tables if t.strip())
idem_tables = frozenset(sorted_tables)
idem_start = request.start_time.isoformat() if request.start_time else None
idem_end = request.end_time.isoformat() if request.end_time else None
# Find existing active event with same idempotency key
active_statuses = [
MaintenanceEventStatus.PENDING,
MaintenanceEventStatus.APPLYING,
MaintenanceEventStatus.ACTIVE,
MaintenanceEventStatus.PARTIAL,
]
existing = (
db.query(MaintenanceEvent)
.filter(
MaintenanceEvent.status.in_(active_statuses),
)
.all()
)
for ev in existing:
ev_tables = frozenset(
sorted(t.lower() for t in (ev.tables or []) if t)
)
ev_start = ev.start_time.isoformat() if ev.start_time else None
ev_end = ev.end_time.isoformat() if ev.end_time else None
if ev_tables == idem_tables and ev_start == idem_start and ev_end == idem_end:
# Same key → return already_active
app_logger.reflect(
"Idempotency hit — event already active",
extra={"existing_id": ev.id},
)
return MaintenanceAlreadyActiveResponse(
maintenance_id=ev.id,
status="already_active",
)
# ── Load settings for environment ──
settings = db.query(MaintenanceSettings).filter(
MaintenanceSettings.id == "default"
).first()
env_id = settings.target_environment_id if settings else ""
# ── Create event ──
event = MaintenanceEvent(
tables=sorted_tables,
start_time=request.start_time,
end_time=request.end_time,
message=message,
status=MaintenanceEventStatus.PENDING,
environment_id=env_id,
)
db.add(event)
db.flush()
# ── Dispatch TaskManager task ──
task = await task_manager.create_task(
plugin_id="maintenance_banner_apply",
params={
"operation": "start",
"event_id": event.id,
"environment_id": env_id,
"user": getattr(user, "username", "system"),
},
)
event.task_id = task.id
db.commit()
app_logger.reflect(
"Maintenance start dispatched",
extra={"event_id": event.id, "task_id": task.id},
)
return MaintenanceStartResponse(
task_id=task.id,
maintenance_id=event.id,
status="pending",
)
# #endregion start_maintenance
# ── POST /api/maintenance/{maintenance_id}/end ───────────────
# FR-006: Remove banners for a specific maintenance event
# FR-015: operator, admin (NOT viewer)
# #region end_maintenance [C:3] [TYPE Function]
# @BRIEF End a specific maintenance event by ID. Removes banners from affected dashboards.
# Always returns 202 with task_id. Idempotent: already_completed returns success.
# @RELATION DEPENDS_ON -> [has_permission("maintenance", "WRITE")]
@router.post(
"/{maintenance_id}/end",
status_code=status.HTTP_202_ACCEPTED,
response_model=MaintenanceEndResponse,
)
async def end_maintenance(
maintenance_id: str,
db: Session = Depends(get_db),
user=Depends(get_current_user),
_=Depends(has_permission("maintenance", "WRITE")),
task_manager: TaskManager = Depends(get_task_manager),
):
with belief_scope("end_maintenance", f"maintenance_id={maintenance_id}"):
# Check event exists
event = db.query(MaintenanceEvent).filter(
MaintenanceEvent.id == maintenance_id
).first()
if not event:
raise HTTPException(
status_code=404,
detail=f"Maintenance event {maintenance_id} not found",
)
# If already completed, return idempotent success
if event.status == MaintenanceEventStatus.COMPLETED:
app_logger.reflect(
"Event already completed, returning idempotent",
extra={"event_id": maintenance_id},
)
return MaintenanceEndResponse(
task_id=event.task_id or "",
status="already_completed",
)
# ── Dispatch TaskManager task ──
task = await task_manager.create_task(
plugin_id="maintenance_banner_apply",
params={
"operation": "end",
"event_id": maintenance_id,
"environment_id": event.environment_id,
"user": getattr(user, "username", "system"),
},
)
app_logger.reflect(
"Maintenance end dispatched",
extra={"event_id": maintenance_id, "task_id": task.id},
)
return MaintenanceEndResponse(
task_id=task.id,
status="pending",
)
# #endregion end_maintenance
# ── POST /api/maintenance/end-all ────────────────────────────
# FR-007: Bulk removal of all active banners
# FR-015: operator, admin (NOT viewer)
# #region end_all_maintenance [C:3] [TYPE Function]
# @BRIEF End all active maintenance events. Removes all banners from all dashboards.
# Always returns 202 with task_id.
# @RELATION DEPENDS_ON -> [has_permission("maintenance", "WRITE")]
@router.post(
"/end-all",
status_code=status.HTTP_202_ACCEPTED,
response_model=MaintenanceEndResponse,
)
async def end_all_maintenance(
db: Session = Depends(get_db),
user=Depends(get_current_user),
_=Depends(has_permission("maintenance", "WRITE")),
task_manager: TaskManager = Depends(get_task_manager),
):
with belief_scope("end_all_maintenance"):
# ── Dispatch TaskManager task ──
task = await task_manager.create_task(
plugin_id="maintenance_banner_apply",
params={
"operation": "end_all",
"user": getattr(user, "username", "system"),
},
)
app_logger.reflect(
"End-all maintenance dispatched",
extra={"task_id": task.id},
)
return MaintenanceEndResponse(
task_id=task.id,
status="pending",
)
# #endregion end_all_maintenance
# ── GET /api/maintenance/settings ────────────────────────────
# FR-009: Get maintenance settings configuration
# FR-015: operator, admin (NOT viewer)
# #region get_maintenance_settings [C:2] [TYPE Function]
# @BRIEF Get current maintenance settings configuration.
# @RELATION DEPENDS_ON -> [has_permission("maintenance", "READ")]
@router.get("/settings", response_model=MaintenanceSettingsResponse)
async def get_maintenance_settings(
db: Session = Depends(get_db),
_=Depends(has_permission("maintenance", "READ")),
):
with belief_scope("get_maintenance_settings"):
settings = db.query(MaintenanceSettings).filter(
MaintenanceSettings.id == "default"
).first()
if not settings:
raise HTTPException(
status_code=404,
detail="Maintenance settings not configured",
)
return MaintenanceSettingsResponse(
target_environment_id=settings.target_environment_id,
display_timezone=settings.display_timezone,
banner_template=settings.banner_template,
dashboard_scope=settings.dashboard_scope.value,
excluded_dashboard_ids=list(settings.excluded_dashboard_ids or []),
forced_dashboard_ids=list(settings.forced_dashboard_ids or []),
updated_at=settings.updated_at.isoformat() if settings.updated_at else None,
)
# #endregion get_maintenance_settings
# ── PUT /api/maintenance/settings ────────────────────────────
# FR-009: Update maintenance settings configuration
# FR-015: admin ONLY (NOT operator, NOT viewer)
# #region update_maintenance_settings [C:2] [TYPE Function]
# @BRIEF Update maintenance settings. All fields optional for partial update.
# admin role required per FR-015.
# @RELATION DEPENDS_ON -> [has_permission("admin:settings", "WRITE")]
@router.put("/settings", response_model=MaintenanceSettingsResponse)
async def update_maintenance_settings(
settings_data: MaintenanceSettingsUpdate,
db: Session = Depends(get_db),
_=Depends(has_permission("admin:settings", "WRITE")),
):
with belief_scope("update_maintenance_settings"):
settings = db.query(MaintenanceSettings).filter(
MaintenanceSettings.id == "default"
).first()
if not settings:
# Auto-create with defaults on first save (upsert behavior).
# Column-level defaults from the model apply for omitted fields.
settings = MaintenanceSettings(
id="default",
target_environment_id=settings_data.target_environment_id or "",
)
db.add(settings)
db.flush()
# Apply partial updates
update_data = settings_data.model_dump(exclude_unset=True)
if "target_environment_id" in update_data:
settings.target_environment_id = update_data["target_environment_id"]
if "display_timezone" in update_data:
settings.display_timezone = update_data["display_timezone"]
if "banner_template" in update_data:
settings.banner_template = update_data["banner_template"]
if "dashboard_scope" in update_data:
from ....models.maintenance import DashboardScope
settings.dashboard_scope = DashboardScope(update_data["dashboard_scope"])
if "excluded_dashboard_ids" in update_data:
settings.excluded_dashboard_ids = update_data["excluded_dashboard_ids"]
if "forced_dashboard_ids" in update_data:
settings.forced_dashboard_ids = update_data["forced_dashboard_ids"]
db.commit()
db.refresh(settings)
return MaintenanceSettingsResponse(
target_environment_id=settings.target_environment_id,
display_timezone=settings.display_timezone,
banner_template=settings.banner_template,
dashboard_scope=settings.dashboard_scope.value,
excluded_dashboard_ids=list(settings.excluded_dashboard_ids or []),
forced_dashboard_ids=list(settings.forced_dashboard_ids or []),
updated_at=settings.updated_at.isoformat() if settings.updated_at else None,
)
# #endregion update_maintenance_settings
# #endregion MaintenanceRoutesModule

View File

@@ -0,0 +1,156 @@
# #region MaintenanceSchemasModule [C:2] [TYPE Module] [SEMANTICS pydantic, schema, maintenance, request, response]
# @BRIEF Pydantic models for Maintenance Banner API: request/response schemas per data-model.md.
# @LAYER API
# @RELATION DEPENDS_ON -> [Pydantic]
# @INVARIANT All response schemas use the standard envelope shape: { status, data, error, meta }
from datetime import datetime
from pydantic import BaseModel, Field
# ── Request Schemas ──────────────────────────────────────────
# #region MaintenanceStartRequest [C:1] [TYPE Class]
# @BRIEF POST /api/maintenance/start request body.
class MaintenanceStartRequest(BaseModel):
tables: list[str] = Field(..., min_length=1, max_length=100)
start_time: datetime
end_time: datetime | None = None
message: str | None = Field(None, max_length=500)
# #endregion MaintenanceStartRequest
# #region MaintenanceSettingsUpdate [C:1] [TYPE Class]
# @BRIEF PUT /api/maintenance/settings request body — all fields optional for partial update.
class MaintenanceSettingsUpdate(BaseModel):
target_environment_id: str | None = None
display_timezone: str | None = None
banner_template: str | None = None
dashboard_scope: str | None = None
excluded_dashboard_ids: list[int] | None = None
forced_dashboard_ids: list[int] | None = None
# #endregion MaintenanceSettingsUpdate
# ── Response Schemas ─────────────────────────────────────────
# #region MaintenanceStartResponse [C:1] [TYPE Class]
# @BRIEF Response for /api/maintenance/start — always 202 with task_id.
class MaintenanceStartResponse(BaseModel):
task_id: str
maintenance_id: str
status: str # "pending"
# #endregion MaintenanceStartResponse
# #region MaintenanceDashboardResult [C:1] [TYPE Class]
# @BRIEF Describes a single dashboard affected by a maintenance operation.
class MaintenanceDashboardResult(BaseModel):
id: int
title: str
chart_id: int | None = None
# #endregion MaintenanceDashboardResult
# #region MaintenanceFailedDashboard [C:1] [TYPE Class]
# @BRIEF Describes a dashboard that failed during banner apply/removal.
class MaintenanceFailedDashboard(BaseModel):
id: int
title: str
error: str
# #endregion MaintenanceFailedDashboard
# #region MaintenanceTaskResult [C:1] [TYPE Class]
# @BRIEF Result payload for a completed maintenance start task.
class MaintenanceTaskResult(BaseModel):
maintenance_id: str
status: str # "active" | "partial" | "completed" | "no_match"
affected_dashboards: int = 0
dashboards: list[MaintenanceDashboardResult] = []
failed_dashboards: list[MaintenanceFailedDashboard] = []
unmatched_tables: list[str] = []
# #endregion MaintenanceTaskResult
# #region MaintenanceEndResponse [C:1] [TYPE Class]
# @BRIEF Response for /api/maintenance/{id}/end and /api/maintenance/end-all — always 202 with task_id.
class MaintenanceEndResponse(BaseModel):
task_id: str
status: str # "pending"
# #endregion MaintenanceEndResponse
# #region MaintenanceEndTaskResult [C:1] [TYPE Class]
# @BRIEF Result payload for a completed end maintenance task.
class MaintenanceEndTaskResult(BaseModel):
maintenance_id: str
removed_from: int = 0
failed_removals: list[MaintenanceFailedDashboard] = []
# #endregion MaintenanceEndTaskResult
# #region MaintenanceEndAllTaskResult [C:1] [TYPE Class]
# @BRIEF Result payload for a completed end-all maintenance task.
class MaintenanceEndAllTaskResult(BaseModel):
closed_events: int = 0
removed_from: int = 0
failed_removals: list[MaintenanceFailedDashboard] = []
# #endregion MaintenanceEndAllTaskResult
# #region MaintenanceSettingsResponse [C:1] [TYPE Class]
# @BRIEF Response for GET /api/maintenance/settings.
class MaintenanceSettingsResponse(BaseModel):
target_environment_id: str
display_timezone: str
banner_template: str
dashboard_scope: str
excluded_dashboard_ids: list[int]
forced_dashboard_ids: list[int]
updated_at: str | None = None
# #endregion MaintenanceSettingsResponse
# #region MaintenanceEventItem [C:1] [TYPE Class]
# @BRIEF Single maintenance event summary for GET /api/maintenance/events.
class MaintenanceEventItem(BaseModel):
id: str
tables: list[str]
start_time: str
end_time: str | None = None
message: str | None = None
status: str
affected_count: int
created_at: str
# #endregion MaintenanceEventItem
# #region MaintenanceEventListResponse [C:1] [TYPE Class]
# @BRIEF Response for GET /api/maintenance/events.
class MaintenanceEventListResponse(BaseModel):
active: list[MaintenanceEventItem]
completed: list[MaintenanceEventItem]
# #endregion MaintenanceEventListResponse
# #region MaintenanceDashboardBannerState [C:1] [TYPE Class]
# @BRIEF Per-dashboard banner state for GET /api/maintenance/dashboard-banners.
class MaintenanceDashboardBannerState(BaseModel):
dashboard_id: int
active: bool
events: list[str]
start_time: str | None = None
end_time: str | None = None
# #endregion MaintenanceDashboardBannerState
# #region MaintenanceAlreadyActiveResponse [C:1] [TYPE Class]
# @BRIEF Response for duplicate /start call — idempotency hit.
class MaintenanceAlreadyActiveResponse(BaseModel):
maintenance_id: str
status: str = "already_active"
# #endregion MaintenanceAlreadyActiveResponse
# #endregion MaintenanceSchemasModule

View File

@@ -393,6 +393,7 @@ class ConsolidatedSettingsResponse(BaseModel):
storage: dict
notifications: dict = {}
features: dict = {}
app_timezone: str = "Europe/Moscow"
# #endregion ConsolidatedSettingsResponse
@@ -463,6 +464,7 @@ async def get_consolidated_settings(
storage=config.settings.storage.dict(),
notifications=notifications_payload,
features=config.settings.features.model_dump(),
app_timezone=config.settings.app_timezone,
)
logger.reflect(
"Consolidated settings payload assembled",
@@ -526,6 +528,15 @@ async def update_consolidated_settings(
payload["notifications"] = settings_patch["notifications"]
config_manager.save_config(payload)
# Update app_timezone if provided
if "app_timezone" in settings_patch:
new_tz = settings_patch["app_timezone"]
from ...core.timezone import validate_timezone, invalidate_timezone_cache
if not validate_timezone(new_tz):
raise HTTPException(status_code=422, detail=f"Invalid IANA timezone: {new_tz}")
current_settings.app_timezone = new_tz
invalidate_timezone_cache()
config_manager.update_global_settings(current_settings)
return {"status": "success", "message": "Settings updated"}

View File

@@ -13,8 +13,9 @@ from sqlalchemy.orm import Session
from ...core.config_manager import ConfigManager
from ...core.database import get_db
from ...core.logger import logger
from ...core.scheduler import SchedulerService
from ...core.task_manager import TaskManager
from ...dependencies import get_config_manager, get_current_user, get_task_manager, has_permission
from ...dependencies import get_config_manager, get_current_user, get_scheduler_service, get_task_manager, has_permission
from ...schemas.auth import User
from ...schemas.validation import (
TriggerRunResponse,
@@ -88,11 +89,14 @@ async def create_task(
current_user: User = Depends(get_current_user),
_=Depends(has_permission("validation.task", "CREATE")),
service: ValidationTaskService = Depends(_get_task_service),
scheduler: SchedulerService = Depends(get_scheduler_service),
):
"""Create a new validation task with provider and environment validation."""
logger.reason(f"create_task — User: {current_user.username}, name: {payload.name}", extra={"src": "validation_routes"})
try:
return service.create_task(payload)
result = service.create_task(payload)
scheduler.reload_validation_policy(result.id)
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))
# #endregion create_task
@@ -114,8 +118,9 @@ async def get_task(
try:
task = task_service.get_task(task_id)
from ...schemas.validation import ValidationRunResponse
# Find recent runs via run_service using dashboard matching
# Find recent runs for this specific task via task_id
recent = run_service.list_runs(
task_id=task_id,
dashboard_id=task.dashboard_ids[0] if task.dashboard_ids else None,
environment_id=task.environment_id,
page=1,
@@ -140,11 +145,14 @@ async def update_task(
current_user: User = Depends(get_current_user),
_=Depends(has_permission("validation.task", "EDIT")),
service: ValidationTaskService = Depends(_get_task_service),
scheduler: SchedulerService = Depends(get_scheduler_service),
):
"""Update a validation task."""
logger.reason(f"update_task — Task: {task_id}, User: {current_user.username}", extra={"src": "validation_routes"})
try:
return service.update_task(task_id, payload)
result = service.update_task(task_id, payload)
scheduler.reload_validation_policy(result.id)
return result
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# #endregion update_task
@@ -160,11 +168,13 @@ async def delete_task(
current_user: User = Depends(get_current_user),
_=Depends(has_permission("validation.task", "DELETE")),
service: ValidationTaskService = Depends(_get_task_service),
scheduler: SchedulerService = Depends(get_scheduler_service),
):
"""Delete a validation task."""
logger.reason(f"delete_task — Task: {task_id}, User: {current_user.username}", extra={"src": "validation_routes"})
try:
service.delete_task(task_id, delete_runs=delete_runs)
scheduler.remove_validation_job(task_id)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# #endregion delete_task

View File

@@ -35,6 +35,7 @@ from .api.routes import (
git,
health,
llm,
maintenance,
mappings,
migration,
plugins,
@@ -219,6 +220,7 @@ app.add_middleware(TraceContextMiddleware)
# @RELATION DEPENDS_ON -> [LlmRoutes]
# @RELATION DEPENDS_ON -> [CleanReleaseV2Api]
# @RELATION DEPENDS_ON -> [ValidationRoutes]
# @RELATION DEPENDS_ON -> [MaintenanceRouter]
# Include API routes
app.include_router(auth.router)
app.include_router(admin.router)
@@ -242,6 +244,7 @@ app.include_router(dataset_review.router)
app.include_router(health.router)
app.include_router(translate.router)
app.include_router(validation.router, prefix="/api/validation", tags=["Validation"])
app.include_router(maintenance.maintenance_router)
# #endregion API_Routes
# #region api.include_routers [C:1] [TYPE Action] [SEMANTICS routes, registration, api]
# @BRIEF Registers all API routers with the FastAPI application.

View File

@@ -82,6 +82,14 @@ class ConfigManager:
"Applied FEATURES__HEALTH_MONITOR from env",
extra={"value": parsed, "raw": health_monitor_env},
)
app_timezone_env = os.getenv("APP_TIMEZONE")
if app_timezone_env is not None:
settings.app_timezone = app_timezone_env.strip()
logger.reason(
"Applied APP_TIMEZONE from env",
extra={"value": settings.app_timezone},
)
# #endregion _apply_features_from_env
# #region _default_config [TYPE Function]
# @PURPOSE: Build default application configuration fallback.

View File

@@ -5,7 +5,7 @@
# @RELATION IMPLEMENTS -> [ConnectionContracts]
from pydantic import BaseModel, Field
from pydantic import BaseModel, Field, field_validator
from ..models.storage import StorageConfig
from ..services.llm_prompt_templates import (
@@ -101,6 +101,19 @@ class GlobalSettings(BaseModel):
}
)
# Application timezone
app_timezone: str = "Europe/Moscow"
@field_validator("app_timezone")
@classmethod
def validate_app_timezone(cls, v: str) -> str:
from zoneinfo import ZoneInfo
try:
ZoneInfo(v)
except (KeyError, TypeError):
raise ValueError(f"Invalid IANA timezone: {v}") from None
return v
# Task retention settings
task_retention_days: int = 30
task_retention_limit: int = 100

View File

@@ -21,6 +21,7 @@ from ..models import (
config as _config_models, # noqa: F401
dataset_review as _dataset_review_models, # noqa: F401
llm as _llm_models, # noqa: F401
maintenance as _maintenance_models, # noqa: F401
profile as _profile_models, # noqa: F401
task as _task_models, # noqa: F401
)

View File

@@ -271,7 +271,10 @@ def explore(self, msg, *args, **kwargs):
Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.
"""
user_extra = kwargs.pop('extra', {})
error_val = kwargs.pop('error', None)
extra = {'marker': 'EXPLORE', 'intent': msg}
if error_val is not None:
extra['error'] = error_val
extra.update(user_extra)
self.warning(msg, *args, extra=extra, **kwargs)
# #endregion explore
@@ -285,7 +288,10 @@ def reason(self, msg, *args, **kwargs):
Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.
"""
user_extra = kwargs.pop('extra', {})
error_val = kwargs.pop('error', None)
extra = {'marker': 'REASON', 'intent': msg}
if error_val is not None:
extra['error'] = error_val
extra.update(user_extra)
self.debug(msg, *args, extra=extra, **kwargs)
# #endregion reason
@@ -299,7 +305,10 @@ def reflect(self, msg, *args, **kwargs):
Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.
"""
user_extra = kwargs.pop('extra', {})
error_val = kwargs.pop('error', None)
extra = {'marker': 'REFLECT', 'intent': msg}
if error_val is not None:
extra['error'] = error_val
extra.update(user_extra)
self.debug(msg, *args, extra=extra, **kwargs)
# #endregion reflect

View File

@@ -88,6 +88,41 @@ class SchedulerService:
except Exception as e:
logger.explore("Failed to load translation schedules on startup",
extra={"error": str(e)})
# Re-register active validation policies with schedules
try:
from ..models.llm import ValidationPolicy
db = SessionLocal()
try:
active_policies = (
db.query(ValidationPolicy)
.filter(
ValidationPolicy.is_active.is_(True),
ValidationPolicy.schedule_days.isnot(None),
)
.all()
)
for policy in active_policies:
schedule_days = policy.schedule_days or []
if not schedule_days:
continue
ws = policy.window_start
if ws is None:
continue
# Build cron: minute hour * * day-of-week
dow = ",".join(str(d) for d in sorted(schedule_days))
cron = f"{ws.minute} {ws.hour} * * {dow}"
self.add_validation_job(
policy_id=policy.id,
cron_expression=cron,
app_timezone=config.settings.app_timezone,
)
if active_policies:
logger.reason(f"Loaded {len(active_policies)} active validation schedule(s)")
finally:
db.close()
except Exception as e:
logger.explore("Failed to load validation schedules on startup",
extra={"error": str(e)})
# #endregion load_schedules
# #region add_backup_job [TYPE Function]
# @PURPOSE: Adds a scheduled backup job for an environment.
@@ -197,6 +232,153 @@ class SchedulerService:
self.loop,
)
# #endregion _trigger_backup
# #region add_validation_job [C:3] [TYPE Function] [SEMANTICS scheduler,validation,cron,apscheduler]
# @BRIEF Register a validation policy schedule with APScheduler.
# @PRE policy_id and cron_expression are valid strings.
# @POST A new APScheduler job is registered or replaced if it already exists.
# @SIDE_EFFECT Mutates APScheduler state; calls _trigger_validation on trigger.
# @RELATION DEPENDS_ON -> [_trigger_validation]
def add_validation_job(self, policy_id: str, cron_expression: str, app_timezone: str = "Europe/Moscow") -> None:
with belief_scope(
"SchedulerService.add_validation_job",
f"policy_id={policy_id}, cron={cron_expression}",
):
from zoneinfo import ZoneInfo
job_id_aps = f"validation_{policy_id}"
try:
tz = ZoneInfo(app_timezone)
self.scheduler.add_job(
self._trigger_validation,
CronTrigger.from_crontab(cron_expression, timezone=tz),
id=job_id_aps,
args=[policy_id],
replace_existing=True,
)
logger.reason(
f"Validation schedule registered: {job_id_aps} ({cron_expression}) [{app_timezone}]"
)
except Exception as e:
logger.explore(
f"Failed to register validation schedule: {job_id_aps}",
extra={"error": str(e)},
)
# #endregion add_validation_job
# #region remove_validation_job [C:2] [TYPE Function] [SEMANTICS scheduler,validation,remove,apscheduler]
# @BRIEF Remove a validation policy schedule from APScheduler.
# @PRE policy_id is a valid string.
# @POST The APScheduler job is removed if it exists; silently ignored otherwise.
# @SIDE_EFFECT Mutates APScheduler state.
def remove_validation_job(self, policy_id: str) -> None:
with belief_scope(
"SchedulerService.remove_validation_job",
f"policy_id={policy_id}",
):
job_id_aps = f"validation_{policy_id}"
try:
self.scheduler.remove_job(job_id_aps)
logger.info(f"Validation schedule removed: {job_id_aps}")
except Exception:
logger.reason(f"Validation schedule not found (already removed): {job_id_aps}")
# #endregion remove_validation_job
# #region reload_validation_policy [C:2] [TYPE Function] [SEMANTICS scheduler,validation,reload,apscheduler]
# @BRIEF Reload a single validation policy schedule — calls remove then add.
# @PRE policy_id is a valid ValidationPolicy id with schedule data in DB.
# @POST Old job is removed; new job is registered if policy is active and has schedule_days.
# @SIDE_EFFECT Mutates APScheduler state; queries ValidationPolicy from DB.
def reload_validation_policy(self, policy_id: str, app_timezone: str = "Europe/Moscow") -> None:
with belief_scope(
"SchedulerService.reload_validation_policy",
f"policy_id={policy_id}",
):
self.remove_validation_job(policy_id)
try:
from ..models.llm import ValidationPolicy
db = SessionLocal()
try:
policy = db.query(ValidationPolicy).filter(ValidationPolicy.id == policy_id).first()
if not policy:
logger.reason(f"Policy {policy_id} not found; schedule removed")
return
if not policy.is_active:
logger.reason(f"Policy {policy_id} is inactive; schedule removed")
return
schedule_days = policy.schedule_days or []
if not schedule_days or policy.window_start is None:
logger.reason(f"Policy {policy_id} has no schedule; nothing to register")
return
dow = ",".join(str(d) for d in sorted(schedule_days))
ws = policy.window_start
cron = f"{ws.minute} {ws.hour} * * {dow}"
self.add_validation_job(policy_id, cron, app_timezone=app_timezone)
finally:
db.close()
except Exception as e:
logger.explore(
f"Failed to reload validation policy {policy_id}",
extra={"error": str(e)},
)
# #endregion reload_validation_policy
# #region _trigger_validation [C:3] [TYPE Function] [SEMANTICS scheduler,validation,trigger]
# @BRIEF APScheduler job handler — triggers validation runs for a policy.
# @PRE policy_id is a valid ValidationPolicy id with dashboard_ids.
# @POST A validation task is spawned via TaskManager for each dashboard in the policy.
# @SIDE_EFFECT Creates TaskRecord entries for validation runs; logs progress.
def _trigger_validation(self, policy_id: str) -> None:
seed_trace_id()
with belief_scope("SchedulerService._trigger_validation", f"policy_id={policy_id}"):
logger.reason(f"Triggering scheduled validation for policy {policy_id}")
db = SessionLocal()
try:
from ..models.llm import ValidationPolicy
policy = db.query(ValidationPolicy).filter(ValidationPolicy.id == policy_id).first()
if not policy:
logger.explore(f"Validation policy not found: {policy_id}")
return
if not policy.is_active:
logger.reason(f"Validation policy {policy_id} is no longer active; skipping")
return
dashboard_ids = list(policy.dashboard_ids or [])
if not dashboard_ids:
logger.explore(f"Validation policy {policy_id} has no dashboards; skipping")
return
ws = policy.window_start
we = policy.window_end
today = date.today()
scheduled_times = ThrottledSchedulerConfigurator.calculate_schedule(
window_start=ws,
window_end=we or time(ws.hour, ws.minute + 30),
dashboard_ids=dashboard_ids,
current_date=today,
)
for idx, dash_id in enumerate(dashboard_ids):
sched_time = scheduled_times[idx] if idx < len(scheduled_times) else scheduled_times[-1]
params: dict = {
"dashboard_id": dash_id,
"environment_id": policy.environment_id,
"provider_id": policy.provider_id or "",
"policy_id": policy_id,
}
asyncio.run_coroutine_threadsafe(
self.task_manager.create_task(
plugin_id="llm_dashboard_validation", params=params
),
self.loop,
)
logger.reason(
f"Scheduled validation for dashboard {dash_id} at {sched_time.isoformat()}"
)
except Exception as e:
logger.explore(
f"Error triggering validation for policy {policy_id}",
extra={"error": str(e)},
)
finally:
db.close()
# #endregion _trigger_validation
# #endregion SchedulerService
# #region ThrottledSchedulerConfigurator [C:5] [TYPE Class] [SEMANTICS scheduler, throttling, distribution]
# @BRIEF Distributes validation tasks evenly within an execution window.

View File

@@ -25,6 +25,7 @@ from ._charts import SupersetChartsMixin
from ._dashboards_crud import SupersetDashboardsCrudMixin
from ._dashboards_filters import SupersetDashboardsFiltersMixin
from ._dashboards_list import SupersetDashboardsListMixin
from ._dashboards_write import SupersetDashboardsWriteMixin
from ._databases import SupersetDatabasesMixin
from ._datasets import SupersetDatasetsMixin
from ._datasets_preview import SupersetDatasetsPreviewMixin
@@ -47,6 +48,7 @@ from ._user_projection import SupersetUserProjectionMixin
# @RELATION INHERITS -> [SupersetDatasetsPreviewMixin]
# @RELATION INHERITS -> [SupersetDatasetsPreviewFiltersMixin]
# @RELATION INHERITS -> [SupersetDatabasesMixin]
# @RELATION INHERITS -> [SupersetDashboardsWriteMixin]
class SupersetClient(
SupersetDatabasesMixin,
SupersetDatasetsPreviewFiltersMixin,
@@ -56,6 +58,7 @@ class SupersetClient(
SupersetDashboardsCrudMixin,
SupersetDashboardsFiltersMixin,
SupersetDashboardsListMixin,
SupersetDashboardsWriteMixin,
SupersetUserProjectionMixin,
SupersetClientBase,
):

View File

@@ -0,0 +1,291 @@
# #region SupersetDashboardsWriteMixin [C:4] [TYPE Module] [SEMANTICS superset, dashboard, chart, write, maintenance, banner]
# @BRIEF Dashboard write mixin for SupersetClient — markdown chart CRUD and layout manipulation for maintenance banners.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [SupersetClientBase]
# @RELATION DEPENDS_ON -> [SupersetChartsMixin]
# @RELATION DEPENDS_ON -> [LayoutUtils]
# @SIDE_EFFECT Creates, updates, deletes markdown charts & modifies dashboard layout via Superset API.
# @DATA_CONTRACT Input: dashboard_id / chart_id / markdown_text → Output: API response dict or chart_id int
# @INVARIANT All network operations use self.network.request()
# @RATIONALE Extracted from monolithic superset_client to satisfy INV_7.
# Uses belief_scope with REASON/REFLECT/EXPLORE markers for all C4 operations per Std.Semantics.Python.
import json
from typing import Any, cast
from ..logger import belief_scope, logger as app_logger
from ._layout_utils import (
insert_banner_markdown_at_top,
parse_position_json,
remove_banner_from_position,
update_banner_markdown_content,
)
app_logger = cast(Any, app_logger)
# #region SupersetDashboardsWriteMixin [C:4] [TYPE Class]
# @BRIEF Mixin providing markdown chart CRUD and dashboard layout manipulation for maintenance banners.
# @RELATION DEPENDS_ON -> [SupersetClientBase]
# @RELATION DEPENDS_ON -> [SupersetChartsMixin]
class SupersetDashboardsWriteMixin:
"""Mixin for creating/updating/deleting markdown charts and manipulating dashboard layouts."""
# #region create_markdown_chart [C:3] [TYPE Function]
# @BRIEF Create a markdown chart and return the new chart ID.
# @PRE dashboard_id exists in Superset. markdown_text not empty.
# @POST Returns chart_id (int). Chart is NOT yet placed in dashboard layout.
# @SIDE_EFFECT Creates a new chart resource in Superset.
# @RELATION CALLS -> [self.network.request]
def create_markdown_chart(self, dashboard_id: int, markdown_text: str) -> int:
with belief_scope(
"SupersetDashboardsWriteMixin.create_markdown_chart",
f"dashboard_id={dashboard_id}",
):
app_logger.reason(
"Creating markdown chart",
extra={"dashboard_id": dashboard_id, "len": len(markdown_text)},
)
datasource_id = self._resolve_markdown_datasource(dashboard_id)
payload = {
"dashboards": [dashboard_id],
"datasource_id": datasource_id,
"datasource_type": "table",
"slice_name": "Maintenance Banner",
"viz_type": "markdown",
"params": json.dumps({
"markdown": markdown_text,
"viz_type": "markdown",
}),
}
try:
response = cast(dict, self.network.request(
method="POST", endpoint="/chart/", json=payload
))
chart_id = int(response.get("id") or response.get("result", {}).get("id", 0))
app_logger.reflect("Chart created", extra={"chart_id": chart_id})
return chart_id
except Exception as e:
app_logger.explore("Create chart failed", extra={"dashboard_id": dashboard_id}, error=str(e))
raise
# #endregion create_markdown_chart
# #region update_markdown_chart [C:3] [TYPE Function]
# @BRIEF Update the markdown text of an existing markdown chart.
# @PRE chart_id exists and is a markdown chart.
# @POST Chart markdown content updated.
# @SIDE_EFFECT Modifies a chart resource in Superset.
def update_markdown_chart(self, chart_id: int, markdown_text: str) -> dict:
with belief_scope(
"SupersetDashboardsWriteMixin.update_markdown_chart",
f"chart_id={chart_id}",
):
app_logger.reason("Updating markdown chart", extra={"chart_id": chart_id, "len": len(markdown_text)})
payload = {"params": json.dumps({"markdown": markdown_text, "viz_type": "markdown"})}
try:
response = cast(dict, self.network.request(method="PUT", endpoint=f"/chart/{chart_id}", json=payload))
app_logger.reflect("Chart updated", extra={"chart_id": chart_id})
return response
except Exception as e:
app_logger.explore("Update chart failed", extra={"chart_id": chart_id}, error=str(e))
raise
# #endregion update_markdown_chart
# #region update_dashboard_layout [C:4] [TYPE Function]
# @BRIEF Insert a native MARKDOWN element at (0,0) with full width (12 cols),
# shift existing charts down. Uses native MARKDOWN for reliable rendering.
# @PRE dashboard_id exists in Superset. chart_id is used for tracking/DB.
# @POST MARKDOWN element placed at top; existing items shifted down.
# @SIDE_EFFECT Modifies dashboard layout JSON.
def update_dashboard_layout(self, dashboard_id: int, chart_id: int) -> dict:
with belief_scope(
"SupersetDashboardsWriteMixin.update_dashboard_layout",
f"dashboard_id={dashboard_id}, chart_id={chart_id}",
):
app_logger.reason(
"Updating layout with native MARKDOWN",
extra={"dashboard_id": dashboard_id, "chart_id": chart_id},
)
try:
dashboard = cast(dict, self.network.request(
method="GET", endpoint=f"/dashboard/{dashboard_id}"
))
result = dashboard.get("result", dashboard)
position_json = parse_position_json(result.get("position_json", "{}"))
# Use native MARKDOWN element inside a ROW, connected to GRID_ID
placeholder = "*Maintenance banner*"
insert_banner_markdown_at_top(position_json, chart_id, placeholder)
response = cast(dict, self.network.request(
method="PUT", endpoint=f"/dashboard/{dashboard_id}",
json={"position_json": json.dumps(position_json)},
))
app_logger.reflect(
"Layout updated with native MARKDOWN",
extra={"dashboard_id": dashboard_id, "chart_id": chart_id},
)
return response
except Exception as e:
app_logger.explore(
"Layout update failed",
extra={"dashboard_id": dashboard_id},
error=str(e),
)
raise
# #endregion update_dashboard_layout
# #region remove_chart_from_layout [C:3] [TYPE Function]
# @BRIEF Remove banner markdown element from dashboard layout without deleting the chart.
# @PRE dashboard_id exists. chart_id exists in layout as MARKDOWN-banner-{chart_id}.
# @POST MARKDOWN element removed from position_json; items shifted up.
# @SIDE_EFFECT Modifies dashboard layout JSON.
def remove_chart_from_layout(self, dashboard_id: int, chart_id: int) -> dict:
with belief_scope(
"SupersetDashboardsWriteMixin.remove_chart_from_layout",
f"dashboard_id={dashboard_id}, chart_id={chart_id}",
):
app_logger.reason(
"Removing banner from layout",
extra={"dashboard_id": dashboard_id, "chart_id": chart_id},
)
try:
dashboard = cast(dict, self.network.request(
method="GET", endpoint=f"/dashboard/{dashboard_id}"
))
result = dashboard.get("result", dashboard)
position_json = parse_position_json(result.get("position_json", "{}"))
# Remove new MARKDOWN+ROW pair (old CHART-{id} format no longer created)
remove_banner_from_position(position_json, chart_id)
response = cast(dict, self.network.request(
method="PUT", endpoint=f"/dashboard/{dashboard_id}",
json={"position_json": json.dumps(position_json)},
))
app_logger.reflect(
"Banner removed from layout",
extra={"dashboard_id": dashboard_id},
)
return response
except Exception as e:
app_logger.explore(
"Remove from layout failed",
extra={"dashboard_id": dashboard_id},
error=str(e),
)
raise
# #endregion remove_chart_from_layout
# #region delete_chart [C:3] [TYPE Function]
# @BRIEF Delete a chart from Superset by ID.
# @PRE chart_id exists.
# @POST Chart permanently deleted from Superset.
# @SIDE_EFFECT Destroys chart resource.
def delete_chart(self, chart_id: int) -> dict:
with belief_scope("SupersetDashboardsWriteMixin.delete_chart", f"chart_id={chart_id}"):
app_logger.reason("Deleting chart", extra={"chart_id": chart_id})
try:
response = cast(dict, self.network.request(method="DELETE", endpoint=f"/chart/{chart_id}"))
app_logger.reflect("Chart deleted", extra={"chart_id": chart_id})
return response
except Exception as e:
app_logger.explore("Delete chart failed", extra={"chart_id": chart_id}, error=str(e))
raise
# #endregion delete_chart
# #region update_banner_on_dashboard [C:3] [TYPE Function]
# @BRIEF Update the content of a native MARKDOWN banner element on a dashboard.
# @PRE dashboard_id exists. chart_id used for key lookup.
# @POST MARKDOWN element content updated on dashboard.
# @SIDE_EFFECT Modifies dashboard layout JSON in Superset.
def update_banner_on_dashboard(
self, dashboard_id: int, chart_id: int, content: str
) -> dict:
with belief_scope(
"SupersetDashboardsWriteMixin.update_banner_on_dashboard",
f"dashboard_id={dashboard_id}, chart_id={chart_id}",
):
app_logger.reason(
"Updating banner content on dashboard",
extra={"dashboard_id": dashboard_id, "chart_id": chart_id, "len": len(content)},
)
try:
dashboard = cast(dict, self.network.request(
method="GET", endpoint=f"/dashboard/{dashboard_id}"
))
result = dashboard.get("result", dashboard)
position_json = parse_position_json(result.get("position_json", "{}"))
markdown_key = f"MARKDOWN-banner-{chart_id}"
update_banner_markdown_content(position_json, markdown_key, content)
response = cast(dict, self.network.request(
method="PUT", endpoint=f"/dashboard/{dashboard_id}",
json={"position_json": json.dumps(position_json)},
))
app_logger.reflect(
"Banner content updated on dashboard",
extra={"dashboard_id": dashboard_id, "chart_id": chart_id},
)
return response
except Exception as e:
app_logger.explore(
"Update banner content failed",
extra={"dashboard_id": dashboard_id},
error=str(e),
)
raise
# #endregion update_banner_on_dashboard
# #region get_dashboard_layout [C:3] [TYPE Function]
# @BRIEF Fetch the position_json layout of a dashboard.
# @PRE dashboard_id exists.
# @POST Returns the dashboard layout dict.
def get_dashboard_layout(self, dashboard_id: int) -> dict:
with belief_scope("SupersetDashboardsWriteMixin.get_dashboard_layout", f"dashboard_id={dashboard_id}"):
app_logger.reason("Fetching layout", extra={"dashboard_id": dashboard_id})
try:
dashboard = cast(dict, self.network.request(method="GET", endpoint=f"/dashboard/{dashboard_id}"))
result = dashboard.get("result", dashboard)
layout = parse_position_json(result.get("position_json", "{}"))
app_logger.reflect("Layout fetched", extra={"dashboard_id": dashboard_id, "items": len(layout)})
return layout
except Exception as e:
app_logger.explore("Fetch layout failed", extra={"dashboard_id": dashboard_id}, error=str(e))
raise
# #endregion get_dashboard_layout
# #region _resolve_markdown_datasource [C:2] [TYPE Function]
# @BRIEF Find a valid datasource_id for markdown chart creation.
# @PRE dashboard_id exists.
# @POST Returns an integer datasource_id.
def _resolve_markdown_datasource(self, dashboard_id: int) -> int:
with belief_scope("_resolve_markdown_datasource", f"dashboard_id={dashboard_id}"):
try:
datasets_response = cast(dict, self.network.request(
method="GET", endpoint=f"/dashboard/{dashboard_id}/datasets"
))
datasets = datasets_response.get("result", [])
if datasets:
ds_id = datasets[0].get("id")
if ds_id:
return int(ds_id)
except Exception:
app_logger.explore("Dashboard datasets fetch failed, falling back",
extra={"dashboard_id": dashboard_id}, error="Dashboard datasets endpoint failed")
try:
all_ds = cast(dict, self.network.request(
method="GET", endpoint="/dataset/",
params={"q": json.dumps({"columns": ["id"], "page_size": 1, "page": 0})},
))
ds_list = all_ds.get("result", [])
if ds_list:
ds_id = ds_list[0].get("id")
if ds_id:
return int(ds_id)
except Exception:
app_logger.explore("No datasets available", extra={}, error="Global dataset list failed")
raise RuntimeError("Cannot create markdown chart: no datasets available in Superset.")
# #endregion _resolve_markdown_datasource
# #endregion SupersetDashboardsWriteMixin
# #endregion SupersetDashboardsWriteMixin

View File

@@ -146,7 +146,13 @@ class SupersetDatasetsMixin:
dashboards_data = related_objects["result"].get("dashboards", [])
else:
dashboards_data = []
for dash in dashboards_data:
# related_objects returns {"count": N, "result": [...]} — extract the array
dashboards_list = (
dashboards_data.get("result", [])
if isinstance(dashboards_data, dict)
else dashboards_data
)
for dash in dashboards_list:
if isinstance(dash, dict):
dash_id = dash.get("id")
if dash_id is None:

View File

@@ -0,0 +1,197 @@
# #region LayoutUtils [C:2] [TYPE Module] [SEMANTICS superset, dashboard, layout, position, json]
# @BRIEF Utility functions for manipulating Superset dashboard position_json layout.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [json]
# @RELATION DEPENDS_ON -> [SupersetDashboardsWriteMixin]
# @RATIONALE Extracted from SupersetDashboardsWriteMixin to stay under INV_7 400 LOC.
import json
import re
from typing import Any
# #region parse_position_json [C:1] [TYPE Function]
# @BRIEF Parse position_json from a dashboard API response (may be string or dict).
# @PRE raw_position is a string, dict, or None.
# @POST Returns a dict.
def parse_position_json(raw_position: Any) -> dict:
if isinstance(raw_position, str):
return json.loads(raw_position) if raw_position.strip() else {}
if isinstance(raw_position, dict):
return raw_position
return {}
# #endregion parse_position_json
# #region _estimate_markdown_height [C:1] [TYPE Function]
# @BRIEF Estimate grid height for a MARKDOWN element based on HTML content.
# Counts block-level tags and newlines to estimate visual line count,
# then converts to Superset grid units (1 unit ≈ 2 text lines + padding).
# @PRE content is a string of HTML/Markdown.
# @POST Returns int between 19 and 200 inclusive.
def _estimate_markdown_height(content: str) -> int:
"""Estimate MARKDOWN element height (position JSON units).
Model: total = CSS min-height(40px) + content pixel height.
Content height = text_lines × line-height(22px) + container padding.
1 position JSON unit ≈ 8 rendered pixels.
Reference: user's banner (5 text lines, padding:12) → height=21 → ~168px.
"""
if not content:
return 19
# Detect container padding (e.g. padding:12 → 24px vertical)
pad_match = re.search(r'padding\s*:\s*(\d+)', content)
container_padding = int(pad_match.group(1)) * 2 if pad_match else 0
# Strip HTML to get plain text
plain_text = re.sub(r'<[^>]+>', '', content)
plain_text = re.sub(r'\s+', ' ', plain_text).strip()
if not plain_text:
return 19
# Text lines (wrapping at ~40 chars/line at 12col width)
text_lines = len(plain_text) // 40 + 1
# Content height (px): line-height(~22px) × lines + container padding
content_px = text_lines * 22 + container_padding
# Total: CSS min-height(40px) + content
total_px = 40 + content_px
# Convert to position JSON units (÷8)
height = max(19, total_px // 8)
return min(200, height)
# #endregion _estimate_markdown_height
# #region _generate_banner_id [C:1] [TYPE Function]
# @BRIEF Generate a deterministic row and markdown key for a banner chart.
def _generate_banner_id(chart_id: int) -> tuple[str, str]:
row_key = f"ROW-banner-{chart_id}"
md_key = f"MARKDOWN-banner-{chart_id}"
return row_key, md_key
# #endregion _generate_banner_id
# #region insert_banner_markdown_at_top [C:2] [TYPE Function]
# @BRIEF Insert a ROW + native MARKDOWN pair at the top of the dashboard grid.
# Creates a ROW entry connected to GRID_ID, and a MARKDOWN child inside it.
# Shifts all existing grid ROWs down by 2 rows.
# @PRE position_json is a mutable dict. content is the HTML/markdown string.
# chart_id is the database chart_id for key generation.
# @POST position_json is mutated; ROW + MARKDOWN inserted at top of GRID.
def insert_banner_markdown_at_top(
position_json: dict, chart_id: int, content: str
) -> dict:
row_key, md_key = _generate_banner_id(chart_id)
# Shift existing ROW y-positions down
for key, value in list(position_json.items()):
if isinstance(value, dict):
meta = value.get("meta", {})
if isinstance(meta, dict):
cur_y = meta.get("y", 0)
if isinstance(cur_y, (int, float)):
meta["y"] = cur_y + 2
# Add MARKDOWN entry with adaptive height
height = _estimate_markdown_height(content)
position_json[md_key] = {
"type": "MARKDOWN",
"id": md_key,
"children": [],
"parents": ["ROOT_ID", "GRID_ID", row_key],
"meta": {
"code": content,
"width": 12,
"height": height,
"y": 0,
},
}
# Add ROW entry connected to GRID_ID
position_json[row_key] = {
"type": "ROW",
"id": row_key,
"children": [md_key],
"parents": ["ROOT_ID", "GRID_ID"],
"meta": {
"background": "BACKGROUND_TRANSPARENT",
"y": 0,
},
}
# Insert ROW at beginning of GRID_ID children
grid = position_json.get("GRID_ID")
if grid and isinstance(grid, dict):
grid_children = grid.get("children", [])
if isinstance(grid_children, list):
if row_key in grid_children:
grid_children.remove(row_key)
grid_children.insert(0, row_key)
return position_json
# #endregion insert_banner_markdown_at_top
# #region update_banner_markdown_content [C:1] [TYPE Function]
# @BRIEF Update the code content and adaptive height of an existing banner markdown element.
# @PRE position_json has markdown_key. content is the new HTML/markdown string.
# @POST position_json is mutated; markdown content and height updated.
def update_banner_markdown_content(
position_json: dict, markdown_key: str, content: str
) -> dict:
if markdown_key in position_json:
meta = position_json[markdown_key].get("meta", {})
if isinstance(meta, dict):
meta["code"] = content
meta["height"] = _estimate_markdown_height(content)
return position_json
# #endregion update_banner_markdown_content
# #region remove_banner_from_position [C:2] [TYPE Function]
# @BRIEF Remove the banner ROW + MARKDOWN pair from the position dict and GRID children.
# Shifts remaining items up by 2 if the banner was at y=0.
# @PRE position_json is a mutable dict. chart_id identifies the banner.
# @POST position_json is mutated; ROW and MARKDOWN entries removed.
def remove_banner_from_position(position_json: dict, chart_id: int) -> dict:
row_key, md_key = _generate_banner_id(chart_id)
removed_y = None
# Remove MARKDOWN entry
if md_key in position_json:
markdown_data = position_json[md_key]
if isinstance(markdown_data, dict):
meta = markdown_data.get("meta", {})
if isinstance(meta, dict):
removed_y = meta.get("y", 0)
del position_json[md_key]
# Remove ROW entry
if row_key in position_json:
del position_json[row_key]
# Remove ROW from GRID_ID children
grid = position_json.get("GRID_ID")
if grid and isinstance(grid, dict):
grid_children = grid.get("children", [])
if isinstance(grid_children, list) and row_key in grid_children:
grid_children.remove(row_key)
# Shift remaining items up
if removed_y is not None and removed_y == 0:
for key, value in position_json.items():
if isinstance(value, dict):
meta = value.get("meta", {})
if isinstance(meta, dict):
cur_y = meta.get("y", 0)
if isinstance(cur_y, (int, float)) and cur_y > 0:
meta["y"] = cur_y - 2
return position_json
# #endregion remove_banner_from_position
# #endregion LayoutUtils

View File

@@ -0,0 +1,91 @@
# #region AppTimezone [C:3] [TYPE Module] [SEMANTICS core,timezone,utilities,datetime]
# @BRIEF Application-level timezone utilities. Reads APP_TIMEZONE from env (default Europe/Moscow)
# and provides helpers for converting UTC datetimes to the configured timezone.
# @RELATION CALLED_BY -> [ConfigManager]
# @RELATION DEPENDS_ON -> [GlobalSettings.app_timezone]
# @RATIONALE Centralised timezone resolution avoids scattering ZoneInfo() calls across the codebase.
# All API responses should emit localised timestamps; internal storage remains UTC.
# @REJECTED Storing non-UTC in the database rejected — UTC is the canonical best practice for
# persistence; timezone conversion is a presentation-layer concern.
from datetime import datetime
import os
from zoneinfo import ZoneInfo
# #region _get_default_tz_name [C:1] [TYPE Function]
# @BRIEF Read APP_TIMEZONE from env, fall back to Europe/Moscow.
# @PRE: Environment is loaded (dotenv or os.environ).
# @POST: Returns a valid IANA timezone string.
def _get_default_tz_name() -> str:
return os.getenv("APP_TIMEZONE", "Europe/Moscow")
# #endregion _get_default_tz_name
# #region get_app_timezone [C:1] [TYPE Function]
# @BRIEF Return cached ZoneInfo for the configured application timezone.
# @SIDE_EFFECT: Reads os.environ on first call; result is cached.
# @POST: Returns a ZoneInfo instance matching APP_TIMEZONE env var.
_APP_TZ_CACHE: ZoneInfo | None = None
def get_app_timezone() -> ZoneInfo:
global _APP_TZ_CACHE
if _APP_TZ_CACHE is None:
_APP_TZ_CACHE = ZoneInfo(_get_default_tz_name())
return _APP_TZ_CACHE
# #endregion get_app_timezone
# #region invalidate_timezone_cache [C:1] [TYPE Function]
# @BRIEF Reset the cached ZoneInfo so the next get_app_timezone() call re-reads from env/DB.
# @POST: _APP_TZ_CACHE is set to None; subsequent get_app_timezone() will re-resolve.
# @RATIONALE: Called after PATCH /settings/consolidated updates app_timezone.
def invalidate_timezone_cache() -> None:
global _APP_TZ_CACHE
_APP_TZ_CACHE = None
# #endregion invalidate_timezone_cache
# #region validate_timezone [C:1] [TYPE Function]
# @BRIEF Validate that a timezone string is a known IANA timezone.
# @PRE: tz_name is a string.
# @POST: Returns True if ZoneInfo accepts the name, False otherwise.
def validate_timezone(tz_name: str) -> bool:
try:
ZoneInfo(tz_name)
return True
except (KeyError, TypeError):
return False
# #endregion validate_timezone
# #region localize [C:1] [TYPE Function]
# @BRIEF Convert a timezone-aware or naive UTC datetime to the configured app timezone.
# @PRE: If dt is timezone-naive, it is assumed to be UTC.
# @POST: Returns a datetime with the app timezone attached (astimezone).
def localize(dt: datetime | None) -> datetime | None:
if dt is None:
return None
if dt.tzinfo is None:
dt = dt.replace(tzinfo=ZoneInfo("UTC"))
return dt.astimezone(get_app_timezone())
# #endregion localize
# #region now [C:1] [TYPE Function]
# @BRIEF Get current time in the configured application timezone.
# @POST: Returns timezone-aware datetime in the app timezone.
def now() -> datetime:
return datetime.now(get_app_timezone())
# #endregion now
# #region format_timezone_offset [C:1] [TYPE Function]
# @BRIEF Return the UTC offset string for the configured timezone (e.g. "+03:00").
# @POST: Returns string like "+03:00" or "+00:00".
def format_timezone_offset() -> str:
offset = now().strftime("%z")
if offset:
return f"{offset[:3]}:{offset[3:]}"
return "+00:00"
# #endregion format_timezone_offset
# #endregion AppTimezone

View File

@@ -0,0 +1,215 @@
# #region MaintenanceModels [C:2] [TYPE Module] [SEMANTICS sqlalchemy, maintenance, banner, event, settings, model]
# @BRIEF SQLAlchemy models for the Maintenance Banner feature: event, banner, dashboard state, and settings.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [sqlalchemy]
# @RELATION DEPENDS_ON -> [MappingBase]
# @INVARIANT MaintenanceDashboardBanner has unique partial index on (environment_id, dashboard_id) WHERE status='active'
# @INVARIANT MaintenanceSettings is a singleton row (id='default') enforced by CheckConstraint
import enum
import uuid
from sqlalchemy import (
CheckConstraint,
Column,
DateTime,
Enum as SQLEnum,
ForeignKey,
Index,
Integer,
JSON,
String,
Text,
)
from sqlalchemy.orm import relationship
from sqlalchemy.sql import func
from .mapping import Base
# #region MaintenanceEventStatus [C:1] [TYPE Class]
class MaintenanceEventStatus(str, enum.Enum):
PENDING = "pending"
APPLYING = "applying"
ACTIVE = "active"
PARTIAL = "partial"
ENDING = "ending"
COMPLETED = "completed"
FAILED = "failed"
# #endregion MaintenanceEventStatus
# #region MaintenanceDashboardBannerStatus [C:1] [TYPE Class]
class MaintenanceDashboardBannerStatus(str, enum.Enum):
ACTIVE = "active"
REMOVED = "removed"
# #endregion MaintenanceDashboardBannerStatus
# #region MaintenanceDashboardStateStatus [C:1] [TYPE Class]
class MaintenanceDashboardStateStatus(str, enum.Enum):
PENDING_APPLY = "pending_apply"
ACTIVE = "active"
REMOVING = "removing"
REMOVED = "removed"
APPLY_FAILED = "apply_failed"
REMOVAL_FAILED = "removal_failed"
# #endregion MaintenanceDashboardStateStatus
# #region DashboardScope [C:1] [TYPE Class]
class DashboardScope(str, enum.Enum):
PUBLISHED_ONLY = "published_only"
DRAFT_ONLY = "draft_only"
ALL = "all"
# #endregion DashboardScope
# #region MaintenanceEvent [C:1] [TYPE Class]
# @BRIEF A record of a maintenance event with table list, time window, status, and linked dashboard states.
# @RELATION DEPENDS_ON -> [MaintenanceDashboardState]
class MaintenanceEvent(Base):
__tablename__ = "maintenance_events"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()), index=True)
task_id = Column(String, nullable=True, index=True)
environment_id = Column(String, nullable=False)
tables = Column(JSON, nullable=False)
start_time = Column(DateTime(timezone=True), nullable=False)
end_time = Column(DateTime(timezone=True), nullable=True)
message = Column(Text, nullable=True)
status = Column(
SQLEnum(MaintenanceEventStatus),
nullable=False,
default=MaintenanceEventStatus.PENDING,
index=True,
)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)
dashboard_states = relationship(
"MaintenanceDashboardState",
back_populates="maintenance_event",
cascade="all, delete-orphan",
)
# #endregion MaintenanceEvent
# #region MaintenanceDashboardBanner [C:1] [TYPE Class]
# @BRIEF The canonical "one chart per dashboard" entity. Unique partial index enforces single active banner per dashboard.
# @INVARIANT Unique partial index (environment_id, dashboard_id) WHERE status='active'
class MaintenanceDashboardBanner(Base):
__tablename__ = "maintenance_dashboard_banners"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()), index=True)
environment_id = Column(String, nullable=False)
dashboard_id = Column(Integer, nullable=False, index=True)
chart_id = Column(Integer, nullable=True)
banner_text = Column(Text, nullable=True)
status = Column(
SQLEnum(MaintenanceDashboardBannerStatus),
nullable=False,
default=MaintenanceDashboardBannerStatus.ACTIVE,
)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)
dashboard_states = relationship(
"MaintenanceDashboardState", back_populates="banner"
)
__table_args__ = (
Index(
"ix_banner_unique_active",
"environment_id",
"dashboard_id",
unique=True,
postgresql_where=(status == "active"),
),
)
# #endregion MaintenanceDashboardBanner
# #region MaintenanceDashboardState [C:1] [TYPE Class]
# @BRIEF Links a maintenance event to a dashboard via a banner. Tracks per-dashboard-per-event lifecycle.
# @RELATION DEPENDS_ON -> [MaintenanceEvent]
# @RELATION DEPENDS_ON -> [MaintenanceDashboardBanner]
class MaintenanceDashboardState(Base):
__tablename__ = "maintenance_dashboard_states"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()), index=True)
event_id = Column(
String, ForeignKey("maintenance_events.id"), nullable=False, index=True
)
dashboard_id = Column(Integer, nullable=False, index=True)
banner_id = Column(
String,
ForeignKey("maintenance_dashboard_banners.id"),
nullable=True,
)
status = Column(
SQLEnum(MaintenanceDashboardStateStatus),
nullable=False,
default=MaintenanceDashboardStateStatus.PENDING_APPLY,
)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)
maintenance_event = relationship(
"MaintenanceEvent", back_populates="dashboard_states"
)
banner = relationship(
"MaintenanceDashboardBanner", back_populates="dashboard_states"
)
__table_args__ = (
Index("ix_mds_dashboard_status", "dashboard_id", "status"),
)
# #endregion MaintenanceDashboardState
# #region MaintenanceSettings [C:1] [TYPE Class]
# @BRIEF Single-row maintenance mode configuration. Enforces singleton via CheckConstraint(id='default').
# @INVARIANT id must always be 'default' — enforced by CheckConstraint
class MaintenanceSettings(Base):
__tablename__ = "maintenance_settings"
id = Column(String, primary_key=True, default="default")
target_environment_id = Column(String, nullable=False)
display_timezone = Column(String, nullable=False, default="UTC")
banner_template = Column(
Text,
nullable=False,
default=(
'<div style="background:#FFF3E0;padding:16px;border-left:4px solid #FF9800;border-radius:4px">\n\n'
"## ⚠️ Технические работы\n\n"
"{message}\n\n"
"**Начало:** {start_time}\n"
"**Конец:** {end_time}\n\n"
"*Данные могут быть неполными или временно недоступны.*\n\n"
"</div>"
),
)
dashboard_scope = Column(
SQLEnum(DashboardScope),
nullable=False,
default=DashboardScope.PUBLISHED_ONLY,
)
excluded_dashboard_ids = Column(JSON, nullable=False, default=list)
forced_dashboard_ids = Column(JSON, nullable=False, default=list)
updated_at = Column(
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)
__table_args__ = (
CheckConstraint("id = 'default'", name="ck_settings_singleton"),
)
# #endregion MaintenanceSettings
# #endregion MaintenanceModels

View File

@@ -14,7 +14,7 @@ class FileCategory(str, Enum):
# #region StorageConfig [C:1] [TYPE Class]
# @BRIEF Configuration model for the storage system, defining paths and naming patterns.
class StorageConfig(BaseModel):
root_path: str = Field(default="backups", description="Absolute path to the storage root directory.")
root_path: str = Field(default="/app/storage", description="Absolute path to the storage root directory.")
backup_path: str = Field(default="backups", description="Subpath for backups.")
repo_path: str = Field(default="repositories", description="Subpath for repositories.")
backup_structure_pattern: str = Field(default="{category}/", description="Pattern for backup directory structure.")

View File

@@ -35,32 +35,22 @@ class GitPlugin(PluginBase):
# region __init__ [TYPE Function]
# @PURPOSE: Инициализирует плагин и его зависимости.
# @PRE: config.json exists or shared config_manager is available.
# @PRE: shared config_manager доступен через src.dependencies.
# @POST: Инициализированы git_service и config_manager.
def __init__(self):
with belief_scope("GitPlugin.__init__"):
app_logger.info("Initializing GitPlugin.")
self.git_service = GitService()
# Robust config path resolution:
# 1. Try absolute path from src/dependencies.py style if possible
# 2. Try relative paths based on common execution patterns
if os.path.exists("../config.json"):
config_path = "../config.json"
elif os.path.exists("config.json"):
config_path = "config.json"
else:
# Fallback to the one initialized in dependencies if we can import it
try:
from src.dependencies import config_manager
self.config_manager = config_manager
app_logger.info("GitPlugin initialized using shared config_manager.")
return
except Exception:
config_path = "config.json"
self.config_manager = ConfigManager(config_path)
app_logger.info(f"GitPlugin initialized with {config_path}")
# Используем shared config_manager из dependencies
try:
from src.dependencies import config_manager
self.config_manager = config_manager
app_logger.info("GitPlugin initialized using shared config_manager.")
except Exception as exc:
app_logger.error(f"GitPlugin: failed to get shared config_manager: {exc}")
self.config_manager = ConfigManager()
app_logger.info("GitPlugin initialized with fallback ConfigManager.")
# endregion __init__
@property

View File

@@ -13,6 +13,7 @@ import base64
import io
import json
import os
import re
import shutil
import tempfile
from typing import Any
@@ -347,16 +348,19 @@ class ScreenshotService:
# endregion ScreenshotService._save_debug_screenshot
# region ScreenshotService.capture_dashboard [TYPE Function] [C:4]
# @PURPOSE Captures a full-page screenshot of a dashboard using Playwright and CDP.
# @PURPOSE Captures a full-page screenshot of a dashboard using Playwright and CDP, plus diagnostic viewport screenshots of tabs that had chart load errors.
# @PRE dashboard_id is a valid string, output_path is a writable path.
# @POST Returns True if screenshot is saved successfully. Debug temp dir cleaned up on success, preserved on failure.
# @SIDE_EFFECT Launches a browser, performs UI login, switches tabs, writes a PNG file; creates and cleans up temp debug directory.
# Extra diagnostic screenshots saved as output_path__tab_{tab_text}.png for each tab with chart errors.
# @SIDE_EFFECT Launches a browser, performs UI login, switches tabs, writes PNG files; creates and cleans up temp debug directory.
# @UX_STATE [Navigating] -> Loading dashboard UI
# @UX_STATE [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading
# @UX_STATE [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading.
# Collects errored_tabs — those where chart content fails to render within timeout.
# @UX_STATE [CalculatingHeight] -> Determining dashboard dimensions
# @UX_STATE [Capturing] -> Executing CDP screenshot
# @RATIONALE Uses tempfile.mkdtemp() for all debug screenshots; cleans up on success (shutil.rmtree), preserves on failure for diagnostic analysis. Pre-resize screenshot is saved to temp dir and auto-cleaned on success.
# @REJECTED Old approach saved debug .png files alongside output — accumulated permanently (F1). In-memory-only logging rejected — visual state cannot be captured in text.
# @UX_STATE [CapturingErroredTabs] -> Taking viewport screenshots of tabs with chart errors.
# @RATIONALE Errored tab screenshots give LLM diagnostic context — an error text description ("charts.length = 0") is not enough to diagnose a chart load failure; a visual screenshot shows whether the tab is empty, has a loading spinner, or renders a Superset error card. Uses tempfile.mkdtemp() for all debug screenshots; cleans up on success (shutil.rmtree), preserves on failure for diagnostic analysis.
# @REJECTED Always-screenshot-all-tabs rejected — ~20 tabs x 5s per screenshot = 100s overhead, most tabs healthy. Only errored + main gives diagnostic value at near-zero cost. Old approach saved debug .png files alongside output — accumulated permanently (F1). In-memory-only logging rejected — visual state cannot be captured in text.
async def capture_dashboard(self, dashboard_id: str, output_path: str) -> bool:
debug_dir = tempfile.mkdtemp(prefix="ss_screenshot_debug_")
success = False
@@ -587,12 +591,15 @@ class ScreenshotService:
try:
async with asyncio.timeout(SCREENSHOT_SERVICE_TIMEOUT_MS / 1000):
# 1. Handle Tabs (Recursive switching)
# @UX_STATE [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading
# @UX_STATE [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading.
# Collects errored_tabs — those where chart content fails to render within timeout.
# These are later screenshotted individually for LLM diagnostic value.
processed_tabs = set()
errored_tabs = [] # Collects {tab_text, depth, error} for tabs with chart load failures
async def switch_tabs(depth=0):
if depth > 3:
return # Limit recursion depth
return # Limit recursion depth
tab_selectors = [
'.ant-tabs-nav-list .ant-tabs-tab',
@@ -635,10 +642,22 @@ class ScreenshotService:
}""", timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS)
except Exception:
logger.warning(f"[TabSwitching] Content verification timed out for tab: {tab_text}")
errored_tabs.append({
"tab_text": tab_text,
"tab_index": i,
"depth": depth,
"error": "content_verification_timeout"
})
await switch_tabs(depth + 1)
except Exception as tab_e:
logger.warning(f"[DEBUG][TabSwitching] Failed to process tab {i}: {tab_e}")
errored_tabs.append({
"tab_text": f"tab_{depth}_{i}",
"tab_index": i,
"depth": depth,
"error": f"processing_exception: {tab_e!s}"
})
try:
first_tab = found_tabs[0]
@@ -660,6 +679,13 @@ class ScreenshotService:
await switch_tabs()
# Log summary of errored tabs
if errored_tabs:
logger.info(f"[TabSwitching] {len(errored_tabs)} tab(s) with errors detected. "
f"Errored tabs: {[t['tab_text'] for t in errored_tabs]}")
else:
logger.info("[TabSwitching] No tab errors detected during preload")
# 2. Calculate full height for screenshot
# @UX_STATE [CalculatingHeight] -> Determining dashboard dimensions
full_height = await page.evaluate("""() => {
@@ -757,12 +783,65 @@ class ScreenshotService:
logger.info(f"[DIAGNOSTIC] Final screenshot saved: {output_path}")
logger.info(f"[DIAGNOSTIC] Final screenshot size: {final_size} bytes ({final_size / 1024:.2f} KB)")
# DIAGNOSTIC: Get image dimensions
# DIAGNOSTIC: Get image dimensions
try:
with Image.open(output_path) as final_img:
logger.info(f"[DIAGNOSTIC] Final screenshot dimensions: {final_img.width}x{final_img.height}")
except Exception as img_err:
logger.warning(f"[DIAGNOSTIC] Could not read final image dimensions: {img_err}")
# 3. Screenshot errored tabs (diagnostic viewport shots)
# @UX_STATE [CapturingErroredTabs] -> Taking viewport screenshots of tabs with chart errors.
# Only screenshots tabs that had errors during preload, plus always keeps the main tab.
# Errored tab screenshots are viewport-only (no full-page resize) to minimize overhead.
if errored_tabs:
logger.info(f"[TabSwitching] Capturing {len(errored_tabs)} errored tab screenshot(s)...")
screenshotted_tabs = set()
for tab_info in errored_tabs:
try:
tab_text = tab_info["tab_text"]
# Dedup: skip if already screenshotted
if tab_text in screenshotted_tabs:
continue
screenshotted_tabs.add(tab_text)
# Sanitize tab text for filename
safe_tab = re.sub(r'[^\w\-_ ]', '', tab_text).strip().replace(' ', '_')[:40]
if not safe_tab:
safe_tab = f"tab_{tab_info.get('depth', 0)}_{tab_info.get('tab_index', 0)}"
# Build tab path robustly (handle any extension or no extension)
base = output_path[:-4] if output_path.lower().endswith('.png') else output_path
tab_path = f"{base}__tab_{safe_tab}.png"
# Switch to the errored tab — try exact text match first
errored_tab_el = page.get_by_role("tab", name=tab_text, exact=True)
if not await errored_tab_el.is_visible():
# Fallback: try nth-child by index
tab_index = tab_info.get("tab_index")
if tab_index is not None:
errored_tab_el = page.locator('.ant-tabs-tab').nth(tab_index)
if await errored_tab_el.is_visible():
await errored_tab_el.click()
await asyncio.sleep(0.5)
await self._wait_for_charts_stabilized(page, timeout_ms=PLAYWRIGHT_SHORT_TIMEOUT_MS)
await page.screenshot(path=tab_path, timeout=PLAYWRIGHT_SHORT_TIMEOUT_MS)
tab_size = os.path.getsize(tab_path) if os.path.exists(tab_path) else 0
logger.info(f"[TabSwitching] Errored tab screenshot saved: {tab_path} ({tab_size} bytes)")
else:
logger.warning(f"[TabSwitching] Errored tab '{tab_text}' not visible, skipping screenshot")
except Exception as tab_shot_err:
logger.warning(f"[TabSwitching] Failed to screenshot errored tab '{tab_info.get('tab_text')}': {tab_shot_err}")
# Return to first tab after errored tab screenshots
try:
first_tab_el = page.locator('.ant-tabs-nav-list .ant-tabs-tab').first
if await first_tab_el.is_visible():
await first_tab_el.click()
await asyncio.sleep(0.5)
except Exception:
pass
except asyncio.TimeoutError:
logger.error(f"Screenshot capture timed out after {SCREENSHOT_SERVICE_TIMEOUT_MS}ms")
try:

View File

@@ -0,0 +1,176 @@
# #region MaintenanceBannerPlugin [C:4] [TYPE Module] [SEMANTICS plugin, maintenance, banner, execution, task]
# @BRIEF TaskManager plugin for executing maintenance banner operations (start, end, end-all).
# Dispatches to MaintenanceService based on operation type in params.
# @LAYER Plugin
# @RELATION IMPLEMENTS -> [PluginBase]
# @RELATION DEPENDS_ON -> [MaintenanceServiceModule]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [TaskContext]
# @INVARIANT TaskManager executes this plugin with params containing operation and event_id.
from typing import Any
from ..core.database import SessionLocal
from ..core.logger import belief_scope, logger as app_logger
from ..core.plugin_base import PluginBase
from ..core.superset_client import SupersetClient
from ..core.task_manager.context import TaskContext
from ..dependencies import get_config_manager
from ..models.maintenance import MaintenanceSettings
from ..services.maintenance import end_all_maintenance, end_maintenance, start_maintenance
# #region MaintenanceBannerPlugin [C:4] [TYPE Class]
# @BRIEF Plugin for executing maintenance banner operations asynchronously via TaskManager.
# @RELATION CALLS -> [start_maintenance]
# @RELATION CALLS -> [end_maintenance]
# @RELATION CALLS -> [end_all_maintenance]
class MaintenanceBannerPlugin(PluginBase):
"""TaskManager plugin for maintenance banner operations."""
# #region id [TYPE Property]
@property
def id(self) -> str:
return "maintenance_banner_apply"
# #endregion id
# #region name [TYPE Property]
@property
def name(self) -> str:
return "Maintenance Banner"
# #endregion name
# #region description [TYPE Property]
@property
def description(self) -> str:
return "Applies or removes maintenance banners on Superset dashboards."
# #endregion description
# #region version [TYPE Property]
@property
def version(self) -> str:
return "1.0.0"
# #endregion version
# #region ui_route [TYPE Property]
@property
def ui_route(self) -> str:
return "" # No UI route — async-only
# #endregion ui_route
# #region get_schema [TYPE Function]
def get_schema(self) -> dict[str, Any]:
return {
"type": "object",
"properties": {
"operation": {
"type": "string",
"enum": ["start", "end", "end_all"],
"title": "Operation",
},
"event_id": {
"type": "string",
"title": "Maintenance Event ID",
},
"environment_id": {
"type": "string",
"title": "Target Environment ID",
},
},
"required": ["operation"],
}
# #endregion get_schema
# #region execute [TYPE Function] [C:4]
# @BRIEF Execute maintenance operation based on params.operation.
# @PARAM params contains: operation (start|end|end_all), event_id, environment_id.
# @PARAM context TaskContext for logging.
# @PRE DB and Superset available.
# @POST Operation completed. Results returned as dict.
# @SIDE_EFFECT Modifies Superset dashboards, writes DB rows.
async def execute(
self,
params: dict[str, Any],
context: TaskContext | None = None,
) -> dict[str, Any]:
with belief_scope("MaintenanceBannerPlugin.execute"):
operation = params.get("operation", "start")
event_id = params.get("event_id", "")
environment_id = params.get("environment_id", "")
app_logger.reason(
"Executing maintenance operation",
extra={
"operation": operation,
"event_id": event_id,
"environment_id": environment_id,
},
)
db = SessionLocal()
try:
# Resolve environment
if not environment_id:
settings = db.query(MaintenanceSettings).filter(
MaintenanceSettings.id == "default"
).first()
if settings:
environment_id = settings.target_environment_id
if not environment_id:
config_manager = get_config_manager()
envs = config_manager.get_environments()
if envs:
environment_id = envs[0].id
env = None
config_manager = get_config_manager()
for e in config_manager.get_environments():
if e.id == environment_id:
env = e
break
if not env:
app_logger.explore(
"Environment not found",
extra={"environment_id": environment_id},
error="EnvironmentNotFound",
)
return {"status": "failed", "error": f"Environment {environment_id} not found"}
superset_client = SupersetClient(env)
if operation == "start":
if not event_id:
return {"status": "failed", "error": "event_id required for start"}
result = start_maintenance(event_id, db, superset_client)
elif operation == "end":
if not event_id:
return {"status": "failed", "error": "event_id required for end"}
result = end_maintenance(event_id, db, superset_client)
elif operation == "end_all":
result = end_all_maintenance(db, superset_client)
else:
result = {"status": "failed", "error": f"Unknown operation: {operation}"}
db.commit()
app_logger.reflect(
"Maintenance operation completed",
extra={"operation": operation, "result": result},
)
return result
except Exception as e:
db.rollback()
app_logger.explore(
"Maintenance operation failed",
extra={"operation": operation, "event_id": event_id},
error=str(e),
)
return {"status": "failed", "error": str(e)}
finally:
db.close()
# #endregion execute
# #endregion MaintenanceBannerPlugin
# #endregion MaintenanceBannerPlugin

View File

@@ -148,20 +148,10 @@ def _resolve_target_envs(env_ids: list[str]) -> dict[str, Environment]:
resolved: dict[str, Environment] = {}
if not configured:
for config_path in [Path("config.json"), Path("backend/config.json")]:
if not config_path.exists():
continue
try:
payload = json.loads(config_path.read_text(encoding="utf-8"))
env_rows = payload.get("environments", [])
for row in env_rows:
env = Environment(**row)
configured[env.id] = env
except Exception as exc:
logger.reflect(
f"Failed loading environments from {config_path}: {exc}",
extra={"src": "_resolve_target_envs"},
)
logger.reflect(
"No environments found in DB — seed script requires environments configured via API first",
extra={"src": "_resolve_target_envs"},
)
for env_id in env_ids:
env = configured.get(env_id)

View File

@@ -6,6 +6,7 @@
# @RELATION DEPENDS_ON -> [TaskCleanupService]
# @RELATION DEPENDS_ON -> [TaskManager]
from datetime import UTC
import os
import time
from typing import Any, cast
@@ -255,6 +256,9 @@ class HealthService:
task_id = str(record.task_id) if record.task_id is not None else None
summary = str(record.summary) if record.summary is not None else None
timestamp = cast(Any, record.timestamp)
# Ensure timestamp is timezone-aware (DB stores naive UTC)
if timestamp is not None and timestamp.tzinfo is None:
timestamp = timestamp.replace(tzinfo=UTC)
meta = self._resolve_dashboard_meta(dashboard_id, resolved_environment_id)
items.append(

View File

@@ -0,0 +1,30 @@
# #region MaintenancePackage [C:1] [TYPE Package] [SEMANTICS maintenance, service, package]
# @BRIEF Maintenance Banner service package. Re-exports all public orchestrators and helpers.
# @LAYER Service
# @RELATION DEPENDS_ON -> [MaintenanceOrchestrators]
# @RELATION DEPENDS_ON -> [MaintenanceDashboardScanner]
# @RELATION DEPENDS_ON -> [MaintenanceBannerRenderer]
# @RELATION DEPENDS_ON -> [MaintenanceChartManager]
from ._banner_renderer import build_banner_text, rebuild_banner
from ._chart_manager import ensure_banner_chart
from ._dashboard_scanner import find_affected_dashboards
from ._orchestrators import (
build_idempotency_key,
end_all_maintenance,
end_maintenance,
start_maintenance,
)
__all__ = [
"build_banner_text",
"build_idempotency_key",
"end_all_maintenance",
"end_maintenance",
"ensure_banner_chart",
"find_affected_dashboards",
"rebuild_banner",
"start_maintenance",
]
# #endregion MaintenancePackage

View File

@@ -0,0 +1,254 @@
# #region MaintenanceBannerRenderer [C:3] [TYPE Module] [SEMANTICS maintenance, banner, renderer, text]
# @BRIEF Banner text rendering and rebuild helpers. Build aggregated markdown from
# active maintenance events, format with template, apply HTML escaping.
# @LAYER Service
# @RELATION DEPENDS_ON -> [MaintenanceModels]
# @RELATION DEPENDS_ON -> [SupersetDashboardsWriteMixin]
import html
import re as _re
from datetime import datetime
from typing import Any
from sqlalchemy.orm import Session
from ...core.logger import belief_scope, logger as app_logger
from ...core.superset_client import SupersetClient
from ...models.maintenance import (
MaintenanceDashboardBanner,
MaintenanceDashboardBannerStatus,
MaintenanceDashboardState,
MaintenanceDashboardStateStatus,
MaintenanceEvent,
MaintenanceEventStatus,
MaintenanceSettings,
)
# #region build_banner_text [C:3] [TYPE Function] [SEMANTICS maintenance, banner, text, template, rendering]
# @BRIEF Build aggregated banner markdown text from events using settings template.
# Single event: substitute {start_time}, {end_time}, {message} directly.
# Multiple events: render each event as sub-block, combine as {message}.
# Missing variables → empty string. Message is HTML-escaped.
# @PRE events is a non-empty list of dicts with start_time, end_time, message keys.
# template is a non-empty string.
# @POST Returns str with substituted template or empty string on failure.
def build_banner_text(
events: list[dict[str, Any]],
template: str,
display_timezone: str = "UTC",
) -> str:
with belief_scope(
"build_banner_text",
f"events={len(events)}",
):
if not events:
app_logger.reflect("No events, returning empty string")
return ""
def _format_dt(dt_str: str | None) -> str:
"""Format datetime string for display."""
if not dt_str:
return "уточняется"
try:
dt = datetime.fromisoformat(dt_str.replace("Z", "+00:00"))
return dt.strftime("%Y-%m-%d %H:%M")
except (ValueError, TypeError):
return dt_str or "уточняется"
def _event_substitutions(event: dict[str, Any]) -> dict[str, str]:
"""Build substitution dict for a single event."""
msg = event.get("message") or ""
escaped_msg = html.escape(msg) if msg else ""
return {
"start_time": _format_dt(event.get("start_time")),
"end_time": _format_dt(event.get("end_time")),
"message": escaped_msg,
}
if len(events) == 1:
subs = _event_substitutions(events[0])
result = template
for key, value in subs.items():
result = result.replace(f"{{{key}}}", value)
# Remove any remaining unreplaced variables
result = _re.sub(r"\{[a-zA-Z_]+\}", "", result)
app_logger.reflect("Banner text built (single event)")
return result
# Multiple events: build aggregated message, substitute as {message}
event_blocks: list[str] = []
for i, event in enumerate(events):
subs = _event_substitutions(event)
block = template
for key, value in subs.items():
block = block.replace(f"{{{key}}}", value)
block = _re.sub(r"\{[a-zA-Z_]+\}", "", block)
event_blocks.append(block)
combined_message = "\n\n---\n\n".join(event_blocks)
# Build aggregator template (use first event for start/end if uniform)
result = template
result = result.replace("{message}", combined_message)
if len(events) == 1:
subs = _event_substitutions(events[0])
for key in ("start_time", "end_time"):
result = result.replace(f"{{{key}}}", subs.get(key, ""))
else:
result = result.replace("{start_time}", "")
result = result.replace("{end_time}", "")
result = _re.sub(r"\{[a-zA-Z_]+\}", "", result)
app_logger.reflect("Banner text built (multi-event)")
return result
# #endregion build_banner_text
# #region _build_banner_text_for_dashboard [C:2] [TYPE Function]
# @BRIEF Build aggregated banner text for a banner based on all active events linked to it.
def _build_banner_text_for_dashboard(
banner_id: str,
db_session: Session,
template: str,
timezone_str: str,
) -> str:
try:
# Find all active dashboard states linked to this banner
active_states = (
db_session.query(MaintenanceDashboardState)
.filter(
MaintenanceDashboardState.banner_id == banner_id,
MaintenanceDashboardState.status == MaintenanceDashboardStateStatus.ACTIVE,
)
.all()
)
if not active_states:
return ""
# Load events for each state
events_data: list[dict[str, Any]] = []
for state in active_states:
ev = (
db_session.query(MaintenanceEvent)
.filter(MaintenanceEvent.id == state.event_id)
.first()
)
if ev and ev.status in (
MaintenanceEventStatus.ACTIVE,
MaintenanceEventStatus.PARTIAL,
MaintenanceEventStatus.APPLYING,
):
events_data.append(
{
"start_time": ev.start_time.isoformat() if ev.start_time else None,
"end_time": ev.end_time.isoformat() if ev.end_time else None,
"message": ev.message or "",
}
)
if not events_data:
return ""
return build_banner_text(events_data, template, timezone_str)
except Exception as e:
app_logger.explore(
"Failed to build banner text",
extra={"banner_id": banner_id},
error=str(e),
)
return ""
# #endregion _build_banner_text_for_dashboard
# #region rebuild_banner [C:3] [TYPE Function] [SEMANTICS maintenance, banner, rebuild, text, update]
# @BRIEF Rebuild aggregated banner text for a banner and update the Superset chart.
# Query all active MaintenanceDashboardState records linked to banner,
# build aggregated text via build_banner_text, update markdown chart in Superset.
# @PRE banner_id points to an active MaintenanceDashboardBanner with a valid chart_id.
# @POST Banner chart in Superset updated with current aggregated text. DB banner_text updated.
# @SIDE_EFFECT Modifies Superset chart. Writes DB.
# @RELATION DEPENDS_ON -> [build_banner_text]
# @RELATION DEPENDS_ON -> [SupersetDashboardsWriteMixin]
def rebuild_banner(
banner_id: str,
db_session: Session,
superset_client: SupersetClient,
) -> bool:
with belief_scope("rebuild_banner", f"banner_id={banner_id}"):
app_logger.reason(
"Rebuilding banner text",
extra={"banner_id": banner_id},
)
banner = db_session.query(MaintenanceDashboardBanner).filter(
MaintenanceDashboardBanner.id == banner_id
).first()
if not banner:
app_logger.explore(
"Banner not found",
extra={"banner_id": banner_id},
error="BannerNotFound",
)
return False
if banner.status != MaintenanceDashboardBannerStatus.ACTIVE:
app_logger.reflect(
"Banner not active, skipping",
extra={"banner_id": banner_id, "status": banner.status.value},
)
return False
# Load settings for template
settings = db_session.query(MaintenanceSettings).filter(
MaintenanceSettings.id == "default"
).first()
template = settings.banner_template if settings else ""
timezone_str = settings.display_timezone if settings else "UTC"
# Build aggregated text
banner_text = _build_banner_text_for_dashboard(
banner_id, db_session, template, timezone_str
)
if not banner_text and banner.chart_id:
# No active events — should be handled by end_maintenance, but clean up just in case
app_logger.reflect(
"No active events for banner, marking as REMOVED",
extra={"banner_id": banner_id},
)
banner.status = MaintenanceDashboardBannerStatus.REMOVED
db_session.flush()
return True
# Update banner on dashboard (native MARKDOWN in layout)
if banner.chart_id:
try:
superset_client.update_banner_on_dashboard(
banner.dashboard_id, banner.chart_id, banner_text
)
except Exception as e:
app_logger.explore(
"Failed to update banner on dashboard during rebuild",
extra={
"chart_id": banner.chart_id,
"banner_id": banner_id,
"dashboard_id": banner.dashboard_id,
},
error=str(e),
)
return False
# Update DB
banner.banner_text = banner_text
db_session.flush()
app_logger.reflect(
"Banner rebuilt successfully",
extra={"banner_id": banner_id},
)
return True
# #endregion rebuild_banner
# #endregion MaintenanceBannerRenderer

View File

@@ -0,0 +1,359 @@
# #region MaintenanceChartManager [C:3] [TYPE Module] [SEMANTICS maintenance, chart, banner, lifecycle]
# @BRIEF Chart operations for maintenance banners: create/get banner charts, process
# per-dashboard state transitions for start and end orchestrators.
# @LAYER Service
# @RELATION DEPENDS_ON -> [SupersetDashboardsWriteMixin]
# @RELATION DEPENDS_ON -> [MaintenanceModels]
from typing import Any
from sqlalchemy.orm import Session
from ...core.logger import belief_scope, logger as app_logger
from ...core.superset_client import SupersetClient
from ...models.maintenance import (
MaintenanceDashboardBanner,
MaintenanceDashboardBannerStatus,
MaintenanceDashboardState,
MaintenanceDashboardStateStatus,
MaintenanceEvent,
MaintenanceEventStatus,
MaintenanceSettings,
)
from ._banner_renderer import _build_banner_text_for_dashboard, rebuild_banner
from ._dashboard_scanner import _resolve_dashboard_title
# #region ensure_banner_chart [C:3] [TYPE Function] [SEMANTICS maintenance, banner, chart, creation]
# @BRIEF Get or create a MaintenanceDashboardBanner for a dashboard in the target environment.
# If an active banner exists in DB, return it. Otherwise, create a markdown chart in Superset,
# update dashboard layout, and store the banner row.
# @PRE dashboard_id exists in Superset. superset_client has write access.
# @POST Returns MaintenanceDashboardBanner with chart_id set. Chart is placed at top of dashboard.
# @SIDE_EFFECT Creates chart in Superset, modifies dashboard layout, writes DB row.
# @RELATION DEPENDS_ON -> [SupersetDashboardsWriteMixin]
def ensure_banner_chart(
dashboard_id: int,
environment_id: str,
superset_client: SupersetClient,
db_session: Session,
) -> MaintenanceDashboardBanner:
with belief_scope(
"ensure_banner_chart",
f"dashboard_id={dashboard_id}",
):
app_logger.reason(
"Ensuring banner chart exists",
extra={"dashboard_id": dashboard_id},
)
# Check for existing active banner
existing = (
db_session.query(MaintenanceDashboardBanner)
.filter(
MaintenanceDashboardBanner.dashboard_id == dashboard_id,
MaintenanceDashboardBanner.environment_id == environment_id,
MaintenanceDashboardBanner.status == MaintenanceDashboardBannerStatus.ACTIVE,
)
.first()
)
if existing:
app_logger.reason(
"Found existing active banner, verifying chart in Superset",
extra={"banner_id": existing.id, "chart_id": existing.chart_id},
)
# Verify the chart still exists in Superset
try:
superset_client.get_chart(existing.chart_id)
app_logger.reflect(
"Existing banner chart is alive, reusing",
extra={"banner_id": existing.id, "chart_id": existing.chart_id},
)
return existing
except Exception:
app_logger.explore(
"Existing banner chart is stale (not found in Superset), marking as REMOVED",
extra={
"banner_id": existing.id,
"chart_id": existing.chart_id,
},
)
existing.status = MaintenanceDashboardBannerStatus.REMOVED
db_session.flush()
# Fall through to create a new chart
# Create markdown chart in Superset
try:
chart_id = superset_client.create_markdown_chart(
dashboard_id, "*Maintenance banner placeholder*"
)
except Exception as e:
app_logger.explore(
"Failed to create markdown chart",
extra={"dashboard_id": dashboard_id},
error=str(e),
)
raise
# Insert chart at top of dashboard layout
try:
superset_client.update_dashboard_layout(dashboard_id, chart_id)
except Exception as e:
app_logger.explore(
"Failed to update dashboard layout, deleting chart",
extra={"dashboard_id": dashboard_id, "chart_id": chart_id},
error=str(e),
)
# Clean up the chart we just created
try:
superset_client.delete_chart(chart_id)
except Exception:
pass
raise
# Persist banner row
banner = MaintenanceDashboardBanner(
environment_id=environment_id,
dashboard_id=dashboard_id,
chart_id=chart_id,
banner_text="",
status=MaintenanceDashboardBannerStatus.ACTIVE,
)
db_session.add(banner)
db_session.flush()
app_logger.reflect(
"Banner chart created and persisted",
extra={
"banner_id": banner.id,
"chart_id": chart_id,
"dashboard_id": dashboard_id,
},
)
return banner
# #endregion ensure_banner_chart
# #region _process_dashboards_for_start [C:3] [TYPE Function]
# @BRIEF Process each dashboard for a start maintenance event: ensure banner chart,
# create dashboard state, build banner text, update chart markdown.
# @SIDE_EFFECT Creates/updates Superset charts, writes DB rows.
# @RELATION CALLS -> [ensure_banner_chart]
# @RELATION CALLS -> [_build_banner_text_for_dashboard]
# @RELATION CALLS -> [_resolve_dashboard_title]
def _process_dashboards_for_start(
dashboard_ids: list[int],
event_id: str,
environment_id: str,
template: str,
timezone_str: str,
superset_client: SupersetClient,
db_session: Session,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[MaintenanceDashboardBanner]]:
successful_dashboards: list[dict[str, Any]] = []
failed_dashboards: list[dict[str, Any]] = []
banners_created: list[MaintenanceDashboardBanner] = []
for dash_id in dashboard_ids:
app_logger.reason(
"Processing dashboard",
extra={"dashboard_id": dash_id, "event_id": event_id},
)
try:
# Ensure banner chart exists
banner = ensure_banner_chart(
dash_id,
environment_id,
superset_client,
db_session,
)
banners_created.append(banner)
# Create dashboard state
state = MaintenanceDashboardState(
event_id=event_id,
dashboard_id=dash_id,
banner_id=banner.id,
status=MaintenanceDashboardStateStatus.ACTIVE,
)
db_session.add(state)
db_session.flush() # Ensure state is visible to subsequent queries
# Build banner text for this dashboard's active events
banner_text = _build_banner_text_for_dashboard(
banner.id, db_session, template, timezone_str
)
# Update banner content on dashboard (native MARKDOWN in layout)
if banner.chart_id:
try:
superset_client.update_banner_on_dashboard(
dash_id, banner.chart_id, banner_text
)
except Exception as e:
app_logger.explore(
"Failed to update banner on dashboard",
extra={
"chart_id": banner.chart_id,
"dashboard_id": dash_id,
},
error=str(e),
)
state.status = MaintenanceDashboardStateStatus.APPLY_FAILED
failed_dashboards.append(
{"id": dash_id, "title": str(dash_id), "error": str(e)}
)
continue
# Update banner text in DB
banner.banner_text = banner_text
db_session.flush()
# Resolve dashboard title
title = _resolve_dashboard_title(dash_id, superset_client)
successful_dashboards.append(
{
"id": dash_id,
"title": title,
"chart_id": banner.chart_id,
}
)
app_logger.reflect(
"Dashboard processed successfully",
extra={"dashboard_id": dash_id},
)
except Exception as e:
app_logger.explore(
"Failed to process dashboard",
extra={"dashboard_id": dash_id},
error=str(e),
)
failed_dashboards.append(
{"id": dash_id, "title": str(dash_id), "error": str(e)}
)
return successful_dashboards, failed_dashboards, banners_created
# #endregion _process_dashboards_for_start
# #region _process_states_for_end [C:3] [TYPE Function]
# @BRIEF Process each dashboard state for an end maintenance event: if other events
# still use same banner → rebuild; if no other events → remove banner (chart + layout).
# @SIDE_EFFECT Modifies Superset dashboards (removes charts, updates layouts). Writes DB.
# @RELATION CALLS -> [rebuild_banner]
def _process_states_for_end(
states: list[MaintenanceDashboardState],
event_id: str,
superset_client: SupersetClient,
db_session: Session,
) -> tuple[int, list[dict[str, Any]]]:
removed_count = 0
failed_removals: list[dict[str, Any]] = []
for state in states:
dash_id = state.dashboard_id
app_logger.reason(
"Processing dashboard state for end",
extra={"dashboard_id": dash_id, "state_id": state.id},
)
try:
banner_id = state.banner_id
if not banner_id:
# No banner linked, just mark as removed
state.status = MaintenanceDashboardStateStatus.REMOVED
removed_count += 1
continue
# Check if other active events still use this banner
other_active = db_session.query(MaintenanceDashboardState).filter(
MaintenanceDashboardState.banner_id == banner_id,
MaintenanceDashboardState.status == MaintenanceDashboardStateStatus.ACTIVE,
MaintenanceDashboardState.event_id != event_id,
).count()
banner = db_session.query(MaintenanceDashboardBanner).filter(
MaintenanceDashboardBanner.id == banner_id
).first()
if other_active > 0 and banner:
# Other events still active — rebuild banner text
state.status = MaintenanceDashboardStateStatus.REMOVED
db_session.flush()
rebuild_banner(banner_id, db_session, superset_client)
app_logger.reflect(
"Banner rebuilt after removing event",
extra={"banner_id": banner_id, "dashboard_id": dash_id},
)
elif banner:
# No other events — remove banner entirely
state.status = MaintenanceDashboardStateStatus.REMOVING
db_session.flush()
# Remove chart from layout
if banner.chart_id:
try:
superset_client.remove_chart_from_layout(
dash_id, banner.chart_id
)
except Exception as e:
app_logger.explore(
"Failed to remove chart from layout",
extra={
"dashboard_id": dash_id,
"chart_id": banner.chart_id,
},
error=str(e),
)
state.status = MaintenanceDashboardStateStatus.REMOVAL_FAILED
failed_removals.append(
{"id": dash_id, "title": str(dash_id), "error": str(e)}
)
continue
# Delete chart
try:
superset_client.delete_chart(banner.chart_id)
except Exception as e:
app_logger.explore(
"Failed to delete chart",
extra={
"chart_id": banner.chart_id,
"dashboard_id": dash_id,
},
error=str(e),
)
state.status = MaintenanceDashboardStateStatus.REMOVAL_FAILED
failed_removals.append(
{"id": dash_id, "title": str(dash_id), "error": str(e)}
)
continue
# Mark banner as removed
banner.status = MaintenanceDashboardBannerStatus.REMOVED
state.status = MaintenanceDashboardStateStatus.REMOVED
app_logger.reflect(
"Banner fully removed",
extra={"banner_id": banner_id, "dashboard_id": dash_id},
)
else:
state.status = MaintenanceDashboardStateStatus.REMOVED
removed_count += 1
except Exception as e:
app_logger.explore(
"Failed to end maintenance for dashboard",
extra={"dashboard_id": dash_id},
error=str(e),
)
state.status = MaintenanceDashboardStateStatus.REMOVAL_FAILED
failed_removals.append(
{"id": dash_id, "title": str(dash_id), "error": str(e)}
)
return removed_count, failed_removals
# #endregion _process_states_for_end
# #endregion MaintenanceChartManager

View File

@@ -0,0 +1,205 @@
# #region MaintenanceDashboardScanner [C:3] [TYPE Module] [SEMANTICS maintenance, dashboard, scanner, tables]
# @BRIEF Dashboard discovery and filtering helpers for maintenance banner feature.
# Scan Superset datasets, match tables, apply scope/excluded/forced filters.
# @LAYER Service
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SqlTableExtractorModule]
# @RELATION DEPENDS_ON -> [MaintenanceModels]
from typing import Any
from sqlalchemy.orm import Session
from ...core.logger import belief_scope, logger as app_logger
from ...core.superset_client import SupersetClient
from ...models.maintenance import DashboardScope, MaintenanceSettings
from ..sql_table_extractor import extract_tables_from_sql
# #region find_affected_dashboards [C:3] [TYPE Function] [SEMANTICS maintenance, dashboard, discovery, tables]
# @BRIEF Find all dashboard IDs whose datasets reference any of the given tables.
# Supports both table-based (schema, table_name) and virtual SQL datasets (via SqlTableExtractor).
# Applies scope/excluded/forced filtering from MaintenanceSettings.
# @PRE tables is a non-empty list of "schema.table" strings. superset_client is authenticated.
# @POST Returns a deduplicated list of dashboard IDs that should receive banners.
# @SIDE_EFFECT Fetches all datasets from Superset (paginated). May be slow with many datasets.
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SqlTableExtractorModule]
def find_affected_dashboards(
tables: list[str],
superset_client: SupersetClient,
settings: MaintenanceSettings | None = None,
) -> list[int]:
with belief_scope(
"find_affected_dashboards",
f"tables={tables}",
):
app_logger.reason(
"Finding affected dashboards",
extra={"tables": tables},
)
if not tables:
app_logger.reflect("No tables provided, returning empty")
return []
# Normalize input tables to lowercase set for matching
target_tables = {t.lower().strip() for t in tables if t.strip()}
if not target_tables:
return []
# Fetch all datasets from Superset
try:
_, datasets = superset_client.get_datasets()
except Exception as e:
app_logger.explore(
"Failed to fetch datasets from Superset",
extra={},
error=str(e),
)
raise
app_logger.reason(
f"Scanning {len(datasets)} datasets for table references",
extra={"target_tables": list(target_tables)},
)
matched_dashboard_ids: set[int] = set()
for ds in datasets:
ds_schema = (ds.get("schema") or "").lower().strip()
ds_table = (ds.get("table_name") or "").lower().strip()
is_virtual = ds.get("is_sqllab_view", False) or ds.get("sql", "") not in (
None,
"",
)
ds_sql = ds.get("sql") or ""
ds_id = ds.get("id")
# Phase 1: Table-based match
if not is_virtual and ds_schema and ds_table:
qualified = f"{ds_schema}.{ds_table}"
if qualified in target_tables:
linked = _get_linked_dashboards(superset_client, ds_id)
matched_dashboard_ids.update(linked)
continue
# Phase 2: Virtual/SQL dataset — extract tables from SQL text
if is_virtual and ds_sql:
extracted = extract_tables_from_sql(ds_sql)
if extracted & target_tables:
linked = _get_linked_dashboards(superset_client, ds_id)
matched_dashboard_ids.update(linked)
app_logger.reflect(
f"Found {len(matched_dashboard_ids)} affected dashboards",
extra={"dashboard_ids": list(matched_dashboard_ids)},
)
# Apply filtering from settings
result = _apply_dashboard_filters(
list(matched_dashboard_ids), superset_client, settings
)
app_logger.reflect(
f"After filtering: {len(result)} dashboards",
extra={"dashboard_ids": result},
)
return result
# #endregion find_affected_dashboards
# #region _get_linked_dashboards [C:2] [TYPE Function]
# @BRIEF Fetch linked dashboards for a dataset from Superset.
# @SIDE_EFFECT Calls Superset API.
def _get_linked_dashboards(
superset_client: SupersetClient,
dataset_id: int | None,
) -> set[int]:
if not dataset_id:
return set()
try:
detail = superset_client.get_dataset_detail(dataset_id)
linked = detail.get("linked_dashboards", [])
return {int(d["id"]) for d in linked if d.get("id")}
except Exception as e:
app_logger.explore(
"Failed to fetch linked dashboards",
extra={"dataset_id": dataset_id},
error=str(e),
)
return set()
# #endregion _get_linked_dashboards
# #region _apply_dashboard_filters [C:2] [TYPE Function]
# @BRIEF Apply scope (published/draft/all), excluded, and forced filters from settings.
def _apply_dashboard_filters(
dashboard_ids: list[int],
superset_client: SupersetClient,
settings: MaintenanceSettings | None,
) -> list[int]:
if not settings:
return dashboard_ids
# If forced list exists, include those unconditionally
forced_ids = set(settings.forced_dashboard_ids or [])
excluded_ids = set(settings.excluded_dashboard_ids or [])
# Apply scope filtering
if settings.dashboard_scope != DashboardScope.ALL:
try:
_, all_dashboards = superset_client.get_dashboards()
scope = settings.dashboard_scope
filtered: list[int] = []
for did in dashboard_ids:
dash = next(
(d for d in all_dashboards if d.get("id") == did), None
)
if dash is None:
continue
is_published = dash.get("published", True)
if scope == DashboardScope.PUBLISHED_ONLY and not is_published:
continue
if scope == DashboardScope.DRAFT_ONLY and is_published:
continue
filtered.append(did)
dashboard_ids = filtered
except Exception as e:
app_logger.explore(
"Scope filtering failed, proceeding without it",
extra={},
error=str(e),
)
# Apply excluded (forced overrides excluded per FR-009-b)
result = [
did for did in dashboard_ids if did not in excluded_ids or did in forced_ids
]
# Add forced dashboards that are not already included
for fid in forced_ids:
if fid not in result:
result.append(fid)
return result
# #endregion _apply_dashboard_filters
# #region _resolve_dashboard_title [C:2] [TYPE Function]
# @BRIEF Resolve dashboard title from Superset by ID.
def _resolve_dashboard_title(
dashboard_id: int,
superset_client: SupersetClient,
) -> str:
try:
_, dashboards = superset_client.get_dashboards()
for d in dashboards:
if d.get("id") == dashboard_id:
return d.get("dashboard_title") or d.get("title") or str(dashboard_id)
except Exception:
pass
return str(dashboard_id)
# #endregion _resolve_dashboard_title
# #endregion MaintenanceDashboardScanner

View File

@@ -0,0 +1,338 @@
# #region MaintenanceOrchestrators [C:4] [TYPE Module] [SEMANTICS maintenance, orchestrator, lifecycle]
# @BRIEF C4 orchestrators for maintenance event lifecycle: start, end, end-all.
# Private helpers extracted to sub-modules. All orchestrators use belief runtime.
# @LAYER Service
# @RELATION DEPENDS_ON -> [MaintenanceModels]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [MaintenanceDashboardScanner]
# @RELATION DEPENDS_ON -> [MaintenanceBannerRenderer]
# @RELATION DEPENDS_ON -> [MaintenanceChartManager]
# @INVARIANT C4 methods use belief_scope + REASON/REFLECT/EXPLORE markers.
# @INVARIANT Idempotency key: sorted lowercased (tables, start_time, end_time). message NOT in key.
from datetime import datetime, timezone
from typing import Any
from sqlalchemy.orm import Session
from ...core.logger import belief_scope, logger as app_logger
from ...core.superset_client import SupersetClient
from ...models.maintenance import (
MaintenanceDashboardState,
MaintenanceEvent,
MaintenanceEventStatus,
MaintenanceSettings,
)
from ._chart_manager import _process_dashboards_for_start, _process_states_for_end
from ._dashboard_scanner import find_affected_dashboards
# #region build_idempotency_key [C:2] [TYPE Function]
# @BRIEF Build idempotency key from (tables, start_time, end_time). Sorted lowercased tables, ISO timestamps.
# @PRE tables is a list of strings. start_time is a datetime or None. end_time is a datetime or None.
# @POST Returns a tuple of (frozenset(str), str|None, str|None) for DB comparison.
# @RATIONALE message is intentionally excluded from key per spec — different message with same tables/times = same event.
def build_idempotency_key(
tables: list[str],
start_time: datetime | None,
end_time: datetime | None,
) -> tuple[frozenset[str], str | None, str | None]:
sorted_tables = sorted(t.strip().lower() for t in tables if t.strip())
start_key = start_time.isoformat() if start_time else None
end_key = end_time.isoformat() if end_time else None
return (frozenset(sorted_tables), start_key, end_key)
# #endregion build_idempotency_key
# #region start_maintenance [C:4] [TYPE Function] [SEMANTICS maintenance, start, orchestration, async]
# @BRIEF C4 orchestrator for starting a maintenance event. Runs inside a TaskManager task.
# Idempotency check with SELECT ... FOR UPDATE. Creates event states, finds dashboards,
# ensures banner charts, builds text, updates charts. Event transitions to ACTIVE or PARTIAL.
# @PRE event_id points to a valid MaintenanceEvent with status=PENDING.
# @POST Event and dashboard states transitioned to ACTIVE/PARTIAL/APPLY_FAILED.
# Banner charts created and updated in Superset.
# @SIDE_EFFECT Modifies Superset dashboards (creates/updates charts, modifies layouts).
# Writes multiple DB rows (dashboard states, banner updates).
# @BELIEF Uses belief_scope with REASON/REFLECT/EXPLORE markers.
# @RELATION CALLS -> [find_affected_dashboards]
# @RELATION CALLS -> [_process_dashboards_for_start]
def start_maintenance(
event_id: str,
db_session: Session,
superset_client: SupersetClient,
) -> dict[str, Any]:
with belief_scope("start_maintenance", f"event_id={event_id}"):
app_logger.reason(
"Starting maintenance event",
extra={"event_id": event_id},
)
# Load event
event = db_session.query(MaintenanceEvent).filter(
MaintenanceEvent.id == event_id
).with_for_update().first()
if not event:
app_logger.explore(
"Event not found",
extra={"event_id": event_id},
error="EventNotFound",
)
return {"maintenance_id": event_id, "status": "failed", "error": "Event not found"}
if event.status != MaintenanceEventStatus.PENDING:
app_logger.reflect(
"Event already processed",
extra={"event_id": event_id, "status": event.status.value},
)
return {"maintenance_id": event_id, "status": event.status.value}
# Transition to APPLYING
event.status = MaintenanceEventStatus.APPLYING
db_session.flush()
# Load settings
settings = db_session.query(MaintenanceSettings).filter(
MaintenanceSettings.id == "default"
).first()
environment_id = settings.target_environment_id if settings else event.environment_id
template = settings.banner_template if settings else ""
timezone_str = settings.display_timezone if settings else "UTC"
# Find affected dashboards
try:
dashboard_ids = find_affected_dashboards(
event.tables,
superset_client,
settings,
)
except Exception as e:
app_logger.explore(
"Dashboard discovery failed",
extra={"event_id": event_id},
error=str(e),
)
event.status = MaintenanceEventStatus.FAILED
db_session.flush()
return {
"maintenance_id": event_id,
"status": "failed",
"error": str(e),
"affected_dashboards": 0,
"dashboards": [],
"failed_dashboards": [],
"unmatched_tables": [],
}
if not dashboard_ids:
app_logger.reflect(
"No matching dashboards found",
extra={"event_id": event_id},
)
event.status = MaintenanceEventStatus.ACTIVE
db_session.flush()
return {
"maintenance_id": event_id,
"status": "no_match",
"affected_dashboards": 0,
"dashboards": [],
"failed_dashboards": [],
"unmatched_tables": [
t for t in event.tables if t not in dashboard_ids
],
}
# Process each dashboard
successful_dashboards, failed_dashboards, _ = _process_dashboards_for_start(
dashboard_ids,
event_id,
environment_id,
template,
timezone_str,
superset_client,
db_session,
)
# Determine final event status
if not failed_dashboards:
event.status = MaintenanceEventStatus.ACTIVE
elif successful_dashboards:
event.status = MaintenanceEventStatus.PARTIAL
else:
event.status = MaintenanceEventStatus.FAILED
db_session.flush()
result = {
"maintenance_id": event_id,
"status": event.status.value,
"affected_dashboards": len(successful_dashboards),
"dashboards": successful_dashboards,
"failed_dashboards": failed_dashboards,
"unmatched_tables": [],
}
app_logger.reflect(
"Maintenance start completed",
extra={
"event_id": event_id,
"status": event.status.value,
"successful": len(successful_dashboards),
"failed": len(failed_dashboards),
},
)
return result
# #endregion start_maintenance
# #region end_maintenance [C:4] [TYPE Function] [SEMANTICS maintenance, end, orchestration, async, removal]
# @BRIEF C4 orchestrator for ending a maintenance event. Runs inside a TaskManager task.
# Event → ENDING. Delegates per-state processing to _process_states_for_end.
# Event → COMPLETED when done.
# @PRE event_id points to a valid active MaintenanceEvent.
# @POST Event transitions to COMPLETED. Dashboard states cleaned up. Charts removed when no other events.
# @SIDE_EFFECT Modifies Superset dashboards (removes charts, updates layouts). Writes DB.
# @BELIEF Uses belief_scope with REASON/REFLECT/EXPLORE markers.
# @RELATION CALLS -> [_process_states_for_end]
def end_maintenance(
event_id: str,
db_session: Session,
superset_client: SupersetClient,
) -> dict[str, Any]:
with belief_scope("end_maintenance", f"event_id={event_id}"):
app_logger.reason(
"Ending maintenance event",
extra={"event_id": event_id},
)
# Load event with lock
event = db_session.query(MaintenanceEvent).filter(
MaintenanceEvent.id == event_id
).with_for_update().first()
if not event:
app_logger.explore(
"Event not found",
extra={"event_id": event_id},
error="EventNotFound",
)
return {"maintenance_id": event_id, "status": "failed", "error": "Event not found"}
# Idempotent: already completed
if event.status == MaintenanceEventStatus.COMPLETED:
app_logger.reflect(
"Event already completed, idempotent",
extra={"event_id": event_id},
)
return {"maintenance_id": event_id, "status": "already_completed", "removed_from": 0, "failed_removals": []}
# Transition to ENDING
event.status = MaintenanceEventStatus.ENDING
db_session.flush()
# Find all dashboard states for this event
states = db_session.query(MaintenanceDashboardState).filter(
MaintenanceDashboardState.event_id == event_id
).all()
# Delegate per-state processing
removed_count, failed_removals = _process_states_for_end(
states, event_id, superset_client, db_session,
)
# Transition event to COMPLETED (even if some removals failed)
event.status = MaintenanceEventStatus.COMPLETED
db_session.flush()
result = {
"maintenance_id": event_id,
"status": "completed" if not failed_removals else "partial",
"removed_from": removed_count,
"failed_removals": failed_removals,
}
app_logger.reflect(
"Maintenance end completed",
extra={
"event_id": event_id,
"removed": removed_count,
"failed": len(failed_removals),
},
)
return result
# #endregion end_maintenance
# #region end_all_maintenance [C:4] [TYPE Function] [SEMANTICS maintenance, end-all, orchestration, bulk]
# @BRIEF End ALL active maintenance events. Iterates active events, calls end_maintenance for each.
# @PRE None.
# @POST All events ended. All banners removed or rebuilt (last event removed).
# @SIDE_EFFECT Modifies multiple Superset dashboards. Writes multiple DB rows.
# @BELIEF Uses belief_scope with REASON/REFLECT/EXPLORE markers.
# @RELATION CALLS -> [end_maintenance]
def end_all_maintenance(
db_session: Session,
superset_client: SupersetClient,
) -> dict[str, Any]:
with belief_scope("end_all_maintenance"):
app_logger.reason("Ending all active maintenance events")
active_events = db_session.query(MaintenanceEvent).filter(
MaintenanceEvent.status.in_([
MaintenanceEventStatus.ACTIVE,
MaintenanceEventStatus.PARTIAL,
MaintenanceEventStatus.APPLYING,
])
).all()
if not active_events:
app_logger.reflect("No active events to end")
return {
"status": "completed",
"closed_events": 0,
"removed_from": 0,
"failed_removals": [],
}
total_removed = 0
all_failed: list[dict[str, Any]] = []
closed_count = 0
for event in active_events:
app_logger.reason(
"Ending event",
extra={"event_id": event.id},
)
try:
result = end_maintenance(
event.id, db_session, superset_client
)
total_removed += result.get("removed_from", 0)
all_failed.extend(result.get("failed_removals", []))
closed_count += 1
except Exception as e:
app_logger.explore(
"Failed to end event",
extra={"event_id": event.id},
error=str(e),
)
all_failed.append(
{"id": 0, "title": event.id, "error": str(e)}
)
app_logger.reflect(
"All events ended",
extra={
"closed": closed_count,
"removed": total_removed,
"failed": len(all_failed),
},
)
return {
"status": "completed" if not all_failed else "partial",
"closed_events": closed_count,
"removed_from": total_removed,
"failed_removals": all_failed,
}
# #endregion end_all_maintenance
# #endregion MaintenanceOrchestrators

View File

@@ -0,0 +1,226 @@
# #region SqlTableExtractorModule [C:2] [TYPE Module] [SEMANTICS sql, jinja, table, extraction, parsing]
# @BRIEF Three-phase SQL+Jinja table name extractor for virtual datasets as per FR-003.
# Phase 1: Detect Jinja spans vs SQL spans
# Phase 2: In Jinja spans, extract "schema.table" from string values
# Phase 3: In SQL spans, regex pattern + sqlparse filter to reject string literal false positives
# @LAYER Service
# @RELATION DEPENDS_ON -> [re, sqlparse]
# @INVARIANT Only exact schema.table matches (case-insensitive); unqualified references are NOT matched.
# @INVARIANT Returns a set[str] of fully-qualified table names (lowercased for case-insensitive matching).
import re
from typing import Iterable
import sqlparse
from sqlparse.sql import Identifier, Parenthesis, Token, TokenList
from sqlparse.tokens import Literal, Name, Punctuation, Whitespace
# ── Phase 1: Jinja span detection ───────────────────────────
# Minimal Jinja token detection: {% ... %}, {{ ... }}, {# ... #}
_JINJA_BLOCK_RE = re.compile(r"{%-?\s*.*?\s*-?%}", re.DOTALL)
_JINJA_EXPR_RE = re.compile(r"{{-?\s*.*?\s*-?}}", re.DOTALL)
_JINJA_COMMENT_RE = re.compile(r"{#.*?#}", re.DOTALL)
# ── Phase 3: schema.table regex (case-insensitive) ───────────
_SCHEMA_TABLE_RE = re.compile(
r"""
(?<!['"`\w]) # not preceded by string delimiter or word char
\b # word boundary
(?: # begin: non-capturing group for full match
(?:"([a-zA-Z_][\w]*)")? # optional quoted schema
(?:([a-zA-Z_][\w]*)) # schema (unquoted)
\.
(?:([a-zA-Z_][\w]*)) # table
)
\b # word boundary
(?!['"`]) # not followed by a string delimiter
""",
re.VERBOSE | re.IGNORECASE,
)
# #region detect_jinja_spans [C:2] [TYPE Function]
# @BRIEF Phase 1: Split raw SQL text into Jinja spans and SQL spans.
# @PRE raw_sql is a string (possibly empty).
# @POST Returns a list of (span_type: str, text: str) tuples.
# span_type is "jinja" or "sql".
def detect_jinja_spans(raw_sql: str) -> list[tuple[str, str]]:
"""Split raw SQL+Jinja text into alternating Jinja and SQL spans.
Phase 1 detects Jinja blocks ({%%}, {{}}, {##}) and returns the
remaining text as SQL spans.
"""
if not raw_sql or not raw_sql.strip():
return [("sql", raw_sql or "")]
spans: list[tuple[str, str]] = []
# Collect all Jinja match positions
jinja_matches: list[tuple[int, int]] = []
for pattern in (_JINJA_BLOCK_RE, _JINJA_EXPR_RE, _JINJA_COMMENT_RE):
for m in pattern.finditer(raw_sql):
jinja_matches.append((m.start(), m.end()))
# Merge overlapping Jinja spans
if jinja_matches:
jinja_matches.sort()
merged: list[tuple[int, int]] = [jinja_matches[0]]
for start, end in jinja_matches[1:]:
if start <= merged[-1][1]:
merged[-1] = (merged[-1][0], max(merged[-1][1], end))
else:
merged.append((start, end))
# Build alternating sql/jinja spans
cursor = 0
for start, end in merged:
if cursor < start:
sql_chunk = raw_sql[cursor:start].strip()
if sql_chunk:
spans.append(("sql", sql_chunk))
jinja_chunk = raw_sql[start:end].strip()
if jinja_chunk:
spans.append(("jinja", jinja_chunk))
cursor = end
if cursor < len(raw_sql):
remaining = raw_sql[cursor:].strip()
if remaining:
spans.append(("sql", remaining))
else:
spans.append(("sql", raw_sql.strip()))
return spans
# #endregion detect_jinja_spans
# #region extract_tables_from_jinja [C:2] [TYPE Function]
# @BRIEF Phase 2: Extract "schema.table" references from Jinja string values.
# Looks for patterns like "raw.sales" inside Jinja text.
# @PRE jinja_text is a string from a Jinja span.
# @POST Returns a set of lowercased schema.table strings.
def extract_tables_from_jinja(jinja_text: str) -> set[str]:
"""Extract ``schema.table`` references from within Jinja template blocks.
Looks for double-quoted or single-quoted string values that match
the ``schema.table`` pattern, e.g. ``"raw.sales"`` or ``'raw.inventory'``.
"""
tables: set[str] = set()
# Match string values: "schema.table" or 'schema.table'
string_pattern = re.compile(
r"""["']([a-zA-Z_][\w]*\.[a-zA-Z_][\w]*)["']"""
)
for m in string_pattern.finditer(jinja_text):
tables.add(m.group(1).lower())
return tables
# #endregion extract_tables_from_jinja
# #region is_string_literal [C:1] [TYPE Function]
# @BRIEF Check if a sqlparse Token is a string literal (not a schema.table identifier).
# @PRE token is a sqlparse Token.
# @POST Returns True if the token is a string/Single/Literal.String.
def is_string_literal(token: Token) -> bool:
"""Return True if token is a string literal type in sqlparse."""
return token.ttype is Literal.String.Single or token.ttype is Literal.String
# #endregion is_string_literal
# #region extract_tables_from_sql_span [C:2] [TYPE Function]
# @BRIEF Phase 3: Extract schema.table references from SQL text using regex + sqlparse filtering.
# Uses regex to find all schema.table candidates, then sqlparse to filter out
# false positives inside string literals.
# @PRE sql_text is a string from a SQL span (non-Jinja).
# @POST Returns a set of lowercased schema.table strings.
def extract_tables_from_sql_span(sql_text: str) -> set[str]:
"""Extract ``schema.table`` references from plain SQL text.
Uses regex to find all ``schema.table`` candidates, then sqlparse
to reject matches that fall inside string literals.
"""
tables: set[str] = set()
raw_matches = _SCHEMA_TABLE_RE.findall(sql_text)
if not raw_matches:
return tables
# Use sqlparse to identify string literal positions
parsed = sqlparse.parse(sql_text)
string_literal_ranges: list[tuple[int, int]] = []
def walk_tokens(tokens: Iterable[Token], base_offset: int = 0) -> None:
offset = base_offset
for token in tokens:
if isinstance(token, TokenList):
walk_tokens(token.flatten(), offset)
else:
ttype = token.ttype
val = token.value
if is_string_literal(token):
string_literal_ranges.append(
(offset, offset + len(val))
)
offset += len(val)
for stmt in parsed:
if stmt is None:
continue
walk_tokens(stmt.flatten(), base_offset=0)
def is_in_string(pos: int) -> bool:
for s_start, s_end in string_literal_ranges:
if s_start <= pos <= s_end:
return True
return False
# Re-scan with full match positions to filter
for m in _SCHEMA_TABLE_RE.finditer(sql_text):
if not is_in_string(m.start()):
# Extract schema and table from match groups
schema = m.group(2) or m.group(1) or ""
table = m.group(3) or ""
# Filter out column aliases (single-char schemas like "a.id", "o.id")
if len(schema) < 2 and schema.isalpha() and schema.islower():
continue
if schema and table:
tables.add(f"{schema.lower()}.{table.lower()}")
return tables
# #endregion extract_tables_from_sql_span
# #region extract_tables_from_sql [C:2] [TYPE Function]
# @BRIEF Three-phase entry point: extract all schema.table references from raw SQL+Jinja text.
# @PRE raw_sql is a string (possibly empty).
# @POST Returns a set of lowercased fully-qualified table names (schema.table format).
# @RELATION CALLS -> [detect_jinja_spans]
# @RELATION CALLS -> [extract_tables_from_jinja]
# @RELATION CALLS -> [extract_tables_from_sql_span]
def extract_tables_from_sql(raw_sql: str) -> set[str]:
"""Extract all ``schema.table`` references from raw SQL + Jinja text.
Three-phase approach per FR-003:
1. Detect Jinja spans vs SQL spans
2. In Jinja spans, extract ``schema.table`` from quoted string values
3. In SQL spans, regex + sqlparse filter to reject string literal false positives
Returns a set of lowercased ``schema.table`` strings for case-insensitive matching.
Example:
>>> extract_tables_from_sql("SELECT * FROM raw.sales JOIN raw.inventory ON ...")
{'raw.sales', 'raw.inventory'}
"""
if not raw_sql or not raw_sql.strip():
return set()
all_tables: set[str] = set()
spans = detect_jinja_spans(raw_sql)
for span_type, text in spans:
if span_type == "jinja":
all_tables.update(extract_tables_from_jinja(text))
else:
all_tables.update(extract_tables_from_sql_span(text))
return all_tables
# #endregion extract_tables_from_sql
# #endregion SqlTableExtractorModule

View File

@@ -55,7 +55,12 @@ class ValidationRunService:
query = self.db.query(ValidationRecord)
if task_id:
query = query.filter(ValidationRecord.task_id == task_id)
# Match both spawned task_id (TaskRecord) and policy_id (ValidationPolicy.id)
# because the frontend sends the validation policy UUID, but ValidationRecord.task_id
# stores the spawned task UUID. Fall back to policy_id for the common case.
query = query.filter(
(ValidationRecord.task_id == task_id) | (ValidationRecord.policy_id == task_id)
)
if dashboard_id:
query = query.filter(ValidationRecord.dashboard_id == dashboard_id)
if environment_id: