fix(maintenance): auto-expiry + adaptive markdown height + tests
- Auto-expiry: expired events auto-end on GET /events via task manager - Height: _estimate_markdown_height adapted to unit=8px scale with padding detection - CRITICAL-1: removed dead import remove_chart_from_position (QA finding) - CRITICAL-2: added chart alive check in ensure_banner_chart (QA finding) - CRITICAL-3: fixed test assertions update_markdown_chart → update_banner_on_dashboard - @POST contract: fixed return range [2,12] → [19,200] - Tests: 8 new tests (auto-expiry + layout height)
This commit is contained in:
@@ -28,6 +28,7 @@ __all__ = [
|
||||
"git",
|
||||
"health",
|
||||
"llm",
|
||||
"maintenance",
|
||||
"mappings",
|
||||
"migration",
|
||||
"plugins",
|
||||
|
||||
10
backend/src/api/routes/maintenance/__init__.py
Normal file
10
backend/src/api/routes/maintenance/__init__.py
Normal file
@@ -0,0 +1,10 @@
|
||||
# #region MaintenanceRoutesPackage [C:1] [TYPE Package] [SEMANTICS maintenance, api, package]
|
||||
# @BRIEF Maintenance Banner API route package.
|
||||
# @LAYER API
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceRouter]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceSchemasModule]
|
||||
|
||||
from ._router import router as maintenance_router
|
||||
|
||||
__all__ = ["maintenance_router"]
|
||||
# #endregion MaintenanceRoutesPackage
|
||||
14
backend/src/api/routes/maintenance/_router.py
Normal file
14
backend/src/api/routes/maintenance/_router.py
Normal file
@@ -0,0 +1,14 @@
|
||||
# #region MaintenanceRouter [C:3] [TYPE Module] [SEMANTICS fastapi, router, maintenance, api]
|
||||
# @BRIEF FastAPI APIRouter for the Maintenance Banner endpoints.
|
||||
# @LAYER API
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceRoutesModule]
|
||||
# @RELATION DEPENDS_ON -> [AppDependencies]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceSchemasModule]
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
router = APIRouter(prefix="/api/maintenance", tags=["Maintenance"])
|
||||
|
||||
# Import route handlers to register them with the router
|
||||
from . import _routes # noqa: F401, E402
|
||||
# #endregion MaintenanceRouter
|
||||
562
backend/src/api/routes/maintenance/_routes.py
Normal file
562
backend/src/api/routes/maintenance/_routes.py
Normal file
@@ -0,0 +1,562 @@
|
||||
# #region MaintenanceRoutesModule [C:3] [TYPE Module] [SEMANTICS fastapi, routes, maintenance, api]
|
||||
# @BRIEF Route handler stubs for all 7 Maintenance Banner API endpoints with RBAC guards per FR-015.
|
||||
# @LAYER API
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceSchemasModule]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceRouter]
|
||||
# @RELATION DEPENDS_ON -> [AppDependencies]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceServiceModule]
|
||||
# @INVARIANT All mutation endpoints return 202 {task_id} per spec.
|
||||
# @INVARIANT RBAC enforced per FR-015 matrix via has_permission() dependency.
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, cast
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ....core.logger import belief_scope, logger as app_logger
|
||||
from ....core.task_manager import TaskManager
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
from ....dependencies import get_current_user, get_db, get_task_manager, has_permission
|
||||
from ....models.maintenance import (
|
||||
MaintenanceDashboardBanner,
|
||||
MaintenanceDashboardBannerStatus,
|
||||
MaintenanceDashboardState,
|
||||
MaintenanceDashboardStateStatus,
|
||||
MaintenanceEvent,
|
||||
MaintenanceEventStatus,
|
||||
MaintenanceSettings,
|
||||
)
|
||||
from ._router import router
|
||||
from ._schemas import (
|
||||
MaintenanceAlreadyActiveResponse,
|
||||
MaintenanceDashboardBannerState,
|
||||
MaintenanceEndResponse,
|
||||
MaintenanceEventItem,
|
||||
MaintenanceEventListResponse,
|
||||
MaintenanceSettingsResponse,
|
||||
MaintenanceSettingsUpdate,
|
||||
MaintenanceStartRequest,
|
||||
MaintenanceStartResponse,
|
||||
)
|
||||
|
||||
|
||||
# ── GET /api/maintenance/dashboard-banners ───────────────────
|
||||
# FR-015: viewer, operator, admin all have READ access
|
||||
|
||||
# #region list_dashboard_banners [C:2] [TYPE Function]
|
||||
# @BRIEF Get per-dashboard banner state for the Dashboard Hub indicator.
|
||||
# @RELATION DEPENDS_ON -> [has_permission("maintenance", "READ")]
|
||||
@router.get("/dashboard-banners", response_model=list[MaintenanceDashboardBannerState])
|
||||
async def list_dashboard_banners(
|
||||
db: Session = Depends(get_db),
|
||||
_=Depends(has_permission("maintenance", "READ")),
|
||||
):
|
||||
with belief_scope("list_dashboard_banners"):
|
||||
# Query active banners with their linked active/pending states
|
||||
active_banners = (
|
||||
db.query(MaintenanceDashboardBanner)
|
||||
.filter(MaintenanceDashboardBanner.status == MaintenanceDashboardBannerStatus.ACTIVE)
|
||||
.all()
|
||||
)
|
||||
|
||||
result: list[MaintenanceDashboardBannerState] = []
|
||||
for banner in active_banners:
|
||||
linked_states = (
|
||||
db.query(MaintenanceDashboardState)
|
||||
.filter(
|
||||
MaintenanceDashboardState.banner_id == banner.id,
|
||||
MaintenanceDashboardState.status.in_([
|
||||
MaintenanceDashboardStateStatus.ACTIVE,
|
||||
MaintenanceDashboardStateStatus.PENDING_APPLY,
|
||||
]),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
event_ids: list[str] = []
|
||||
start_times: list[str] = []
|
||||
end_times: list[str] = []
|
||||
|
||||
for state in linked_states:
|
||||
event = (
|
||||
db.query(MaintenanceEvent)
|
||||
.filter(MaintenanceEvent.id == state.event_id)
|
||||
.first()
|
||||
)
|
||||
if event:
|
||||
event_ids.append(event.id)
|
||||
if event.start_time:
|
||||
start_times.append(event.start_time.isoformat())
|
||||
if event.end_time:
|
||||
end_times.append(event.end_time.isoformat())
|
||||
|
||||
result.append(
|
||||
MaintenanceDashboardBannerState(
|
||||
dashboard_id=banner.dashboard_id,
|
||||
active=True,
|
||||
events=event_ids,
|
||||
start_time=min(start_times) if start_times else None,
|
||||
end_time=max(end_times) if end_times else None,
|
||||
)
|
||||
)
|
||||
|
||||
return result
|
||||
# #endregion list_dashboard_banners
|
||||
|
||||
|
||||
# ── GET /api/maintenance/events ──────────────────────────────
|
||||
# FR-015: operator, admin (NOT viewer)
|
||||
|
||||
# #region list_events [C:2] [TYPE Function]
|
||||
# @BRIEF Get active and completed maintenance event lists with affected dashboard counts.
|
||||
# @RELATION DEPENDS_ON -> [has_permission("maintenance", "READ")]
|
||||
@router.get("/events", response_model=MaintenanceEventListResponse)
|
||||
async def list_events(
|
||||
db: Session = Depends(get_db),
|
||||
_=Depends(has_permission("maintenance", "READ")),
|
||||
task_manager: TaskManager = Depends(get_task_manager),
|
||||
):
|
||||
with belief_scope("list_events"):
|
||||
# ── Auto-expiry: end events past their end_time ──
|
||||
now = datetime.now(timezone.utc)
|
||||
expired = (
|
||||
db.query(MaintenanceEvent)
|
||||
.filter(
|
||||
MaintenanceEvent.status.in_([
|
||||
MaintenanceEventStatus.ACTIVE,
|
||||
MaintenanceEventStatus.PARTIAL,
|
||||
MaintenanceEventStatus.ENDING,
|
||||
]),
|
||||
MaintenanceEvent.end_time.isnot(None),
|
||||
MaintenanceEvent.end_time < now,
|
||||
)
|
||||
.all()
|
||||
)
|
||||
for ev in expired:
|
||||
app_logger.reason(
|
||||
"Auto-ending expired maintenance event",
|
||||
extra={"event_id": ev.id, "end_time": ev.end_time.isoformat()},
|
||||
)
|
||||
# Schedule proper cleanup via task manager (removes banner from dashboard)
|
||||
task = await task_manager.create_task(
|
||||
plugin_id="maintenance_banner_apply",
|
||||
params={
|
||||
"operation": "end",
|
||||
"event_id": ev.id,
|
||||
"environment_id": ev.environment_id,
|
||||
"user": "system",
|
||||
},
|
||||
)
|
||||
app_logger.reflect(
|
||||
"Scheduled cleanup for expired event",
|
||||
extra={"event_id": ev.id, "task_id": task.id},
|
||||
)
|
||||
|
||||
# Active events: ACTIVE, PARTIAL (per spec T040)
|
||||
active_statuses = [
|
||||
MaintenanceEventStatus.ACTIVE,
|
||||
MaintenanceEventStatus.PARTIAL,
|
||||
]
|
||||
active_events = (
|
||||
db.query(MaintenanceEvent)
|
||||
.filter(MaintenanceEvent.status.in_(active_statuses))
|
||||
.order_by(MaintenanceEvent.created_at.desc())
|
||||
.all()
|
||||
)
|
||||
|
||||
# Completed events: COMPLETED, FAILED
|
||||
completed_events = (
|
||||
db.query(MaintenanceEvent)
|
||||
.filter(
|
||||
MaintenanceEvent.status.in_([
|
||||
MaintenanceEventStatus.COMPLETED,
|
||||
MaintenanceEventStatus.FAILED,
|
||||
])
|
||||
)
|
||||
.order_by(MaintenanceEvent.created_at.desc())
|
||||
.limit(50)
|
||||
.all()
|
||||
)
|
||||
|
||||
def _to_item(event: MaintenanceEvent) -> MaintenanceEventItem:
|
||||
affected_count = (
|
||||
db.query(MaintenanceDashboardState)
|
||||
.filter(
|
||||
MaintenanceDashboardState.event_id == event.id,
|
||||
MaintenanceDashboardState.status.in_([
|
||||
MaintenanceDashboardStateStatus.ACTIVE,
|
||||
MaintenanceDashboardStateStatus.PENDING_APPLY,
|
||||
]),
|
||||
)
|
||||
.count()
|
||||
)
|
||||
return MaintenanceEventItem(
|
||||
id=event.id,
|
||||
tables=list(event.tables) if event.tables else [],
|
||||
start_time=event.start_time.isoformat() if event.start_time else "",
|
||||
end_time=event.end_time.isoformat() if event.end_time else None,
|
||||
message=event.message or None,
|
||||
status=event.status.value,
|
||||
affected_count=affected_count,
|
||||
created_at=event.created_at.isoformat() if event.created_at else "",
|
||||
)
|
||||
|
||||
return MaintenanceEventListResponse(
|
||||
active=[_to_item(e) for e in active_events],
|
||||
completed=[_to_item(e) for e in completed_events],
|
||||
)
|
||||
# #endregion list_events
|
||||
|
||||
|
||||
# ── POST /api/maintenance/start ──────────────────────────────
|
||||
# FR-001: Always returns 202 {task_id} after validation
|
||||
# FR-015: operator, admin (NOT viewer)
|
||||
|
||||
# #region start_maintenance [C:3] [TYPE Function]
|
||||
# @BRIEF Start a maintenance event. Validates input, creates event, dispatches TaskManager task.
|
||||
# Always returns 202 with task_id and maintenance_id.
|
||||
# Idempotency: same (tables, start_time, end_time) returns already_active (409-like but 200).
|
||||
# Tables sorted & lowered for idempotency key. Message NOT in key.
|
||||
# @RELATION DEPENDS_ON -> [has_permission("maintenance", "WRITE")]
|
||||
# @RELATION DEPENDS_ON -> [TaskManager]
|
||||
@router.post(
|
||||
"/start",
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
responses={
|
||||
202: {"model": MaintenanceStartResponse},
|
||||
409: {"model": MaintenanceAlreadyActiveResponse},
|
||||
},
|
||||
)
|
||||
async def start_maintenance(
|
||||
request: MaintenanceStartRequest,
|
||||
db: Session = Depends(get_db),
|
||||
user=Depends(get_current_user),
|
||||
_=Depends(has_permission("maintenance", "WRITE")),
|
||||
task_manager: TaskManager = Depends(get_task_manager),
|
||||
):
|
||||
with belief_scope("start_maintenance"):
|
||||
# ── Validation ──
|
||||
|
||||
# Validate end_time > start_time
|
||||
if request.end_time and request.start_time and request.end_time <= request.start_time:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="end_time must be after start_time",
|
||||
)
|
||||
|
||||
# Validate times are within reasonable range (future ±1h grace period)
|
||||
now = datetime.now(timezone.utc)
|
||||
grace_threshold = 3600 # 1 hour in seconds
|
||||
if request.start_time:
|
||||
diff = (request.start_time - now).total_seconds() if request.start_time.tzinfo else \
|
||||
(request.start_time.replace(tzinfo=timezone.utc) - now).total_seconds()
|
||||
if diff < -grace_threshold:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="start_time is too far in the past (max 1h grace period)",
|
||||
)
|
||||
|
||||
# Validate tables list length
|
||||
if len(request.tables) > 100:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Maximum 100 tables allowed",
|
||||
)
|
||||
|
||||
# Sanitize message (HTML escape already done in rendering, but check for raw HTML)
|
||||
message = request.message
|
||||
if message:
|
||||
import html as _html
|
||||
sanitized = _html.escape(message)
|
||||
if sanitized != message:
|
||||
# Message contained HTML — accept it, it will be escaped at render time
|
||||
pass
|
||||
|
||||
# ── Idempotency check ──
|
||||
from ....services.maintenance import build_idempotency_key
|
||||
|
||||
sorted_tables = sorted(t.strip().lower() for t in request.tables if t.strip())
|
||||
idem_tables = frozenset(sorted_tables)
|
||||
idem_start = request.start_time.isoformat() if request.start_time else None
|
||||
idem_end = request.end_time.isoformat() if request.end_time else None
|
||||
|
||||
# Find existing active event with same idempotency key
|
||||
active_statuses = [
|
||||
MaintenanceEventStatus.PENDING,
|
||||
MaintenanceEventStatus.APPLYING,
|
||||
MaintenanceEventStatus.ACTIVE,
|
||||
MaintenanceEventStatus.PARTIAL,
|
||||
]
|
||||
existing = (
|
||||
db.query(MaintenanceEvent)
|
||||
.filter(
|
||||
MaintenanceEvent.status.in_(active_statuses),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
for ev in existing:
|
||||
ev_tables = frozenset(
|
||||
sorted(t.lower() for t in (ev.tables or []) if t)
|
||||
)
|
||||
ev_start = ev.start_time.isoformat() if ev.start_time else None
|
||||
ev_end = ev.end_time.isoformat() if ev.end_time else None
|
||||
if ev_tables == idem_tables and ev_start == idem_start and ev_end == idem_end:
|
||||
# Same key → return already_active
|
||||
app_logger.reflect(
|
||||
"Idempotency hit — event already active",
|
||||
extra={"existing_id": ev.id},
|
||||
)
|
||||
return MaintenanceAlreadyActiveResponse(
|
||||
maintenance_id=ev.id,
|
||||
status="already_active",
|
||||
)
|
||||
|
||||
# ── Load settings for environment ──
|
||||
settings = db.query(MaintenanceSettings).filter(
|
||||
MaintenanceSettings.id == "default"
|
||||
).first()
|
||||
env_id = settings.target_environment_id if settings else ""
|
||||
|
||||
# ── Create event ──
|
||||
event = MaintenanceEvent(
|
||||
tables=sorted_tables,
|
||||
start_time=request.start_time,
|
||||
end_time=request.end_time,
|
||||
message=message,
|
||||
status=MaintenanceEventStatus.PENDING,
|
||||
environment_id=env_id,
|
||||
)
|
||||
db.add(event)
|
||||
db.flush()
|
||||
|
||||
# ── Dispatch TaskManager task ──
|
||||
task = await task_manager.create_task(
|
||||
plugin_id="maintenance_banner_apply",
|
||||
params={
|
||||
"operation": "start",
|
||||
"event_id": event.id,
|
||||
"environment_id": env_id,
|
||||
"user": getattr(user, "username", "system"),
|
||||
},
|
||||
)
|
||||
event.task_id = task.id
|
||||
db.commit()
|
||||
|
||||
app_logger.reflect(
|
||||
"Maintenance start dispatched",
|
||||
extra={"event_id": event.id, "task_id": task.id},
|
||||
)
|
||||
|
||||
return MaintenanceStartResponse(
|
||||
task_id=task.id,
|
||||
maintenance_id=event.id,
|
||||
status="pending",
|
||||
)
|
||||
# #endregion start_maintenance
|
||||
|
||||
|
||||
# ── POST /api/maintenance/{maintenance_id}/end ───────────────
|
||||
# FR-006: Remove banners for a specific maintenance event
|
||||
# FR-015: operator, admin (NOT viewer)
|
||||
|
||||
# #region end_maintenance [C:3] [TYPE Function]
|
||||
# @BRIEF End a specific maintenance event by ID. Removes banners from affected dashboards.
|
||||
# Always returns 202 with task_id. Idempotent: already_completed returns success.
|
||||
# @RELATION DEPENDS_ON -> [has_permission("maintenance", "WRITE")]
|
||||
@router.post(
|
||||
"/{maintenance_id}/end",
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
response_model=MaintenanceEndResponse,
|
||||
)
|
||||
async def end_maintenance(
|
||||
maintenance_id: str,
|
||||
db: Session = Depends(get_db),
|
||||
user=Depends(get_current_user),
|
||||
_=Depends(has_permission("maintenance", "WRITE")),
|
||||
task_manager: TaskManager = Depends(get_task_manager),
|
||||
):
|
||||
with belief_scope("end_maintenance", f"maintenance_id={maintenance_id}"):
|
||||
# Check event exists
|
||||
event = db.query(MaintenanceEvent).filter(
|
||||
MaintenanceEvent.id == maintenance_id
|
||||
).first()
|
||||
|
||||
if not event:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Maintenance event {maintenance_id} not found",
|
||||
)
|
||||
|
||||
# If already completed, return idempotent success
|
||||
if event.status == MaintenanceEventStatus.COMPLETED:
|
||||
app_logger.reflect(
|
||||
"Event already completed, returning idempotent",
|
||||
extra={"event_id": maintenance_id},
|
||||
)
|
||||
return MaintenanceEndResponse(
|
||||
task_id=event.task_id or "",
|
||||
status="already_completed",
|
||||
)
|
||||
|
||||
# ── Dispatch TaskManager task ──
|
||||
task = await task_manager.create_task(
|
||||
plugin_id="maintenance_banner_apply",
|
||||
params={
|
||||
"operation": "end",
|
||||
"event_id": maintenance_id,
|
||||
"environment_id": event.environment_id,
|
||||
"user": getattr(user, "username", "system"),
|
||||
},
|
||||
)
|
||||
|
||||
app_logger.reflect(
|
||||
"Maintenance end dispatched",
|
||||
extra={"event_id": maintenance_id, "task_id": task.id},
|
||||
)
|
||||
|
||||
return MaintenanceEndResponse(
|
||||
task_id=task.id,
|
||||
status="pending",
|
||||
)
|
||||
# #endregion end_maintenance
|
||||
|
||||
|
||||
# ── POST /api/maintenance/end-all ────────────────────────────
|
||||
# FR-007: Bulk removal of all active banners
|
||||
# FR-015: operator, admin (NOT viewer)
|
||||
|
||||
# #region end_all_maintenance [C:3] [TYPE Function]
|
||||
# @BRIEF End all active maintenance events. Removes all banners from all dashboards.
|
||||
# Always returns 202 with task_id.
|
||||
# @RELATION DEPENDS_ON -> [has_permission("maintenance", "WRITE")]
|
||||
@router.post(
|
||||
"/end-all",
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
response_model=MaintenanceEndResponse,
|
||||
)
|
||||
async def end_all_maintenance(
|
||||
db: Session = Depends(get_db),
|
||||
user=Depends(get_current_user),
|
||||
_=Depends(has_permission("maintenance", "WRITE")),
|
||||
task_manager: TaskManager = Depends(get_task_manager),
|
||||
):
|
||||
with belief_scope("end_all_maintenance"):
|
||||
# ── Dispatch TaskManager task ──
|
||||
task = await task_manager.create_task(
|
||||
plugin_id="maintenance_banner_apply",
|
||||
params={
|
||||
"operation": "end_all",
|
||||
"user": getattr(user, "username", "system"),
|
||||
},
|
||||
)
|
||||
|
||||
app_logger.reflect(
|
||||
"End-all maintenance dispatched",
|
||||
extra={"task_id": task.id},
|
||||
)
|
||||
|
||||
return MaintenanceEndResponse(
|
||||
task_id=task.id,
|
||||
status="pending",
|
||||
)
|
||||
# #endregion end_all_maintenance
|
||||
|
||||
|
||||
# ── GET /api/maintenance/settings ────────────────────────────
|
||||
# FR-009: Get maintenance settings configuration
|
||||
# FR-015: operator, admin (NOT viewer)
|
||||
|
||||
# #region get_maintenance_settings [C:2] [TYPE Function]
|
||||
# @BRIEF Get current maintenance settings configuration.
|
||||
# @RELATION DEPENDS_ON -> [has_permission("maintenance", "READ")]
|
||||
@router.get("/settings", response_model=MaintenanceSettingsResponse)
|
||||
async def get_maintenance_settings(
|
||||
db: Session = Depends(get_db),
|
||||
_=Depends(has_permission("maintenance", "READ")),
|
||||
):
|
||||
with belief_scope("get_maintenance_settings"):
|
||||
settings = db.query(MaintenanceSettings).filter(
|
||||
MaintenanceSettings.id == "default"
|
||||
).first()
|
||||
|
||||
if not settings:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="Maintenance settings not configured",
|
||||
)
|
||||
|
||||
return MaintenanceSettingsResponse(
|
||||
target_environment_id=settings.target_environment_id,
|
||||
display_timezone=settings.display_timezone,
|
||||
banner_template=settings.banner_template,
|
||||
dashboard_scope=settings.dashboard_scope.value,
|
||||
excluded_dashboard_ids=list(settings.excluded_dashboard_ids or []),
|
||||
forced_dashboard_ids=list(settings.forced_dashboard_ids or []),
|
||||
updated_at=settings.updated_at.isoformat() if settings.updated_at else None,
|
||||
)
|
||||
# #endregion get_maintenance_settings
|
||||
|
||||
|
||||
# ── PUT /api/maintenance/settings ────────────────────────────
|
||||
# FR-009: Update maintenance settings configuration
|
||||
# FR-015: admin ONLY (NOT operator, NOT viewer)
|
||||
|
||||
# #region update_maintenance_settings [C:2] [TYPE Function]
|
||||
# @BRIEF Update maintenance settings. All fields optional for partial update.
|
||||
# admin role required per FR-015.
|
||||
# @RELATION DEPENDS_ON -> [has_permission("admin:settings", "WRITE")]
|
||||
@router.put("/settings", response_model=MaintenanceSettingsResponse)
|
||||
async def update_maintenance_settings(
|
||||
settings_data: MaintenanceSettingsUpdate,
|
||||
db: Session = Depends(get_db),
|
||||
_=Depends(has_permission("admin:settings", "WRITE")),
|
||||
):
|
||||
with belief_scope("update_maintenance_settings"):
|
||||
settings = db.query(MaintenanceSettings).filter(
|
||||
MaintenanceSettings.id == "default"
|
||||
).first()
|
||||
|
||||
if not settings:
|
||||
# Auto-create with defaults on first save (upsert behavior).
|
||||
# Column-level defaults from the model apply for omitted fields.
|
||||
settings = MaintenanceSettings(
|
||||
id="default",
|
||||
target_environment_id=settings_data.target_environment_id or "",
|
||||
)
|
||||
db.add(settings)
|
||||
db.flush()
|
||||
|
||||
# Apply partial updates
|
||||
update_data = settings_data.model_dump(exclude_unset=True)
|
||||
|
||||
if "target_environment_id" in update_data:
|
||||
settings.target_environment_id = update_data["target_environment_id"]
|
||||
if "display_timezone" in update_data:
|
||||
settings.display_timezone = update_data["display_timezone"]
|
||||
if "banner_template" in update_data:
|
||||
settings.banner_template = update_data["banner_template"]
|
||||
if "dashboard_scope" in update_data:
|
||||
from ....models.maintenance import DashboardScope
|
||||
settings.dashboard_scope = DashboardScope(update_data["dashboard_scope"])
|
||||
if "excluded_dashboard_ids" in update_data:
|
||||
settings.excluded_dashboard_ids = update_data["excluded_dashboard_ids"]
|
||||
if "forced_dashboard_ids" in update_data:
|
||||
settings.forced_dashboard_ids = update_data["forced_dashboard_ids"]
|
||||
|
||||
db.commit()
|
||||
db.refresh(settings)
|
||||
|
||||
return MaintenanceSettingsResponse(
|
||||
target_environment_id=settings.target_environment_id,
|
||||
display_timezone=settings.display_timezone,
|
||||
banner_template=settings.banner_template,
|
||||
dashboard_scope=settings.dashboard_scope.value,
|
||||
excluded_dashboard_ids=list(settings.excluded_dashboard_ids or []),
|
||||
forced_dashboard_ids=list(settings.forced_dashboard_ids or []),
|
||||
updated_at=settings.updated_at.isoformat() if settings.updated_at else None,
|
||||
)
|
||||
# #endregion update_maintenance_settings
|
||||
|
||||
# #endregion MaintenanceRoutesModule
|
||||
156
backend/src/api/routes/maintenance/_schemas.py
Normal file
156
backend/src/api/routes/maintenance/_schemas.py
Normal file
@@ -0,0 +1,156 @@
|
||||
# #region MaintenanceSchemasModule [C:2] [TYPE Module] [SEMANTICS pydantic, schema, maintenance, request, response]
|
||||
# @BRIEF Pydantic models for Maintenance Banner API: request/response schemas per data-model.md.
|
||||
# @LAYER API
|
||||
# @RELATION DEPENDS_ON -> [Pydantic]
|
||||
# @INVARIANT All response schemas use the standard envelope shape: { status, data, error, meta }
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
# ── Request Schemas ──────────────────────────────────────────
|
||||
|
||||
# #region MaintenanceStartRequest [C:1] [TYPE Class]
|
||||
# @BRIEF POST /api/maintenance/start request body.
|
||||
class MaintenanceStartRequest(BaseModel):
|
||||
tables: list[str] = Field(..., min_length=1, max_length=100)
|
||||
start_time: datetime
|
||||
end_time: datetime | None = None
|
||||
message: str | None = Field(None, max_length=500)
|
||||
# #endregion MaintenanceStartRequest
|
||||
|
||||
|
||||
# #region MaintenanceSettingsUpdate [C:1] [TYPE Class]
|
||||
# @BRIEF PUT /api/maintenance/settings request body — all fields optional for partial update.
|
||||
class MaintenanceSettingsUpdate(BaseModel):
|
||||
target_environment_id: str | None = None
|
||||
display_timezone: str | None = None
|
||||
banner_template: str | None = None
|
||||
dashboard_scope: str | None = None
|
||||
excluded_dashboard_ids: list[int] | None = None
|
||||
forced_dashboard_ids: list[int] | None = None
|
||||
# #endregion MaintenanceSettingsUpdate
|
||||
|
||||
|
||||
# ── Response Schemas ─────────────────────────────────────────
|
||||
|
||||
# #region MaintenanceStartResponse [C:1] [TYPE Class]
|
||||
# @BRIEF Response for /api/maintenance/start — always 202 with task_id.
|
||||
class MaintenanceStartResponse(BaseModel):
|
||||
task_id: str
|
||||
maintenance_id: str
|
||||
status: str # "pending"
|
||||
# #endregion MaintenanceStartResponse
|
||||
|
||||
|
||||
# #region MaintenanceDashboardResult [C:1] [TYPE Class]
|
||||
# @BRIEF Describes a single dashboard affected by a maintenance operation.
|
||||
class MaintenanceDashboardResult(BaseModel):
|
||||
id: int
|
||||
title: str
|
||||
chart_id: int | None = None
|
||||
# #endregion MaintenanceDashboardResult
|
||||
|
||||
|
||||
# #region MaintenanceFailedDashboard [C:1] [TYPE Class]
|
||||
# @BRIEF Describes a dashboard that failed during banner apply/removal.
|
||||
class MaintenanceFailedDashboard(BaseModel):
|
||||
id: int
|
||||
title: str
|
||||
error: str
|
||||
# #endregion MaintenanceFailedDashboard
|
||||
|
||||
|
||||
# #region MaintenanceTaskResult [C:1] [TYPE Class]
|
||||
# @BRIEF Result payload for a completed maintenance start task.
|
||||
class MaintenanceTaskResult(BaseModel):
|
||||
maintenance_id: str
|
||||
status: str # "active" | "partial" | "completed" | "no_match"
|
||||
affected_dashboards: int = 0
|
||||
dashboards: list[MaintenanceDashboardResult] = []
|
||||
failed_dashboards: list[MaintenanceFailedDashboard] = []
|
||||
unmatched_tables: list[str] = []
|
||||
# #endregion MaintenanceTaskResult
|
||||
|
||||
|
||||
# #region MaintenanceEndResponse [C:1] [TYPE Class]
|
||||
# @BRIEF Response for /api/maintenance/{id}/end and /api/maintenance/end-all — always 202 with task_id.
|
||||
class MaintenanceEndResponse(BaseModel):
|
||||
task_id: str
|
||||
status: str # "pending"
|
||||
# #endregion MaintenanceEndResponse
|
||||
|
||||
|
||||
# #region MaintenanceEndTaskResult [C:1] [TYPE Class]
|
||||
# @BRIEF Result payload for a completed end maintenance task.
|
||||
class MaintenanceEndTaskResult(BaseModel):
|
||||
maintenance_id: str
|
||||
removed_from: int = 0
|
||||
failed_removals: list[MaintenanceFailedDashboard] = []
|
||||
# #endregion MaintenanceEndTaskResult
|
||||
|
||||
|
||||
# #region MaintenanceEndAllTaskResult [C:1] [TYPE Class]
|
||||
# @BRIEF Result payload for a completed end-all maintenance task.
|
||||
class MaintenanceEndAllTaskResult(BaseModel):
|
||||
closed_events: int = 0
|
||||
removed_from: int = 0
|
||||
failed_removals: list[MaintenanceFailedDashboard] = []
|
||||
# #endregion MaintenanceEndAllTaskResult
|
||||
|
||||
|
||||
# #region MaintenanceSettingsResponse [C:1] [TYPE Class]
|
||||
# @BRIEF Response for GET /api/maintenance/settings.
|
||||
class MaintenanceSettingsResponse(BaseModel):
|
||||
target_environment_id: str
|
||||
display_timezone: str
|
||||
banner_template: str
|
||||
dashboard_scope: str
|
||||
excluded_dashboard_ids: list[int]
|
||||
forced_dashboard_ids: list[int]
|
||||
updated_at: str | None = None
|
||||
# #endregion MaintenanceSettingsResponse
|
||||
|
||||
|
||||
# #region MaintenanceEventItem [C:1] [TYPE Class]
|
||||
# @BRIEF Single maintenance event summary for GET /api/maintenance/events.
|
||||
class MaintenanceEventItem(BaseModel):
|
||||
id: str
|
||||
tables: list[str]
|
||||
start_time: str
|
||||
end_time: str | None = None
|
||||
message: str | None = None
|
||||
status: str
|
||||
affected_count: int
|
||||
created_at: str
|
||||
# #endregion MaintenanceEventItem
|
||||
|
||||
|
||||
# #region MaintenanceEventListResponse [C:1] [TYPE Class]
|
||||
# @BRIEF Response for GET /api/maintenance/events.
|
||||
class MaintenanceEventListResponse(BaseModel):
|
||||
active: list[MaintenanceEventItem]
|
||||
completed: list[MaintenanceEventItem]
|
||||
# #endregion MaintenanceEventListResponse
|
||||
|
||||
|
||||
# #region MaintenanceDashboardBannerState [C:1] [TYPE Class]
|
||||
# @BRIEF Per-dashboard banner state for GET /api/maintenance/dashboard-banners.
|
||||
class MaintenanceDashboardBannerState(BaseModel):
|
||||
dashboard_id: int
|
||||
active: bool
|
||||
events: list[str]
|
||||
start_time: str | None = None
|
||||
end_time: str | None = None
|
||||
# #endregion MaintenanceDashboardBannerState
|
||||
|
||||
|
||||
# #region MaintenanceAlreadyActiveResponse [C:1] [TYPE Class]
|
||||
# @BRIEF Response for duplicate /start call — idempotency hit.
|
||||
class MaintenanceAlreadyActiveResponse(BaseModel):
|
||||
maintenance_id: str
|
||||
status: str = "already_active"
|
||||
# #endregion MaintenanceAlreadyActiveResponse
|
||||
|
||||
# #endregion MaintenanceSchemasModule
|
||||
@@ -393,6 +393,7 @@ class ConsolidatedSettingsResponse(BaseModel):
|
||||
storage: dict
|
||||
notifications: dict = {}
|
||||
features: dict = {}
|
||||
app_timezone: str = "Europe/Moscow"
|
||||
|
||||
|
||||
# #endregion ConsolidatedSettingsResponse
|
||||
@@ -463,6 +464,7 @@ async def get_consolidated_settings(
|
||||
storage=config.settings.storage.dict(),
|
||||
notifications=notifications_payload,
|
||||
features=config.settings.features.model_dump(),
|
||||
app_timezone=config.settings.app_timezone,
|
||||
)
|
||||
logger.reflect(
|
||||
"Consolidated settings payload assembled",
|
||||
@@ -526,6 +528,15 @@ async def update_consolidated_settings(
|
||||
payload["notifications"] = settings_patch["notifications"]
|
||||
config_manager.save_config(payload)
|
||||
|
||||
# Update app_timezone if provided
|
||||
if "app_timezone" in settings_patch:
|
||||
new_tz = settings_patch["app_timezone"]
|
||||
from ...core.timezone import validate_timezone, invalidate_timezone_cache
|
||||
if not validate_timezone(new_tz):
|
||||
raise HTTPException(status_code=422, detail=f"Invalid IANA timezone: {new_tz}")
|
||||
current_settings.app_timezone = new_tz
|
||||
invalidate_timezone_cache()
|
||||
|
||||
config_manager.update_global_settings(current_settings)
|
||||
return {"status": "success", "message": "Settings updated"}
|
||||
|
||||
|
||||
@@ -13,8 +13,9 @@ from sqlalchemy.orm import Session
|
||||
from ...core.config_manager import ConfigManager
|
||||
from ...core.database import get_db
|
||||
from ...core.logger import logger
|
||||
from ...core.scheduler import SchedulerService
|
||||
from ...core.task_manager import TaskManager
|
||||
from ...dependencies import get_config_manager, get_current_user, get_task_manager, has_permission
|
||||
from ...dependencies import get_config_manager, get_current_user, get_scheduler_service, get_task_manager, has_permission
|
||||
from ...schemas.auth import User
|
||||
from ...schemas.validation import (
|
||||
TriggerRunResponse,
|
||||
@@ -88,11 +89,14 @@ async def create_task(
|
||||
current_user: User = Depends(get_current_user),
|
||||
_=Depends(has_permission("validation.task", "CREATE")),
|
||||
service: ValidationTaskService = Depends(_get_task_service),
|
||||
scheduler: SchedulerService = Depends(get_scheduler_service),
|
||||
):
|
||||
"""Create a new validation task with provider and environment validation."""
|
||||
logger.reason(f"create_task — User: {current_user.username}, name: {payload.name}", extra={"src": "validation_routes"})
|
||||
try:
|
||||
return service.create_task(payload)
|
||||
result = service.create_task(payload)
|
||||
scheduler.reload_validation_policy(result.id)
|
||||
return result
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))
|
||||
# #endregion create_task
|
||||
@@ -114,8 +118,9 @@ async def get_task(
|
||||
try:
|
||||
task = task_service.get_task(task_id)
|
||||
from ...schemas.validation import ValidationRunResponse
|
||||
# Find recent runs via run_service using dashboard matching
|
||||
# Find recent runs for this specific task via task_id
|
||||
recent = run_service.list_runs(
|
||||
task_id=task_id,
|
||||
dashboard_id=task.dashboard_ids[0] if task.dashboard_ids else None,
|
||||
environment_id=task.environment_id,
|
||||
page=1,
|
||||
@@ -140,11 +145,14 @@ async def update_task(
|
||||
current_user: User = Depends(get_current_user),
|
||||
_=Depends(has_permission("validation.task", "EDIT")),
|
||||
service: ValidationTaskService = Depends(_get_task_service),
|
||||
scheduler: SchedulerService = Depends(get_scheduler_service),
|
||||
):
|
||||
"""Update a validation task."""
|
||||
logger.reason(f"update_task — Task: {task_id}, User: {current_user.username}", extra={"src": "validation_routes"})
|
||||
try:
|
||||
return service.update_task(task_id, payload)
|
||||
result = service.update_task(task_id, payload)
|
||||
scheduler.reload_validation_policy(result.id)
|
||||
return result
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
|
||||
# #endregion update_task
|
||||
@@ -160,11 +168,13 @@ async def delete_task(
|
||||
current_user: User = Depends(get_current_user),
|
||||
_=Depends(has_permission("validation.task", "DELETE")),
|
||||
service: ValidationTaskService = Depends(_get_task_service),
|
||||
scheduler: SchedulerService = Depends(get_scheduler_service),
|
||||
):
|
||||
"""Delete a validation task."""
|
||||
logger.reason(f"delete_task — Task: {task_id}, User: {current_user.username}", extra={"src": "validation_routes"})
|
||||
try:
|
||||
service.delete_task(task_id, delete_runs=delete_runs)
|
||||
scheduler.remove_validation_job(task_id)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
|
||||
# #endregion delete_task
|
||||
|
||||
@@ -35,6 +35,7 @@ from .api.routes import (
|
||||
git,
|
||||
health,
|
||||
llm,
|
||||
maintenance,
|
||||
mappings,
|
||||
migration,
|
||||
plugins,
|
||||
@@ -219,6 +220,7 @@ app.add_middleware(TraceContextMiddleware)
|
||||
# @RELATION DEPENDS_ON -> [LlmRoutes]
|
||||
# @RELATION DEPENDS_ON -> [CleanReleaseV2Api]
|
||||
# @RELATION DEPENDS_ON -> [ValidationRoutes]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceRouter]
|
||||
# Include API routes
|
||||
app.include_router(auth.router)
|
||||
app.include_router(admin.router)
|
||||
@@ -242,6 +244,7 @@ app.include_router(dataset_review.router)
|
||||
app.include_router(health.router)
|
||||
app.include_router(translate.router)
|
||||
app.include_router(validation.router, prefix="/api/validation", tags=["Validation"])
|
||||
app.include_router(maintenance.maintenance_router)
|
||||
# #endregion API_Routes
|
||||
# #region api.include_routers [C:1] [TYPE Action] [SEMANTICS routes, registration, api]
|
||||
# @BRIEF Registers all API routers with the FastAPI application.
|
||||
|
||||
@@ -82,6 +82,14 @@ class ConfigManager:
|
||||
"Applied FEATURES__HEALTH_MONITOR from env",
|
||||
extra={"value": parsed, "raw": health_monitor_env},
|
||||
)
|
||||
|
||||
app_timezone_env = os.getenv("APP_TIMEZONE")
|
||||
if app_timezone_env is not None:
|
||||
settings.app_timezone = app_timezone_env.strip()
|
||||
logger.reason(
|
||||
"Applied APP_TIMEZONE from env",
|
||||
extra={"value": settings.app_timezone},
|
||||
)
|
||||
# #endregion _apply_features_from_env
|
||||
# #region _default_config [TYPE Function]
|
||||
# @PURPOSE: Build default application configuration fallback.
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
# @RELATION IMPLEMENTS -> [ConnectionContracts]
|
||||
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from ..models.storage import StorageConfig
|
||||
from ..services.llm_prompt_templates import (
|
||||
@@ -101,6 +101,19 @@ class GlobalSettings(BaseModel):
|
||||
}
|
||||
)
|
||||
|
||||
# Application timezone
|
||||
app_timezone: str = "Europe/Moscow"
|
||||
|
||||
@field_validator("app_timezone")
|
||||
@classmethod
|
||||
def validate_app_timezone(cls, v: str) -> str:
|
||||
from zoneinfo import ZoneInfo
|
||||
try:
|
||||
ZoneInfo(v)
|
||||
except (KeyError, TypeError):
|
||||
raise ValueError(f"Invalid IANA timezone: {v}") from None
|
||||
return v
|
||||
|
||||
# Task retention settings
|
||||
task_retention_days: int = 30
|
||||
task_retention_limit: int = 100
|
||||
|
||||
@@ -21,6 +21,7 @@ from ..models import (
|
||||
config as _config_models, # noqa: F401
|
||||
dataset_review as _dataset_review_models, # noqa: F401
|
||||
llm as _llm_models, # noqa: F401
|
||||
maintenance as _maintenance_models, # noqa: F401
|
||||
profile as _profile_models, # noqa: F401
|
||||
task as _task_models, # noqa: F401
|
||||
)
|
||||
|
||||
@@ -271,7 +271,10 @@ def explore(self, msg, *args, **kwargs):
|
||||
Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.
|
||||
"""
|
||||
user_extra = kwargs.pop('extra', {})
|
||||
error_val = kwargs.pop('error', None)
|
||||
extra = {'marker': 'EXPLORE', 'intent': msg}
|
||||
if error_val is not None:
|
||||
extra['error'] = error_val
|
||||
extra.update(user_extra)
|
||||
self.warning(msg, *args, extra=extra, **kwargs)
|
||||
# #endregion explore
|
||||
@@ -285,7 +288,10 @@ def reason(self, msg, *args, **kwargs):
|
||||
Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.
|
||||
"""
|
||||
user_extra = kwargs.pop('extra', {})
|
||||
error_val = kwargs.pop('error', None)
|
||||
extra = {'marker': 'REASON', 'intent': msg}
|
||||
if error_val is not None:
|
||||
extra['error'] = error_val
|
||||
extra.update(user_extra)
|
||||
self.debug(msg, *args, extra=extra, **kwargs)
|
||||
# #endregion reason
|
||||
@@ -299,7 +305,10 @@ def reflect(self, msg, *args, **kwargs):
|
||||
Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.
|
||||
"""
|
||||
user_extra = kwargs.pop('extra', {})
|
||||
error_val = kwargs.pop('error', None)
|
||||
extra = {'marker': 'REFLECT', 'intent': msg}
|
||||
if error_val is not None:
|
||||
extra['error'] = error_val
|
||||
extra.update(user_extra)
|
||||
self.debug(msg, *args, extra=extra, **kwargs)
|
||||
# #endregion reflect
|
||||
|
||||
@@ -88,6 +88,41 @@ class SchedulerService:
|
||||
except Exception as e:
|
||||
logger.explore("Failed to load translation schedules on startup",
|
||||
extra={"error": str(e)})
|
||||
# Re-register active validation policies with schedules
|
||||
try:
|
||||
from ..models.llm import ValidationPolicy
|
||||
db = SessionLocal()
|
||||
try:
|
||||
active_policies = (
|
||||
db.query(ValidationPolicy)
|
||||
.filter(
|
||||
ValidationPolicy.is_active.is_(True),
|
||||
ValidationPolicy.schedule_days.isnot(None),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
for policy in active_policies:
|
||||
schedule_days = policy.schedule_days or []
|
||||
if not schedule_days:
|
||||
continue
|
||||
ws = policy.window_start
|
||||
if ws is None:
|
||||
continue
|
||||
# Build cron: minute hour * * day-of-week
|
||||
dow = ",".join(str(d) for d in sorted(schedule_days))
|
||||
cron = f"{ws.minute} {ws.hour} * * {dow}"
|
||||
self.add_validation_job(
|
||||
policy_id=policy.id,
|
||||
cron_expression=cron,
|
||||
app_timezone=config.settings.app_timezone,
|
||||
)
|
||||
if active_policies:
|
||||
logger.reason(f"Loaded {len(active_policies)} active validation schedule(s)")
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as e:
|
||||
logger.explore("Failed to load validation schedules on startup",
|
||||
extra={"error": str(e)})
|
||||
# #endregion load_schedules
|
||||
# #region add_backup_job [TYPE Function]
|
||||
# @PURPOSE: Adds a scheduled backup job for an environment.
|
||||
@@ -197,6 +232,153 @@ class SchedulerService:
|
||||
self.loop,
|
||||
)
|
||||
# #endregion _trigger_backup
|
||||
# #region add_validation_job [C:3] [TYPE Function] [SEMANTICS scheduler,validation,cron,apscheduler]
|
||||
# @BRIEF Register a validation policy schedule with APScheduler.
|
||||
# @PRE policy_id and cron_expression are valid strings.
|
||||
# @POST A new APScheduler job is registered or replaced if it already exists.
|
||||
# @SIDE_EFFECT Mutates APScheduler state; calls _trigger_validation on trigger.
|
||||
# @RELATION DEPENDS_ON -> [_trigger_validation]
|
||||
def add_validation_job(self, policy_id: str, cron_expression: str, app_timezone: str = "Europe/Moscow") -> None:
|
||||
with belief_scope(
|
||||
"SchedulerService.add_validation_job",
|
||||
f"policy_id={policy_id}, cron={cron_expression}",
|
||||
):
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
job_id_aps = f"validation_{policy_id}"
|
||||
try:
|
||||
tz = ZoneInfo(app_timezone)
|
||||
self.scheduler.add_job(
|
||||
self._trigger_validation,
|
||||
CronTrigger.from_crontab(cron_expression, timezone=tz),
|
||||
id=job_id_aps,
|
||||
args=[policy_id],
|
||||
replace_existing=True,
|
||||
)
|
||||
logger.reason(
|
||||
f"Validation schedule registered: {job_id_aps} ({cron_expression}) [{app_timezone}]"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.explore(
|
||||
f"Failed to register validation schedule: {job_id_aps}",
|
||||
extra={"error": str(e)},
|
||||
)
|
||||
# #endregion add_validation_job
|
||||
# #region remove_validation_job [C:2] [TYPE Function] [SEMANTICS scheduler,validation,remove,apscheduler]
|
||||
# @BRIEF Remove a validation policy schedule from APScheduler.
|
||||
# @PRE policy_id is a valid string.
|
||||
# @POST The APScheduler job is removed if it exists; silently ignored otherwise.
|
||||
# @SIDE_EFFECT Mutates APScheduler state.
|
||||
def remove_validation_job(self, policy_id: str) -> None:
|
||||
with belief_scope(
|
||||
"SchedulerService.remove_validation_job",
|
||||
f"policy_id={policy_id}",
|
||||
):
|
||||
job_id_aps = f"validation_{policy_id}"
|
||||
try:
|
||||
self.scheduler.remove_job(job_id_aps)
|
||||
logger.info(f"Validation schedule removed: {job_id_aps}")
|
||||
except Exception:
|
||||
logger.reason(f"Validation schedule not found (already removed): {job_id_aps}")
|
||||
# #endregion remove_validation_job
|
||||
# #region reload_validation_policy [C:2] [TYPE Function] [SEMANTICS scheduler,validation,reload,apscheduler]
|
||||
# @BRIEF Reload a single validation policy schedule — calls remove then add.
|
||||
# @PRE policy_id is a valid ValidationPolicy id with schedule data in DB.
|
||||
# @POST Old job is removed; new job is registered if policy is active and has schedule_days.
|
||||
# @SIDE_EFFECT Mutates APScheduler state; queries ValidationPolicy from DB.
|
||||
def reload_validation_policy(self, policy_id: str, app_timezone: str = "Europe/Moscow") -> None:
|
||||
with belief_scope(
|
||||
"SchedulerService.reload_validation_policy",
|
||||
f"policy_id={policy_id}",
|
||||
):
|
||||
self.remove_validation_job(policy_id)
|
||||
try:
|
||||
from ..models.llm import ValidationPolicy
|
||||
db = SessionLocal()
|
||||
try:
|
||||
policy = db.query(ValidationPolicy).filter(ValidationPolicy.id == policy_id).first()
|
||||
if not policy:
|
||||
logger.reason(f"Policy {policy_id} not found; schedule removed")
|
||||
return
|
||||
if not policy.is_active:
|
||||
logger.reason(f"Policy {policy_id} is inactive; schedule removed")
|
||||
return
|
||||
schedule_days = policy.schedule_days or []
|
||||
if not schedule_days or policy.window_start is None:
|
||||
logger.reason(f"Policy {policy_id} has no schedule; nothing to register")
|
||||
return
|
||||
dow = ",".join(str(d) for d in sorted(schedule_days))
|
||||
ws = policy.window_start
|
||||
cron = f"{ws.minute} {ws.hour} * * {dow}"
|
||||
self.add_validation_job(policy_id, cron, app_timezone=app_timezone)
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as e:
|
||||
logger.explore(
|
||||
f"Failed to reload validation policy {policy_id}",
|
||||
extra={"error": str(e)},
|
||||
)
|
||||
# #endregion reload_validation_policy
|
||||
# #region _trigger_validation [C:3] [TYPE Function] [SEMANTICS scheduler,validation,trigger]
|
||||
# @BRIEF APScheduler job handler — triggers validation runs for a policy.
|
||||
# @PRE policy_id is a valid ValidationPolicy id with dashboard_ids.
|
||||
# @POST A validation task is spawned via TaskManager for each dashboard in the policy.
|
||||
# @SIDE_EFFECT Creates TaskRecord entries for validation runs; logs progress.
|
||||
def _trigger_validation(self, policy_id: str) -> None:
|
||||
seed_trace_id()
|
||||
with belief_scope("SchedulerService._trigger_validation", f"policy_id={policy_id}"):
|
||||
logger.reason(f"Triggering scheduled validation for policy {policy_id}")
|
||||
db = SessionLocal()
|
||||
try:
|
||||
from ..models.llm import ValidationPolicy
|
||||
|
||||
policy = db.query(ValidationPolicy).filter(ValidationPolicy.id == policy_id).first()
|
||||
if not policy:
|
||||
logger.explore(f"Validation policy not found: {policy_id}")
|
||||
return
|
||||
if not policy.is_active:
|
||||
logger.reason(f"Validation policy {policy_id} is no longer active; skipping")
|
||||
return
|
||||
dashboard_ids = list(policy.dashboard_ids or [])
|
||||
if not dashboard_ids:
|
||||
logger.explore(f"Validation policy {policy_id} has no dashboards; skipping")
|
||||
return
|
||||
|
||||
ws = policy.window_start
|
||||
we = policy.window_end
|
||||
today = date.today()
|
||||
scheduled_times = ThrottledSchedulerConfigurator.calculate_schedule(
|
||||
window_start=ws,
|
||||
window_end=we or time(ws.hour, ws.minute + 30),
|
||||
dashboard_ids=dashboard_ids,
|
||||
current_date=today,
|
||||
)
|
||||
|
||||
for idx, dash_id in enumerate(dashboard_ids):
|
||||
sched_time = scheduled_times[idx] if idx < len(scheduled_times) else scheduled_times[-1]
|
||||
params: dict = {
|
||||
"dashboard_id": dash_id,
|
||||
"environment_id": policy.environment_id,
|
||||
"provider_id": policy.provider_id or "",
|
||||
"policy_id": policy_id,
|
||||
}
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.task_manager.create_task(
|
||||
plugin_id="llm_dashboard_validation", params=params
|
||||
),
|
||||
self.loop,
|
||||
)
|
||||
logger.reason(
|
||||
f"Scheduled validation for dashboard {dash_id} at {sched_time.isoformat()}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.explore(
|
||||
f"Error triggering validation for policy {policy_id}",
|
||||
extra={"error": str(e)},
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
# #endregion _trigger_validation
|
||||
# #endregion SchedulerService
|
||||
# #region ThrottledSchedulerConfigurator [C:5] [TYPE Class] [SEMANTICS scheduler, throttling, distribution]
|
||||
# @BRIEF Distributes validation tasks evenly within an execution window.
|
||||
|
||||
@@ -25,6 +25,7 @@ from ._charts import SupersetChartsMixin
|
||||
from ._dashboards_crud import SupersetDashboardsCrudMixin
|
||||
from ._dashboards_filters import SupersetDashboardsFiltersMixin
|
||||
from ._dashboards_list import SupersetDashboardsListMixin
|
||||
from ._dashboards_write import SupersetDashboardsWriteMixin
|
||||
from ._databases import SupersetDatabasesMixin
|
||||
from ._datasets import SupersetDatasetsMixin
|
||||
from ._datasets_preview import SupersetDatasetsPreviewMixin
|
||||
@@ -47,6 +48,7 @@ from ._user_projection import SupersetUserProjectionMixin
|
||||
# @RELATION INHERITS -> [SupersetDatasetsPreviewMixin]
|
||||
# @RELATION INHERITS -> [SupersetDatasetsPreviewFiltersMixin]
|
||||
# @RELATION INHERITS -> [SupersetDatabasesMixin]
|
||||
# @RELATION INHERITS -> [SupersetDashboardsWriteMixin]
|
||||
class SupersetClient(
|
||||
SupersetDatabasesMixin,
|
||||
SupersetDatasetsPreviewFiltersMixin,
|
||||
@@ -56,6 +58,7 @@ class SupersetClient(
|
||||
SupersetDashboardsCrudMixin,
|
||||
SupersetDashboardsFiltersMixin,
|
||||
SupersetDashboardsListMixin,
|
||||
SupersetDashboardsWriteMixin,
|
||||
SupersetUserProjectionMixin,
|
||||
SupersetClientBase,
|
||||
):
|
||||
|
||||
291
backend/src/core/superset_client/_dashboards_write.py
Normal file
291
backend/src/core/superset_client/_dashboards_write.py
Normal file
@@ -0,0 +1,291 @@
|
||||
# #region SupersetDashboardsWriteMixin [C:4] [TYPE Module] [SEMANTICS superset, dashboard, chart, write, maintenance, banner]
|
||||
# @BRIEF Dashboard write mixin for SupersetClient — markdown chart CRUD and layout manipulation for maintenance banners.
|
||||
# @LAYER Infra
|
||||
# @RELATION DEPENDS_ON -> [SupersetClientBase]
|
||||
# @RELATION DEPENDS_ON -> [SupersetChartsMixin]
|
||||
# @RELATION DEPENDS_ON -> [LayoutUtils]
|
||||
# @SIDE_EFFECT Creates, updates, deletes markdown charts & modifies dashboard layout via Superset API.
|
||||
# @DATA_CONTRACT Input: dashboard_id / chart_id / markdown_text → Output: API response dict or chart_id int
|
||||
# @INVARIANT All network operations use self.network.request()
|
||||
# @RATIONALE Extracted from monolithic superset_client to satisfy INV_7.
|
||||
# Uses belief_scope with REASON/REFLECT/EXPLORE markers for all C4 operations per Std.Semantics.Python.
|
||||
|
||||
import json
|
||||
from typing import Any, cast
|
||||
|
||||
from ..logger import belief_scope, logger as app_logger
|
||||
from ._layout_utils import (
|
||||
insert_banner_markdown_at_top,
|
||||
parse_position_json,
|
||||
remove_banner_from_position,
|
||||
update_banner_markdown_content,
|
||||
)
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
|
||||
|
||||
# #region SupersetDashboardsWriteMixin [C:4] [TYPE Class]
|
||||
# @BRIEF Mixin providing markdown chart CRUD and dashboard layout manipulation for maintenance banners.
|
||||
# @RELATION DEPENDS_ON -> [SupersetClientBase]
|
||||
# @RELATION DEPENDS_ON -> [SupersetChartsMixin]
|
||||
class SupersetDashboardsWriteMixin:
|
||||
"""Mixin for creating/updating/deleting markdown charts and manipulating dashboard layouts."""
|
||||
|
||||
# #region create_markdown_chart [C:3] [TYPE Function]
|
||||
# @BRIEF Create a markdown chart and return the new chart ID.
|
||||
# @PRE dashboard_id exists in Superset. markdown_text not empty.
|
||||
# @POST Returns chart_id (int). Chart is NOT yet placed in dashboard layout.
|
||||
# @SIDE_EFFECT Creates a new chart resource in Superset.
|
||||
# @RELATION CALLS -> [self.network.request]
|
||||
def create_markdown_chart(self, dashboard_id: int, markdown_text: str) -> int:
|
||||
with belief_scope(
|
||||
"SupersetDashboardsWriteMixin.create_markdown_chart",
|
||||
f"dashboard_id={dashboard_id}",
|
||||
):
|
||||
app_logger.reason(
|
||||
"Creating markdown chart",
|
||||
extra={"dashboard_id": dashboard_id, "len": len(markdown_text)},
|
||||
)
|
||||
datasource_id = self._resolve_markdown_datasource(dashboard_id)
|
||||
payload = {
|
||||
"dashboards": [dashboard_id],
|
||||
"datasource_id": datasource_id,
|
||||
"datasource_type": "table",
|
||||
"slice_name": "Maintenance Banner",
|
||||
"viz_type": "markdown",
|
||||
"params": json.dumps({
|
||||
"markdown": markdown_text,
|
||||
"viz_type": "markdown",
|
||||
}),
|
||||
}
|
||||
try:
|
||||
response = cast(dict, self.network.request(
|
||||
method="POST", endpoint="/chart/", json=payload
|
||||
))
|
||||
chart_id = int(response.get("id") or response.get("result", {}).get("id", 0))
|
||||
app_logger.reflect("Chart created", extra={"chart_id": chart_id})
|
||||
return chart_id
|
||||
except Exception as e:
|
||||
app_logger.explore("Create chart failed", extra={"dashboard_id": dashboard_id}, error=str(e))
|
||||
raise
|
||||
# #endregion create_markdown_chart
|
||||
|
||||
# #region update_markdown_chart [C:3] [TYPE Function]
|
||||
# @BRIEF Update the markdown text of an existing markdown chart.
|
||||
# @PRE chart_id exists and is a markdown chart.
|
||||
# @POST Chart markdown content updated.
|
||||
# @SIDE_EFFECT Modifies a chart resource in Superset.
|
||||
def update_markdown_chart(self, chart_id: int, markdown_text: str) -> dict:
|
||||
with belief_scope(
|
||||
"SupersetDashboardsWriteMixin.update_markdown_chart",
|
||||
f"chart_id={chart_id}",
|
||||
):
|
||||
app_logger.reason("Updating markdown chart", extra={"chart_id": chart_id, "len": len(markdown_text)})
|
||||
payload = {"params": json.dumps({"markdown": markdown_text, "viz_type": "markdown"})}
|
||||
try:
|
||||
response = cast(dict, self.network.request(method="PUT", endpoint=f"/chart/{chart_id}", json=payload))
|
||||
app_logger.reflect("Chart updated", extra={"chart_id": chart_id})
|
||||
return response
|
||||
except Exception as e:
|
||||
app_logger.explore("Update chart failed", extra={"chart_id": chart_id}, error=str(e))
|
||||
raise
|
||||
# #endregion update_markdown_chart
|
||||
|
||||
# #region update_dashboard_layout [C:4] [TYPE Function]
|
||||
# @BRIEF Insert a native MARKDOWN element at (0,0) with full width (12 cols),
|
||||
# shift existing charts down. Uses native MARKDOWN for reliable rendering.
|
||||
# @PRE dashboard_id exists in Superset. chart_id is used for tracking/DB.
|
||||
# @POST MARKDOWN element placed at top; existing items shifted down.
|
||||
# @SIDE_EFFECT Modifies dashboard layout JSON.
|
||||
def update_dashboard_layout(self, dashboard_id: int, chart_id: int) -> dict:
|
||||
with belief_scope(
|
||||
"SupersetDashboardsWriteMixin.update_dashboard_layout",
|
||||
f"dashboard_id={dashboard_id}, chart_id={chart_id}",
|
||||
):
|
||||
app_logger.reason(
|
||||
"Updating layout with native MARKDOWN",
|
||||
extra={"dashboard_id": dashboard_id, "chart_id": chart_id},
|
||||
)
|
||||
try:
|
||||
dashboard = cast(dict, self.network.request(
|
||||
method="GET", endpoint=f"/dashboard/{dashboard_id}"
|
||||
))
|
||||
result = dashboard.get("result", dashboard)
|
||||
position_json = parse_position_json(result.get("position_json", "{}"))
|
||||
|
||||
# Use native MARKDOWN element inside a ROW, connected to GRID_ID
|
||||
placeholder = "*Maintenance banner*"
|
||||
insert_banner_markdown_at_top(position_json, chart_id, placeholder)
|
||||
|
||||
response = cast(dict, self.network.request(
|
||||
method="PUT", endpoint=f"/dashboard/{dashboard_id}",
|
||||
json={"position_json": json.dumps(position_json)},
|
||||
))
|
||||
app_logger.reflect(
|
||||
"Layout updated with native MARKDOWN",
|
||||
extra={"dashboard_id": dashboard_id, "chart_id": chart_id},
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Layout update failed",
|
||||
extra={"dashboard_id": dashboard_id},
|
||||
error=str(e),
|
||||
)
|
||||
raise
|
||||
# #endregion update_dashboard_layout
|
||||
|
||||
# #region remove_chart_from_layout [C:3] [TYPE Function]
|
||||
# @BRIEF Remove banner markdown element from dashboard layout without deleting the chart.
|
||||
# @PRE dashboard_id exists. chart_id exists in layout as MARKDOWN-banner-{chart_id}.
|
||||
# @POST MARKDOWN element removed from position_json; items shifted up.
|
||||
# @SIDE_EFFECT Modifies dashboard layout JSON.
|
||||
def remove_chart_from_layout(self, dashboard_id: int, chart_id: int) -> dict:
|
||||
with belief_scope(
|
||||
"SupersetDashboardsWriteMixin.remove_chart_from_layout",
|
||||
f"dashboard_id={dashboard_id}, chart_id={chart_id}",
|
||||
):
|
||||
app_logger.reason(
|
||||
"Removing banner from layout",
|
||||
extra={"dashboard_id": dashboard_id, "chart_id": chart_id},
|
||||
)
|
||||
try:
|
||||
dashboard = cast(dict, self.network.request(
|
||||
method="GET", endpoint=f"/dashboard/{dashboard_id}"
|
||||
))
|
||||
result = dashboard.get("result", dashboard)
|
||||
position_json = parse_position_json(result.get("position_json", "{}"))
|
||||
# Remove new MARKDOWN+ROW pair (old CHART-{id} format no longer created)
|
||||
remove_banner_from_position(position_json, chart_id)
|
||||
|
||||
response = cast(dict, self.network.request(
|
||||
method="PUT", endpoint=f"/dashboard/{dashboard_id}",
|
||||
json={"position_json": json.dumps(position_json)},
|
||||
))
|
||||
app_logger.reflect(
|
||||
"Banner removed from layout",
|
||||
extra={"dashboard_id": dashboard_id},
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Remove from layout failed",
|
||||
extra={"dashboard_id": dashboard_id},
|
||||
error=str(e),
|
||||
)
|
||||
raise
|
||||
# #endregion remove_chart_from_layout
|
||||
|
||||
# #region delete_chart [C:3] [TYPE Function]
|
||||
# @BRIEF Delete a chart from Superset by ID.
|
||||
# @PRE chart_id exists.
|
||||
# @POST Chart permanently deleted from Superset.
|
||||
# @SIDE_EFFECT Destroys chart resource.
|
||||
def delete_chart(self, chart_id: int) -> dict:
|
||||
with belief_scope("SupersetDashboardsWriteMixin.delete_chart", f"chart_id={chart_id}"):
|
||||
app_logger.reason("Deleting chart", extra={"chart_id": chart_id})
|
||||
try:
|
||||
response = cast(dict, self.network.request(method="DELETE", endpoint=f"/chart/{chart_id}"))
|
||||
app_logger.reflect("Chart deleted", extra={"chart_id": chart_id})
|
||||
return response
|
||||
except Exception as e:
|
||||
app_logger.explore("Delete chart failed", extra={"chart_id": chart_id}, error=str(e))
|
||||
raise
|
||||
# #endregion delete_chart
|
||||
|
||||
# #region update_banner_on_dashboard [C:3] [TYPE Function]
|
||||
# @BRIEF Update the content of a native MARKDOWN banner element on a dashboard.
|
||||
# @PRE dashboard_id exists. chart_id used for key lookup.
|
||||
# @POST MARKDOWN element content updated on dashboard.
|
||||
# @SIDE_EFFECT Modifies dashboard layout JSON in Superset.
|
||||
def update_banner_on_dashboard(
|
||||
self, dashboard_id: int, chart_id: int, content: str
|
||||
) -> dict:
|
||||
with belief_scope(
|
||||
"SupersetDashboardsWriteMixin.update_banner_on_dashboard",
|
||||
f"dashboard_id={dashboard_id}, chart_id={chart_id}",
|
||||
):
|
||||
app_logger.reason(
|
||||
"Updating banner content on dashboard",
|
||||
extra={"dashboard_id": dashboard_id, "chart_id": chart_id, "len": len(content)},
|
||||
)
|
||||
try:
|
||||
dashboard = cast(dict, self.network.request(
|
||||
method="GET", endpoint=f"/dashboard/{dashboard_id}"
|
||||
))
|
||||
result = dashboard.get("result", dashboard)
|
||||
position_json = parse_position_json(result.get("position_json", "{}"))
|
||||
markdown_key = f"MARKDOWN-banner-{chart_id}"
|
||||
update_banner_markdown_content(position_json, markdown_key, content)
|
||||
response = cast(dict, self.network.request(
|
||||
method="PUT", endpoint=f"/dashboard/{dashboard_id}",
|
||||
json={"position_json": json.dumps(position_json)},
|
||||
))
|
||||
app_logger.reflect(
|
||||
"Banner content updated on dashboard",
|
||||
extra={"dashboard_id": dashboard_id, "chart_id": chart_id},
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Update banner content failed",
|
||||
extra={"dashboard_id": dashboard_id},
|
||||
error=str(e),
|
||||
)
|
||||
raise
|
||||
# #endregion update_banner_on_dashboard
|
||||
|
||||
|
||||
# #region get_dashboard_layout [C:3] [TYPE Function]
|
||||
# @BRIEF Fetch the position_json layout of a dashboard.
|
||||
# @PRE dashboard_id exists.
|
||||
# @POST Returns the dashboard layout dict.
|
||||
def get_dashboard_layout(self, dashboard_id: int) -> dict:
|
||||
with belief_scope("SupersetDashboardsWriteMixin.get_dashboard_layout", f"dashboard_id={dashboard_id}"):
|
||||
app_logger.reason("Fetching layout", extra={"dashboard_id": dashboard_id})
|
||||
try:
|
||||
dashboard = cast(dict, self.network.request(method="GET", endpoint=f"/dashboard/{dashboard_id}"))
|
||||
result = dashboard.get("result", dashboard)
|
||||
layout = parse_position_json(result.get("position_json", "{}"))
|
||||
app_logger.reflect("Layout fetched", extra={"dashboard_id": dashboard_id, "items": len(layout)})
|
||||
return layout
|
||||
except Exception as e:
|
||||
app_logger.explore("Fetch layout failed", extra={"dashboard_id": dashboard_id}, error=str(e))
|
||||
raise
|
||||
# #endregion get_dashboard_layout
|
||||
|
||||
# #region _resolve_markdown_datasource [C:2] [TYPE Function]
|
||||
# @BRIEF Find a valid datasource_id for markdown chart creation.
|
||||
# @PRE dashboard_id exists.
|
||||
# @POST Returns an integer datasource_id.
|
||||
def _resolve_markdown_datasource(self, dashboard_id: int) -> int:
|
||||
with belief_scope("_resolve_markdown_datasource", f"dashboard_id={dashboard_id}"):
|
||||
try:
|
||||
datasets_response = cast(dict, self.network.request(
|
||||
method="GET", endpoint=f"/dashboard/{dashboard_id}/datasets"
|
||||
))
|
||||
datasets = datasets_response.get("result", [])
|
||||
if datasets:
|
||||
ds_id = datasets[0].get("id")
|
||||
if ds_id:
|
||||
return int(ds_id)
|
||||
except Exception:
|
||||
app_logger.explore("Dashboard datasets fetch failed, falling back",
|
||||
extra={"dashboard_id": dashboard_id}, error="Dashboard datasets endpoint failed")
|
||||
try:
|
||||
all_ds = cast(dict, self.network.request(
|
||||
method="GET", endpoint="/dataset/",
|
||||
params={"q": json.dumps({"columns": ["id"], "page_size": 1, "page": 0})},
|
||||
))
|
||||
ds_list = all_ds.get("result", [])
|
||||
if ds_list:
|
||||
ds_id = ds_list[0].get("id")
|
||||
if ds_id:
|
||||
return int(ds_id)
|
||||
except Exception:
|
||||
app_logger.explore("No datasets available", extra={}, error="Global dataset list failed")
|
||||
raise RuntimeError("Cannot create markdown chart: no datasets available in Superset.")
|
||||
# #endregion _resolve_markdown_datasource
|
||||
|
||||
# #endregion SupersetDashboardsWriteMixin
|
||||
# #endregion SupersetDashboardsWriteMixin
|
||||
@@ -146,7 +146,13 @@ class SupersetDatasetsMixin:
|
||||
dashboards_data = related_objects["result"].get("dashboards", [])
|
||||
else:
|
||||
dashboards_data = []
|
||||
for dash in dashboards_data:
|
||||
# related_objects returns {"count": N, "result": [...]} — extract the array
|
||||
dashboards_list = (
|
||||
dashboards_data.get("result", [])
|
||||
if isinstance(dashboards_data, dict)
|
||||
else dashboards_data
|
||||
)
|
||||
for dash in dashboards_list:
|
||||
if isinstance(dash, dict):
|
||||
dash_id = dash.get("id")
|
||||
if dash_id is None:
|
||||
|
||||
197
backend/src/core/superset_client/_layout_utils.py
Normal file
197
backend/src/core/superset_client/_layout_utils.py
Normal file
@@ -0,0 +1,197 @@
|
||||
# #region LayoutUtils [C:2] [TYPE Module] [SEMANTICS superset, dashboard, layout, position, json]
|
||||
# @BRIEF Utility functions for manipulating Superset dashboard position_json layout.
|
||||
# @LAYER Infra
|
||||
# @RELATION DEPENDS_ON -> [json]
|
||||
# @RELATION DEPENDS_ON -> [SupersetDashboardsWriteMixin]
|
||||
# @RATIONALE Extracted from SupersetDashboardsWriteMixin to stay under INV_7 400 LOC.
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
|
||||
# #region parse_position_json [C:1] [TYPE Function]
|
||||
# @BRIEF Parse position_json from a dashboard API response (may be string or dict).
|
||||
# @PRE raw_position is a string, dict, or None.
|
||||
# @POST Returns a dict.
|
||||
def parse_position_json(raw_position: Any) -> dict:
|
||||
if isinstance(raw_position, str):
|
||||
return json.loads(raw_position) if raw_position.strip() else {}
|
||||
if isinstance(raw_position, dict):
|
||||
return raw_position
|
||||
return {}
|
||||
# #endregion parse_position_json
|
||||
|
||||
|
||||
# #region _estimate_markdown_height [C:1] [TYPE Function]
|
||||
# @BRIEF Estimate grid height for a MARKDOWN element based on HTML content.
|
||||
# Counts block-level tags and newlines to estimate visual line count,
|
||||
# then converts to Superset grid units (1 unit ≈ 2 text lines + padding).
|
||||
# @PRE content is a string of HTML/Markdown.
|
||||
# @POST Returns int between 19 and 200 inclusive.
|
||||
def _estimate_markdown_height(content: str) -> int:
|
||||
"""Estimate MARKDOWN element height (position JSON units).
|
||||
|
||||
Model: total = CSS min-height(40px) + content pixel height.
|
||||
Content height = text_lines × line-height(22px) + container padding.
|
||||
1 position JSON unit ≈ 8 rendered pixels.
|
||||
|
||||
Reference: user's banner (5 text lines, padding:12) → height=21 → ~168px.
|
||||
"""
|
||||
if not content:
|
||||
return 19
|
||||
|
||||
# Detect container padding (e.g. padding:12 → 24px vertical)
|
||||
pad_match = re.search(r'padding\s*:\s*(\d+)', content)
|
||||
container_padding = int(pad_match.group(1)) * 2 if pad_match else 0
|
||||
|
||||
# Strip HTML to get plain text
|
||||
plain_text = re.sub(r'<[^>]+>', '', content)
|
||||
plain_text = re.sub(r'\s+', ' ', plain_text).strip()
|
||||
|
||||
if not plain_text:
|
||||
return 19
|
||||
|
||||
# Text lines (wrapping at ~40 chars/line at 12col width)
|
||||
text_lines = len(plain_text) // 40 + 1
|
||||
|
||||
# Content height (px): line-height(~22px) × lines + container padding
|
||||
content_px = text_lines * 22 + container_padding
|
||||
|
||||
# Total: CSS min-height(40px) + content
|
||||
total_px = 40 + content_px
|
||||
|
||||
# Convert to position JSON units (÷8)
|
||||
height = max(19, total_px // 8)
|
||||
return min(200, height)
|
||||
# #endregion _estimate_markdown_height
|
||||
|
||||
|
||||
# #region _generate_banner_id [C:1] [TYPE Function]
|
||||
# @BRIEF Generate a deterministic row and markdown key for a banner chart.
|
||||
def _generate_banner_id(chart_id: int) -> tuple[str, str]:
|
||||
row_key = f"ROW-banner-{chart_id}"
|
||||
md_key = f"MARKDOWN-banner-{chart_id}"
|
||||
return row_key, md_key
|
||||
# #endregion _generate_banner_id
|
||||
|
||||
|
||||
# #region insert_banner_markdown_at_top [C:2] [TYPE Function]
|
||||
# @BRIEF Insert a ROW + native MARKDOWN pair at the top of the dashboard grid.
|
||||
# Creates a ROW entry connected to GRID_ID, and a MARKDOWN child inside it.
|
||||
# Shifts all existing grid ROWs down by 2 rows.
|
||||
# @PRE position_json is a mutable dict. content is the HTML/markdown string.
|
||||
# chart_id is the database chart_id for key generation.
|
||||
# @POST position_json is mutated; ROW + MARKDOWN inserted at top of GRID.
|
||||
def insert_banner_markdown_at_top(
|
||||
position_json: dict, chart_id: int, content: str
|
||||
) -> dict:
|
||||
row_key, md_key = _generate_banner_id(chart_id)
|
||||
|
||||
# Shift existing ROW y-positions down
|
||||
for key, value in list(position_json.items()):
|
||||
if isinstance(value, dict):
|
||||
meta = value.get("meta", {})
|
||||
if isinstance(meta, dict):
|
||||
cur_y = meta.get("y", 0)
|
||||
if isinstance(cur_y, (int, float)):
|
||||
meta["y"] = cur_y + 2
|
||||
|
||||
# Add MARKDOWN entry with adaptive height
|
||||
height = _estimate_markdown_height(content)
|
||||
position_json[md_key] = {
|
||||
"type": "MARKDOWN",
|
||||
"id": md_key,
|
||||
"children": [],
|
||||
"parents": ["ROOT_ID", "GRID_ID", row_key],
|
||||
"meta": {
|
||||
"code": content,
|
||||
"width": 12,
|
||||
"height": height,
|
||||
"y": 0,
|
||||
},
|
||||
}
|
||||
|
||||
# Add ROW entry connected to GRID_ID
|
||||
position_json[row_key] = {
|
||||
"type": "ROW",
|
||||
"id": row_key,
|
||||
"children": [md_key],
|
||||
"parents": ["ROOT_ID", "GRID_ID"],
|
||||
"meta": {
|
||||
"background": "BACKGROUND_TRANSPARENT",
|
||||
"y": 0,
|
||||
},
|
||||
}
|
||||
|
||||
# Insert ROW at beginning of GRID_ID children
|
||||
grid = position_json.get("GRID_ID")
|
||||
if grid and isinstance(grid, dict):
|
||||
grid_children = grid.get("children", [])
|
||||
if isinstance(grid_children, list):
|
||||
if row_key in grid_children:
|
||||
grid_children.remove(row_key)
|
||||
grid_children.insert(0, row_key)
|
||||
|
||||
return position_json
|
||||
# #endregion insert_banner_markdown_at_top
|
||||
|
||||
|
||||
# #region update_banner_markdown_content [C:1] [TYPE Function]
|
||||
# @BRIEF Update the code content and adaptive height of an existing banner markdown element.
|
||||
# @PRE position_json has markdown_key. content is the new HTML/markdown string.
|
||||
# @POST position_json is mutated; markdown content and height updated.
|
||||
def update_banner_markdown_content(
|
||||
position_json: dict, markdown_key: str, content: str
|
||||
) -> dict:
|
||||
if markdown_key in position_json:
|
||||
meta = position_json[markdown_key].get("meta", {})
|
||||
if isinstance(meta, dict):
|
||||
meta["code"] = content
|
||||
meta["height"] = _estimate_markdown_height(content)
|
||||
return position_json
|
||||
# #endregion update_banner_markdown_content
|
||||
|
||||
|
||||
# #region remove_banner_from_position [C:2] [TYPE Function]
|
||||
# @BRIEF Remove the banner ROW + MARKDOWN pair from the position dict and GRID children.
|
||||
# Shifts remaining items up by 2 if the banner was at y=0.
|
||||
# @PRE position_json is a mutable dict. chart_id identifies the banner.
|
||||
# @POST position_json is mutated; ROW and MARKDOWN entries removed.
|
||||
def remove_banner_from_position(position_json: dict, chart_id: int) -> dict:
|
||||
row_key, md_key = _generate_banner_id(chart_id)
|
||||
|
||||
removed_y = None
|
||||
# Remove MARKDOWN entry
|
||||
if md_key in position_json:
|
||||
markdown_data = position_json[md_key]
|
||||
if isinstance(markdown_data, dict):
|
||||
meta = markdown_data.get("meta", {})
|
||||
if isinstance(meta, dict):
|
||||
removed_y = meta.get("y", 0)
|
||||
del position_json[md_key]
|
||||
|
||||
# Remove ROW entry
|
||||
if row_key in position_json:
|
||||
del position_json[row_key]
|
||||
|
||||
# Remove ROW from GRID_ID children
|
||||
grid = position_json.get("GRID_ID")
|
||||
if grid and isinstance(grid, dict):
|
||||
grid_children = grid.get("children", [])
|
||||
if isinstance(grid_children, list) and row_key in grid_children:
|
||||
grid_children.remove(row_key)
|
||||
|
||||
# Shift remaining items up
|
||||
if removed_y is not None and removed_y == 0:
|
||||
for key, value in position_json.items():
|
||||
if isinstance(value, dict):
|
||||
meta = value.get("meta", {})
|
||||
if isinstance(meta, dict):
|
||||
cur_y = meta.get("y", 0)
|
||||
if isinstance(cur_y, (int, float)) and cur_y > 0:
|
||||
meta["y"] = cur_y - 2
|
||||
return position_json
|
||||
# #endregion remove_banner_from_position
|
||||
|
||||
# #endregion LayoutUtils
|
||||
91
backend/src/core/timezone.py
Normal file
91
backend/src/core/timezone.py
Normal file
@@ -0,0 +1,91 @@
|
||||
# #region AppTimezone [C:3] [TYPE Module] [SEMANTICS core,timezone,utilities,datetime]
|
||||
# @BRIEF Application-level timezone utilities. Reads APP_TIMEZONE from env (default Europe/Moscow)
|
||||
# and provides helpers for converting UTC datetimes to the configured timezone.
|
||||
# @RELATION CALLED_BY -> [ConfigManager]
|
||||
# @RELATION DEPENDS_ON -> [GlobalSettings.app_timezone]
|
||||
# @RATIONALE Centralised timezone resolution avoids scattering ZoneInfo() calls across the codebase.
|
||||
# All API responses should emit localised timestamps; internal storage remains UTC.
|
||||
# @REJECTED Storing non-UTC in the database rejected — UTC is the canonical best practice for
|
||||
# persistence; timezone conversion is a presentation-layer concern.
|
||||
from datetime import datetime
|
||||
import os
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
|
||||
# #region _get_default_tz_name [C:1] [TYPE Function]
|
||||
# @BRIEF Read APP_TIMEZONE from env, fall back to Europe/Moscow.
|
||||
# @PRE: Environment is loaded (dotenv or os.environ).
|
||||
# @POST: Returns a valid IANA timezone string.
|
||||
def _get_default_tz_name() -> str:
|
||||
return os.getenv("APP_TIMEZONE", "Europe/Moscow")
|
||||
# #endregion _get_default_tz_name
|
||||
|
||||
|
||||
# #region get_app_timezone [C:1] [TYPE Function]
|
||||
# @BRIEF Return cached ZoneInfo for the configured application timezone.
|
||||
# @SIDE_EFFECT: Reads os.environ on first call; result is cached.
|
||||
# @POST: Returns a ZoneInfo instance matching APP_TIMEZONE env var.
|
||||
_APP_TZ_CACHE: ZoneInfo | None = None
|
||||
|
||||
def get_app_timezone() -> ZoneInfo:
|
||||
global _APP_TZ_CACHE
|
||||
if _APP_TZ_CACHE is None:
|
||||
_APP_TZ_CACHE = ZoneInfo(_get_default_tz_name())
|
||||
return _APP_TZ_CACHE
|
||||
# #endregion get_app_timezone
|
||||
|
||||
|
||||
# #region invalidate_timezone_cache [C:1] [TYPE Function]
|
||||
# @BRIEF Reset the cached ZoneInfo so the next get_app_timezone() call re-reads from env/DB.
|
||||
# @POST: _APP_TZ_CACHE is set to None; subsequent get_app_timezone() will re-resolve.
|
||||
# @RATIONALE: Called after PATCH /settings/consolidated updates app_timezone.
|
||||
def invalidate_timezone_cache() -> None:
|
||||
global _APP_TZ_CACHE
|
||||
_APP_TZ_CACHE = None
|
||||
# #endregion invalidate_timezone_cache
|
||||
|
||||
|
||||
# #region validate_timezone [C:1] [TYPE Function]
|
||||
# @BRIEF Validate that a timezone string is a known IANA timezone.
|
||||
# @PRE: tz_name is a string.
|
||||
# @POST: Returns True if ZoneInfo accepts the name, False otherwise.
|
||||
def validate_timezone(tz_name: str) -> bool:
|
||||
try:
|
||||
ZoneInfo(tz_name)
|
||||
return True
|
||||
except (KeyError, TypeError):
|
||||
return False
|
||||
# #endregion validate_timezone
|
||||
|
||||
|
||||
# #region localize [C:1] [TYPE Function]
|
||||
# @BRIEF Convert a timezone-aware or naive UTC datetime to the configured app timezone.
|
||||
# @PRE: If dt is timezone-naive, it is assumed to be UTC.
|
||||
# @POST: Returns a datetime with the app timezone attached (astimezone).
|
||||
def localize(dt: datetime | None) -> datetime | None:
|
||||
if dt is None:
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=ZoneInfo("UTC"))
|
||||
return dt.astimezone(get_app_timezone())
|
||||
# #endregion localize
|
||||
|
||||
|
||||
# #region now [C:1] [TYPE Function]
|
||||
# @BRIEF Get current time in the configured application timezone.
|
||||
# @POST: Returns timezone-aware datetime in the app timezone.
|
||||
def now() -> datetime:
|
||||
return datetime.now(get_app_timezone())
|
||||
# #endregion now
|
||||
|
||||
|
||||
# #region format_timezone_offset [C:1] [TYPE Function]
|
||||
# @BRIEF Return the UTC offset string for the configured timezone (e.g. "+03:00").
|
||||
# @POST: Returns string like "+03:00" or "+00:00".
|
||||
def format_timezone_offset() -> str:
|
||||
offset = now().strftime("%z")
|
||||
if offset:
|
||||
return f"{offset[:3]}:{offset[3:]}"
|
||||
return "+00:00"
|
||||
# #endregion format_timezone_offset
|
||||
# #endregion AppTimezone
|
||||
215
backend/src/models/maintenance.py
Normal file
215
backend/src/models/maintenance.py
Normal file
@@ -0,0 +1,215 @@
|
||||
# #region MaintenanceModels [C:2] [TYPE Module] [SEMANTICS sqlalchemy, maintenance, banner, event, settings, model]
|
||||
# @BRIEF SQLAlchemy models for the Maintenance Banner feature: event, banner, dashboard state, and settings.
|
||||
# @LAYER Domain
|
||||
# @RELATION DEPENDS_ON -> [sqlalchemy]
|
||||
# @RELATION DEPENDS_ON -> [MappingBase]
|
||||
# @INVARIANT MaintenanceDashboardBanner has unique partial index on (environment_id, dashboard_id) WHERE status='active'
|
||||
# @INVARIANT MaintenanceSettings is a singleton row (id='default') enforced by CheckConstraint
|
||||
|
||||
import enum
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import (
|
||||
CheckConstraint,
|
||||
Column,
|
||||
DateTime,
|
||||
Enum as SQLEnum,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
String,
|
||||
Text,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from .mapping import Base
|
||||
|
||||
|
||||
# #region MaintenanceEventStatus [C:1] [TYPE Class]
|
||||
class MaintenanceEventStatus(str, enum.Enum):
|
||||
PENDING = "pending"
|
||||
APPLYING = "applying"
|
||||
ACTIVE = "active"
|
||||
PARTIAL = "partial"
|
||||
ENDING = "ending"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
# #endregion MaintenanceEventStatus
|
||||
|
||||
|
||||
# #region MaintenanceDashboardBannerStatus [C:1] [TYPE Class]
|
||||
class MaintenanceDashboardBannerStatus(str, enum.Enum):
|
||||
ACTIVE = "active"
|
||||
REMOVED = "removed"
|
||||
# #endregion MaintenanceDashboardBannerStatus
|
||||
|
||||
|
||||
# #region MaintenanceDashboardStateStatus [C:1] [TYPE Class]
|
||||
class MaintenanceDashboardStateStatus(str, enum.Enum):
|
||||
PENDING_APPLY = "pending_apply"
|
||||
ACTIVE = "active"
|
||||
REMOVING = "removing"
|
||||
REMOVED = "removed"
|
||||
APPLY_FAILED = "apply_failed"
|
||||
REMOVAL_FAILED = "removal_failed"
|
||||
# #endregion MaintenanceDashboardStateStatus
|
||||
|
||||
|
||||
# #region DashboardScope [C:1] [TYPE Class]
|
||||
class DashboardScope(str, enum.Enum):
|
||||
PUBLISHED_ONLY = "published_only"
|
||||
DRAFT_ONLY = "draft_only"
|
||||
ALL = "all"
|
||||
# #endregion DashboardScope
|
||||
|
||||
|
||||
# #region MaintenanceEvent [C:1] [TYPE Class]
|
||||
# @BRIEF A record of a maintenance event with table list, time window, status, and linked dashboard states.
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceDashboardState]
|
||||
class MaintenanceEvent(Base):
|
||||
__tablename__ = "maintenance_events"
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()), index=True)
|
||||
task_id = Column(String, nullable=True, index=True)
|
||||
environment_id = Column(String, nullable=False)
|
||||
tables = Column(JSON, nullable=False)
|
||||
start_time = Column(DateTime(timezone=True), nullable=False)
|
||||
end_time = Column(DateTime(timezone=True), nullable=True)
|
||||
message = Column(Text, nullable=True)
|
||||
status = Column(
|
||||
SQLEnum(MaintenanceEventStatus),
|
||||
nullable=False,
|
||||
default=MaintenanceEventStatus.PENDING,
|
||||
index=True,
|
||||
)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(
|
||||
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
|
||||
dashboard_states = relationship(
|
||||
"MaintenanceDashboardState",
|
||||
back_populates="maintenance_event",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
# #endregion MaintenanceEvent
|
||||
|
||||
|
||||
# #region MaintenanceDashboardBanner [C:1] [TYPE Class]
|
||||
# @BRIEF The canonical "one chart per dashboard" entity. Unique partial index enforces single active banner per dashboard.
|
||||
# @INVARIANT Unique partial index (environment_id, dashboard_id) WHERE status='active'
|
||||
class MaintenanceDashboardBanner(Base):
|
||||
__tablename__ = "maintenance_dashboard_banners"
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()), index=True)
|
||||
environment_id = Column(String, nullable=False)
|
||||
dashboard_id = Column(Integer, nullable=False, index=True)
|
||||
chart_id = Column(Integer, nullable=True)
|
||||
banner_text = Column(Text, nullable=True)
|
||||
status = Column(
|
||||
SQLEnum(MaintenanceDashboardBannerStatus),
|
||||
nullable=False,
|
||||
default=MaintenanceDashboardBannerStatus.ACTIVE,
|
||||
)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(
|
||||
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
|
||||
dashboard_states = relationship(
|
||||
"MaintenanceDashboardState", back_populates="banner"
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ix_banner_unique_active",
|
||||
"environment_id",
|
||||
"dashboard_id",
|
||||
unique=True,
|
||||
postgresql_where=(status == "active"),
|
||||
),
|
||||
)
|
||||
# #endregion MaintenanceDashboardBanner
|
||||
|
||||
|
||||
# #region MaintenanceDashboardState [C:1] [TYPE Class]
|
||||
# @BRIEF Links a maintenance event to a dashboard via a banner. Tracks per-dashboard-per-event lifecycle.
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceEvent]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceDashboardBanner]
|
||||
class MaintenanceDashboardState(Base):
|
||||
__tablename__ = "maintenance_dashboard_states"
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()), index=True)
|
||||
event_id = Column(
|
||||
String, ForeignKey("maintenance_events.id"), nullable=False, index=True
|
||||
)
|
||||
dashboard_id = Column(Integer, nullable=False, index=True)
|
||||
banner_id = Column(
|
||||
String,
|
||||
ForeignKey("maintenance_dashboard_banners.id"),
|
||||
nullable=True,
|
||||
)
|
||||
status = Column(
|
||||
SQLEnum(MaintenanceDashboardStateStatus),
|
||||
nullable=False,
|
||||
default=MaintenanceDashboardStateStatus.PENDING_APPLY,
|
||||
)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(
|
||||
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
|
||||
maintenance_event = relationship(
|
||||
"MaintenanceEvent", back_populates="dashboard_states"
|
||||
)
|
||||
banner = relationship(
|
||||
"MaintenanceDashboardBanner", back_populates="dashboard_states"
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_mds_dashboard_status", "dashboard_id", "status"),
|
||||
)
|
||||
# #endregion MaintenanceDashboardState
|
||||
|
||||
|
||||
# #region MaintenanceSettings [C:1] [TYPE Class]
|
||||
# @BRIEF Single-row maintenance mode configuration. Enforces singleton via CheckConstraint(id='default').
|
||||
# @INVARIANT id must always be 'default' — enforced by CheckConstraint
|
||||
class MaintenanceSettings(Base):
|
||||
__tablename__ = "maintenance_settings"
|
||||
|
||||
id = Column(String, primary_key=True, default="default")
|
||||
target_environment_id = Column(String, nullable=False)
|
||||
display_timezone = Column(String, nullable=False, default="UTC")
|
||||
banner_template = Column(
|
||||
Text,
|
||||
nullable=False,
|
||||
default=(
|
||||
'<div style="background:#FFF3E0;padding:16px;border-left:4px solid #FF9800;border-radius:4px">\n\n'
|
||||
"## ⚠️ Технические работы\n\n"
|
||||
"{message}\n\n"
|
||||
"**Начало:** {start_time}\n"
|
||||
"**Конец:** {end_time}\n\n"
|
||||
"*Данные могут быть неполными или временно недоступны.*\n\n"
|
||||
"</div>"
|
||||
),
|
||||
)
|
||||
dashboard_scope = Column(
|
||||
SQLEnum(DashboardScope),
|
||||
nullable=False,
|
||||
default=DashboardScope.PUBLISHED_ONLY,
|
||||
)
|
||||
excluded_dashboard_ids = Column(JSON, nullable=False, default=list)
|
||||
forced_dashboard_ids = Column(JSON, nullable=False, default=list)
|
||||
updated_at = Column(
|
||||
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
CheckConstraint("id = 'default'", name="ck_settings_singleton"),
|
||||
)
|
||||
# #endregion MaintenanceSettings
|
||||
|
||||
# #endregion MaintenanceModels
|
||||
@@ -14,7 +14,7 @@ class FileCategory(str, Enum):
|
||||
# #region StorageConfig [C:1] [TYPE Class]
|
||||
# @BRIEF Configuration model for the storage system, defining paths and naming patterns.
|
||||
class StorageConfig(BaseModel):
|
||||
root_path: str = Field(default="backups", description="Absolute path to the storage root directory.")
|
||||
root_path: str = Field(default="/app/storage", description="Absolute path to the storage root directory.")
|
||||
backup_path: str = Field(default="backups", description="Subpath for backups.")
|
||||
repo_path: str = Field(default="repositories", description="Subpath for repositories.")
|
||||
backup_structure_pattern: str = Field(default="{category}/", description="Pattern for backup directory structure.")
|
||||
|
||||
@@ -35,32 +35,22 @@ class GitPlugin(PluginBase):
|
||||
|
||||
# region __init__ [TYPE Function]
|
||||
# @PURPOSE: Инициализирует плагин и его зависимости.
|
||||
# @PRE: config.json exists or shared config_manager is available.
|
||||
# @PRE: shared config_manager доступен через src.dependencies.
|
||||
# @POST: Инициализированы git_service и config_manager.
|
||||
def __init__(self):
|
||||
with belief_scope("GitPlugin.__init__"):
|
||||
app_logger.info("Initializing GitPlugin.")
|
||||
self.git_service = GitService()
|
||||
|
||||
# Robust config path resolution:
|
||||
# 1. Try absolute path from src/dependencies.py style if possible
|
||||
# 2. Try relative paths based on common execution patterns
|
||||
if os.path.exists("../config.json"):
|
||||
config_path = "../config.json"
|
||||
elif os.path.exists("config.json"):
|
||||
config_path = "config.json"
|
||||
else:
|
||||
# Fallback to the one initialized in dependencies if we can import it
|
||||
try:
|
||||
from src.dependencies import config_manager
|
||||
self.config_manager = config_manager
|
||||
app_logger.info("GitPlugin initialized using shared config_manager.")
|
||||
return
|
||||
except Exception:
|
||||
config_path = "config.json"
|
||||
|
||||
self.config_manager = ConfigManager(config_path)
|
||||
app_logger.info(f"GitPlugin initialized with {config_path}")
|
||||
# Используем shared config_manager из dependencies
|
||||
try:
|
||||
from src.dependencies import config_manager
|
||||
self.config_manager = config_manager
|
||||
app_logger.info("GitPlugin initialized using shared config_manager.")
|
||||
except Exception as exc:
|
||||
app_logger.error(f"GitPlugin: failed to get shared config_manager: {exc}")
|
||||
self.config_manager = ConfigManager()
|
||||
app_logger.info("GitPlugin initialized with fallback ConfigManager.")
|
||||
# endregion __init__
|
||||
|
||||
@property
|
||||
|
||||
@@ -13,6 +13,7 @@ import base64
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
from typing import Any
|
||||
@@ -347,16 +348,19 @@ class ScreenshotService:
|
||||
# endregion ScreenshotService._save_debug_screenshot
|
||||
|
||||
# region ScreenshotService.capture_dashboard [TYPE Function] [C:4]
|
||||
# @PURPOSE Captures a full-page screenshot of a dashboard using Playwright and CDP.
|
||||
# @PURPOSE Captures a full-page screenshot of a dashboard using Playwright and CDP, plus diagnostic viewport screenshots of tabs that had chart load errors.
|
||||
# @PRE dashboard_id is a valid string, output_path is a writable path.
|
||||
# @POST Returns True if screenshot is saved successfully. Debug temp dir cleaned up on success, preserved on failure.
|
||||
# @SIDE_EFFECT Launches a browser, performs UI login, switches tabs, writes a PNG file; creates and cleans up temp debug directory.
|
||||
# Extra diagnostic screenshots saved as output_path__tab_{tab_text}.png for each tab with chart errors.
|
||||
# @SIDE_EFFECT Launches a browser, performs UI login, switches tabs, writes PNG files; creates and cleans up temp debug directory.
|
||||
# @UX_STATE [Navigating] -> Loading dashboard UI
|
||||
# @UX_STATE [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading
|
||||
# @UX_STATE [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading.
|
||||
# Collects errored_tabs — those where chart content fails to render within timeout.
|
||||
# @UX_STATE [CalculatingHeight] -> Determining dashboard dimensions
|
||||
# @UX_STATE [Capturing] -> Executing CDP screenshot
|
||||
# @RATIONALE Uses tempfile.mkdtemp() for all debug screenshots; cleans up on success (shutil.rmtree), preserves on failure for diagnostic analysis. Pre-resize screenshot is saved to temp dir and auto-cleaned on success.
|
||||
# @REJECTED Old approach saved debug .png files alongside output — accumulated permanently (F1). In-memory-only logging rejected — visual state cannot be captured in text.
|
||||
# @UX_STATE [CapturingErroredTabs] -> Taking viewport screenshots of tabs with chart errors.
|
||||
# @RATIONALE Errored tab screenshots give LLM diagnostic context — an error text description ("charts.length = 0") is not enough to diagnose a chart load failure; a visual screenshot shows whether the tab is empty, has a loading spinner, or renders a Superset error card. Uses tempfile.mkdtemp() for all debug screenshots; cleans up on success (shutil.rmtree), preserves on failure for diagnostic analysis.
|
||||
# @REJECTED Always-screenshot-all-tabs rejected — ~20 tabs x 5s per screenshot = 100s overhead, most tabs healthy. Only errored + main gives diagnostic value at near-zero cost. Old approach saved debug .png files alongside output — accumulated permanently (F1). In-memory-only logging rejected — visual state cannot be captured in text.
|
||||
async def capture_dashboard(self, dashboard_id: str, output_path: str) -> bool:
|
||||
debug_dir = tempfile.mkdtemp(prefix="ss_screenshot_debug_")
|
||||
success = False
|
||||
@@ -587,12 +591,15 @@ class ScreenshotService:
|
||||
try:
|
||||
async with asyncio.timeout(SCREENSHOT_SERVICE_TIMEOUT_MS / 1000):
|
||||
# 1. Handle Tabs (Recursive switching)
|
||||
# @UX_STATE [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading
|
||||
# @UX_STATE [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading.
|
||||
# Collects errored_tabs — those where chart content fails to render within timeout.
|
||||
# These are later screenshotted individually for LLM diagnostic value.
|
||||
processed_tabs = set()
|
||||
errored_tabs = [] # Collects {tab_text, depth, error} for tabs with chart load failures
|
||||
|
||||
async def switch_tabs(depth=0):
|
||||
if depth > 3:
|
||||
return # Limit recursion depth
|
||||
return # Limit recursion depth
|
||||
|
||||
tab_selectors = [
|
||||
'.ant-tabs-nav-list .ant-tabs-tab',
|
||||
@@ -635,10 +642,22 @@ class ScreenshotService:
|
||||
}""", timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS)
|
||||
except Exception:
|
||||
logger.warning(f"[TabSwitching] Content verification timed out for tab: {tab_text}")
|
||||
errored_tabs.append({
|
||||
"tab_text": tab_text,
|
||||
"tab_index": i,
|
||||
"depth": depth,
|
||||
"error": "content_verification_timeout"
|
||||
})
|
||||
|
||||
await switch_tabs(depth + 1)
|
||||
except Exception as tab_e:
|
||||
logger.warning(f"[DEBUG][TabSwitching] Failed to process tab {i}: {tab_e}")
|
||||
errored_tabs.append({
|
||||
"tab_text": f"tab_{depth}_{i}",
|
||||
"tab_index": i,
|
||||
"depth": depth,
|
||||
"error": f"processing_exception: {tab_e!s}"
|
||||
})
|
||||
|
||||
try:
|
||||
first_tab = found_tabs[0]
|
||||
@@ -660,6 +679,13 @@ class ScreenshotService:
|
||||
|
||||
await switch_tabs()
|
||||
|
||||
# Log summary of errored tabs
|
||||
if errored_tabs:
|
||||
logger.info(f"[TabSwitching] {len(errored_tabs)} tab(s) with errors detected. "
|
||||
f"Errored tabs: {[t['tab_text'] for t in errored_tabs]}")
|
||||
else:
|
||||
logger.info("[TabSwitching] No tab errors detected during preload")
|
||||
|
||||
# 2. Calculate full height for screenshot
|
||||
# @UX_STATE [CalculatingHeight] -> Determining dashboard dimensions
|
||||
full_height = await page.evaluate("""() => {
|
||||
@@ -757,12 +783,65 @@ class ScreenshotService:
|
||||
logger.info(f"[DIAGNOSTIC] Final screenshot saved: {output_path}")
|
||||
logger.info(f"[DIAGNOSTIC] Final screenshot size: {final_size} bytes ({final_size / 1024:.2f} KB)")
|
||||
|
||||
# DIAGNOSTIC: Get image dimensions
|
||||
# DIAGNOSTIC: Get image dimensions
|
||||
try:
|
||||
with Image.open(output_path) as final_img:
|
||||
logger.info(f"[DIAGNOSTIC] Final screenshot dimensions: {final_img.width}x{final_img.height}")
|
||||
except Exception as img_err:
|
||||
logger.warning(f"[DIAGNOSTIC] Could not read final image dimensions: {img_err}")
|
||||
|
||||
# 3. Screenshot errored tabs (diagnostic viewport shots)
|
||||
# @UX_STATE [CapturingErroredTabs] -> Taking viewport screenshots of tabs with chart errors.
|
||||
# Only screenshots tabs that had errors during preload, plus always keeps the main tab.
|
||||
# Errored tab screenshots are viewport-only (no full-page resize) to minimize overhead.
|
||||
if errored_tabs:
|
||||
logger.info(f"[TabSwitching] Capturing {len(errored_tabs)} errored tab screenshot(s)...")
|
||||
screenshotted_tabs = set()
|
||||
for tab_info in errored_tabs:
|
||||
try:
|
||||
tab_text = tab_info["tab_text"]
|
||||
|
||||
# Dedup: skip if already screenshotted
|
||||
if tab_text in screenshotted_tabs:
|
||||
continue
|
||||
screenshotted_tabs.add(tab_text)
|
||||
|
||||
# Sanitize tab text for filename
|
||||
safe_tab = re.sub(r'[^\w\-_ ]', '', tab_text).strip().replace(' ', '_')[:40]
|
||||
if not safe_tab:
|
||||
safe_tab = f"tab_{tab_info.get('depth', 0)}_{tab_info.get('tab_index', 0)}"
|
||||
|
||||
# Build tab path robustly (handle any extension or no extension)
|
||||
base = output_path[:-4] if output_path.lower().endswith('.png') else output_path
|
||||
tab_path = f"{base}__tab_{safe_tab}.png"
|
||||
|
||||
# Switch to the errored tab — try exact text match first
|
||||
errored_tab_el = page.get_by_role("tab", name=tab_text, exact=True)
|
||||
if not await errored_tab_el.is_visible():
|
||||
# Fallback: try nth-child by index
|
||||
tab_index = tab_info.get("tab_index")
|
||||
if tab_index is not None:
|
||||
errored_tab_el = page.locator('.ant-tabs-tab').nth(tab_index)
|
||||
if await errored_tab_el.is_visible():
|
||||
await errored_tab_el.click()
|
||||
await asyncio.sleep(0.5)
|
||||
await self._wait_for_charts_stabilized(page, timeout_ms=PLAYWRIGHT_SHORT_TIMEOUT_MS)
|
||||
await page.screenshot(path=tab_path, timeout=PLAYWRIGHT_SHORT_TIMEOUT_MS)
|
||||
tab_size = os.path.getsize(tab_path) if os.path.exists(tab_path) else 0
|
||||
logger.info(f"[TabSwitching] Errored tab screenshot saved: {tab_path} ({tab_size} bytes)")
|
||||
else:
|
||||
logger.warning(f"[TabSwitching] Errored tab '{tab_text}' not visible, skipping screenshot")
|
||||
except Exception as tab_shot_err:
|
||||
logger.warning(f"[TabSwitching] Failed to screenshot errored tab '{tab_info.get('tab_text')}': {tab_shot_err}")
|
||||
|
||||
# Return to first tab after errored tab screenshots
|
||||
try:
|
||||
first_tab_el = page.locator('.ant-tabs-nav-list .ant-tabs-tab').first
|
||||
if await first_tab_el.is_visible():
|
||||
await first_tab_el.click()
|
||||
await asyncio.sleep(0.5)
|
||||
except Exception:
|
||||
pass
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(f"Screenshot capture timed out after {SCREENSHOT_SERVICE_TIMEOUT_MS}ms")
|
||||
try:
|
||||
|
||||
176
backend/src/plugins/maintenance_banner.py
Normal file
176
backend/src/plugins/maintenance_banner.py
Normal file
@@ -0,0 +1,176 @@
|
||||
# #region MaintenanceBannerPlugin [C:4] [TYPE Module] [SEMANTICS plugin, maintenance, banner, execution, task]
|
||||
# @BRIEF TaskManager plugin for executing maintenance banner operations (start, end, end-all).
|
||||
# Dispatches to MaintenanceService based on operation type in params.
|
||||
# @LAYER Plugin
|
||||
# @RELATION IMPLEMENTS -> [PluginBase]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceServiceModule]
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [TaskContext]
|
||||
# @INVARIANT TaskManager executes this plugin with params containing operation and event_id.
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ..core.database import SessionLocal
|
||||
from ..core.logger import belief_scope, logger as app_logger
|
||||
from ..core.plugin_base import PluginBase
|
||||
from ..core.superset_client import SupersetClient
|
||||
from ..core.task_manager.context import TaskContext
|
||||
from ..dependencies import get_config_manager
|
||||
from ..models.maintenance import MaintenanceSettings
|
||||
from ..services.maintenance import end_all_maintenance, end_maintenance, start_maintenance
|
||||
|
||||
|
||||
# #region MaintenanceBannerPlugin [C:4] [TYPE Class]
|
||||
# @BRIEF Plugin for executing maintenance banner operations asynchronously via TaskManager.
|
||||
# @RELATION CALLS -> [start_maintenance]
|
||||
# @RELATION CALLS -> [end_maintenance]
|
||||
# @RELATION CALLS -> [end_all_maintenance]
|
||||
class MaintenanceBannerPlugin(PluginBase):
|
||||
"""TaskManager plugin for maintenance banner operations."""
|
||||
|
||||
# #region id [TYPE Property]
|
||||
@property
|
||||
def id(self) -> str:
|
||||
return "maintenance_banner_apply"
|
||||
# #endregion id
|
||||
|
||||
# #region name [TYPE Property]
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "Maintenance Banner"
|
||||
# #endregion name
|
||||
|
||||
# #region description [TYPE Property]
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return "Applies or removes maintenance banners on Superset dashboards."
|
||||
# #endregion description
|
||||
|
||||
# #region version [TYPE Property]
|
||||
@property
|
||||
def version(self) -> str:
|
||||
return "1.0.0"
|
||||
# #endregion version
|
||||
|
||||
# #region ui_route [TYPE Property]
|
||||
@property
|
||||
def ui_route(self) -> str:
|
||||
return "" # No UI route — async-only
|
||||
# #endregion ui_route
|
||||
|
||||
# #region get_schema [TYPE Function]
|
||||
def get_schema(self) -> dict[str, Any]:
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"operation": {
|
||||
"type": "string",
|
||||
"enum": ["start", "end", "end_all"],
|
||||
"title": "Operation",
|
||||
},
|
||||
"event_id": {
|
||||
"type": "string",
|
||||
"title": "Maintenance Event ID",
|
||||
},
|
||||
"environment_id": {
|
||||
"type": "string",
|
||||
"title": "Target Environment ID",
|
||||
},
|
||||
},
|
||||
"required": ["operation"],
|
||||
}
|
||||
# #endregion get_schema
|
||||
|
||||
# #region execute [TYPE Function] [C:4]
|
||||
# @BRIEF Execute maintenance operation based on params.operation.
|
||||
# @PARAM params contains: operation (start|end|end_all), event_id, environment_id.
|
||||
# @PARAM context TaskContext for logging.
|
||||
# @PRE DB and Superset available.
|
||||
# @POST Operation completed. Results returned as dict.
|
||||
# @SIDE_EFFECT Modifies Superset dashboards, writes DB rows.
|
||||
async def execute(
|
||||
self,
|
||||
params: dict[str, Any],
|
||||
context: TaskContext | None = None,
|
||||
) -> dict[str, Any]:
|
||||
with belief_scope("MaintenanceBannerPlugin.execute"):
|
||||
operation = params.get("operation", "start")
|
||||
event_id = params.get("event_id", "")
|
||||
environment_id = params.get("environment_id", "")
|
||||
|
||||
app_logger.reason(
|
||||
"Executing maintenance operation",
|
||||
extra={
|
||||
"operation": operation,
|
||||
"event_id": event_id,
|
||||
"environment_id": environment_id,
|
||||
},
|
||||
)
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
# Resolve environment
|
||||
if not environment_id:
|
||||
settings = db.query(MaintenanceSettings).filter(
|
||||
MaintenanceSettings.id == "default"
|
||||
).first()
|
||||
if settings:
|
||||
environment_id = settings.target_environment_id
|
||||
|
||||
if not environment_id:
|
||||
config_manager = get_config_manager()
|
||||
envs = config_manager.get_environments()
|
||||
if envs:
|
||||
environment_id = envs[0].id
|
||||
|
||||
env = None
|
||||
config_manager = get_config_manager()
|
||||
for e in config_manager.get_environments():
|
||||
if e.id == environment_id:
|
||||
env = e
|
||||
break
|
||||
|
||||
if not env:
|
||||
app_logger.explore(
|
||||
"Environment not found",
|
||||
extra={"environment_id": environment_id},
|
||||
error="EnvironmentNotFound",
|
||||
)
|
||||
return {"status": "failed", "error": f"Environment {environment_id} not found"}
|
||||
|
||||
superset_client = SupersetClient(env)
|
||||
|
||||
if operation == "start":
|
||||
if not event_id:
|
||||
return {"status": "failed", "error": "event_id required for start"}
|
||||
result = start_maintenance(event_id, db, superset_client)
|
||||
elif operation == "end":
|
||||
if not event_id:
|
||||
return {"status": "failed", "error": "event_id required for end"}
|
||||
result = end_maintenance(event_id, db, superset_client)
|
||||
elif operation == "end_all":
|
||||
result = end_all_maintenance(db, superset_client)
|
||||
else:
|
||||
result = {"status": "failed", "error": f"Unknown operation: {operation}"}
|
||||
|
||||
db.commit()
|
||||
app_logger.reflect(
|
||||
"Maintenance operation completed",
|
||||
extra={"operation": operation, "result": result},
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
app_logger.explore(
|
||||
"Maintenance operation failed",
|
||||
extra={"operation": operation, "event_id": event_id},
|
||||
error=str(e),
|
||||
)
|
||||
return {"status": "failed", "error": str(e)}
|
||||
finally:
|
||||
db.close()
|
||||
# #endregion execute
|
||||
# #endregion MaintenanceBannerPlugin
|
||||
|
||||
# #endregion MaintenanceBannerPlugin
|
||||
@@ -148,20 +148,10 @@ def _resolve_target_envs(env_ids: list[str]) -> dict[str, Environment]:
|
||||
resolved: dict[str, Environment] = {}
|
||||
|
||||
if not configured:
|
||||
for config_path in [Path("config.json"), Path("backend/config.json")]:
|
||||
if not config_path.exists():
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
env_rows = payload.get("environments", [])
|
||||
for row in env_rows:
|
||||
env = Environment(**row)
|
||||
configured[env.id] = env
|
||||
except Exception as exc:
|
||||
logger.reflect(
|
||||
f"Failed loading environments from {config_path}: {exc}",
|
||||
extra={"src": "_resolve_target_envs"},
|
||||
)
|
||||
logger.reflect(
|
||||
"No environments found in DB — seed script requires environments configured via API first",
|
||||
extra={"src": "_resolve_target_envs"},
|
||||
)
|
||||
|
||||
for env_id in env_ids:
|
||||
env = configured.get(env_id)
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
# @RELATION DEPENDS_ON -> [TaskCleanupService]
|
||||
# @RELATION DEPENDS_ON -> [TaskManager]
|
||||
|
||||
from datetime import UTC
|
||||
import os
|
||||
import time
|
||||
from typing import Any, cast
|
||||
@@ -255,6 +256,9 @@ class HealthService:
|
||||
task_id = str(record.task_id) if record.task_id is not None else None
|
||||
summary = str(record.summary) if record.summary is not None else None
|
||||
timestamp = cast(Any, record.timestamp)
|
||||
# Ensure timestamp is timezone-aware (DB stores naive UTC)
|
||||
if timestamp is not None and timestamp.tzinfo is None:
|
||||
timestamp = timestamp.replace(tzinfo=UTC)
|
||||
|
||||
meta = self._resolve_dashboard_meta(dashboard_id, resolved_environment_id)
|
||||
items.append(
|
||||
|
||||
30
backend/src/services/maintenance/__init__.py
Normal file
30
backend/src/services/maintenance/__init__.py
Normal file
@@ -0,0 +1,30 @@
|
||||
# #region MaintenancePackage [C:1] [TYPE Package] [SEMANTICS maintenance, service, package]
|
||||
# @BRIEF Maintenance Banner service package. Re-exports all public orchestrators and helpers.
|
||||
# @LAYER Service
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceOrchestrators]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceDashboardScanner]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceBannerRenderer]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceChartManager]
|
||||
|
||||
from ._banner_renderer import build_banner_text, rebuild_banner
|
||||
from ._chart_manager import ensure_banner_chart
|
||||
from ._dashboard_scanner import find_affected_dashboards
|
||||
from ._orchestrators import (
|
||||
build_idempotency_key,
|
||||
end_all_maintenance,
|
||||
end_maintenance,
|
||||
start_maintenance,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"build_banner_text",
|
||||
"build_idempotency_key",
|
||||
"end_all_maintenance",
|
||||
"end_maintenance",
|
||||
"ensure_banner_chart",
|
||||
"find_affected_dashboards",
|
||||
"rebuild_banner",
|
||||
"start_maintenance",
|
||||
]
|
||||
|
||||
# #endregion MaintenancePackage
|
||||
254
backend/src/services/maintenance/_banner_renderer.py
Normal file
254
backend/src/services/maintenance/_banner_renderer.py
Normal file
@@ -0,0 +1,254 @@
|
||||
# #region MaintenanceBannerRenderer [C:3] [TYPE Module] [SEMANTICS maintenance, banner, renderer, text]
|
||||
# @BRIEF Banner text rendering and rebuild helpers. Build aggregated markdown from
|
||||
# active maintenance events, format with template, apply HTML escaping.
|
||||
# @LAYER Service
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceModels]
|
||||
# @RELATION DEPENDS_ON -> [SupersetDashboardsWriteMixin]
|
||||
|
||||
import html
|
||||
import re as _re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...core.logger import belief_scope, logger as app_logger
|
||||
from ...core.superset_client import SupersetClient
|
||||
from ...models.maintenance import (
|
||||
MaintenanceDashboardBanner,
|
||||
MaintenanceDashboardBannerStatus,
|
||||
MaintenanceDashboardState,
|
||||
MaintenanceDashboardStateStatus,
|
||||
MaintenanceEvent,
|
||||
MaintenanceEventStatus,
|
||||
MaintenanceSettings,
|
||||
)
|
||||
|
||||
|
||||
# #region build_banner_text [C:3] [TYPE Function] [SEMANTICS maintenance, banner, text, template, rendering]
|
||||
# @BRIEF Build aggregated banner markdown text from events using settings template.
|
||||
# Single event: substitute {start_time}, {end_time}, {message} directly.
|
||||
# Multiple events: render each event as sub-block, combine as {message}.
|
||||
# Missing variables → empty string. Message is HTML-escaped.
|
||||
# @PRE events is a non-empty list of dicts with start_time, end_time, message keys.
|
||||
# template is a non-empty string.
|
||||
# @POST Returns str with substituted template or empty string on failure.
|
||||
def build_banner_text(
|
||||
events: list[dict[str, Any]],
|
||||
template: str,
|
||||
display_timezone: str = "UTC",
|
||||
) -> str:
|
||||
with belief_scope(
|
||||
"build_banner_text",
|
||||
f"events={len(events)}",
|
||||
):
|
||||
if not events:
|
||||
app_logger.reflect("No events, returning empty string")
|
||||
return ""
|
||||
|
||||
def _format_dt(dt_str: str | None) -> str:
|
||||
"""Format datetime string for display."""
|
||||
if not dt_str:
|
||||
return "уточняется"
|
||||
try:
|
||||
dt = datetime.fromisoformat(dt_str.replace("Z", "+00:00"))
|
||||
return dt.strftime("%Y-%m-%d %H:%M")
|
||||
except (ValueError, TypeError):
|
||||
return dt_str or "уточняется"
|
||||
|
||||
def _event_substitutions(event: dict[str, Any]) -> dict[str, str]:
|
||||
"""Build substitution dict for a single event."""
|
||||
msg = event.get("message") or ""
|
||||
escaped_msg = html.escape(msg) if msg else ""
|
||||
return {
|
||||
"start_time": _format_dt(event.get("start_time")),
|
||||
"end_time": _format_dt(event.get("end_time")),
|
||||
"message": escaped_msg,
|
||||
}
|
||||
|
||||
if len(events) == 1:
|
||||
subs = _event_substitutions(events[0])
|
||||
result = template
|
||||
for key, value in subs.items():
|
||||
result = result.replace(f"{{{key}}}", value)
|
||||
# Remove any remaining unreplaced variables
|
||||
result = _re.sub(r"\{[a-zA-Z_]+\}", "", result)
|
||||
app_logger.reflect("Banner text built (single event)")
|
||||
return result
|
||||
|
||||
# Multiple events: build aggregated message, substitute as {message}
|
||||
event_blocks: list[str] = []
|
||||
for i, event in enumerate(events):
|
||||
subs = _event_substitutions(event)
|
||||
block = template
|
||||
for key, value in subs.items():
|
||||
block = block.replace(f"{{{key}}}", value)
|
||||
block = _re.sub(r"\{[a-zA-Z_]+\}", "", block)
|
||||
event_blocks.append(block)
|
||||
|
||||
combined_message = "\n\n---\n\n".join(event_blocks)
|
||||
|
||||
# Build aggregator template (use first event for start/end if uniform)
|
||||
result = template
|
||||
result = result.replace("{message}", combined_message)
|
||||
if len(events) == 1:
|
||||
subs = _event_substitutions(events[0])
|
||||
for key in ("start_time", "end_time"):
|
||||
result = result.replace(f"{{{key}}}", subs.get(key, ""))
|
||||
else:
|
||||
result = result.replace("{start_time}", "")
|
||||
result = result.replace("{end_time}", "")
|
||||
result = _re.sub(r"\{[a-zA-Z_]+\}", "", result)
|
||||
|
||||
app_logger.reflect("Banner text built (multi-event)")
|
||||
return result
|
||||
# #endregion build_banner_text
|
||||
|
||||
|
||||
# #region _build_banner_text_for_dashboard [C:2] [TYPE Function]
|
||||
# @BRIEF Build aggregated banner text for a banner based on all active events linked to it.
|
||||
def _build_banner_text_for_dashboard(
|
||||
banner_id: str,
|
||||
db_session: Session,
|
||||
template: str,
|
||||
timezone_str: str,
|
||||
) -> str:
|
||||
try:
|
||||
# Find all active dashboard states linked to this banner
|
||||
active_states = (
|
||||
db_session.query(MaintenanceDashboardState)
|
||||
.filter(
|
||||
MaintenanceDashboardState.banner_id == banner_id,
|
||||
MaintenanceDashboardState.status == MaintenanceDashboardStateStatus.ACTIVE,
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
if not active_states:
|
||||
return ""
|
||||
|
||||
# Load events for each state
|
||||
events_data: list[dict[str, Any]] = []
|
||||
for state in active_states:
|
||||
ev = (
|
||||
db_session.query(MaintenanceEvent)
|
||||
.filter(MaintenanceEvent.id == state.event_id)
|
||||
.first()
|
||||
)
|
||||
if ev and ev.status in (
|
||||
MaintenanceEventStatus.ACTIVE,
|
||||
MaintenanceEventStatus.PARTIAL,
|
||||
MaintenanceEventStatus.APPLYING,
|
||||
):
|
||||
events_data.append(
|
||||
{
|
||||
"start_time": ev.start_time.isoformat() if ev.start_time else None,
|
||||
"end_time": ev.end_time.isoformat() if ev.end_time else None,
|
||||
"message": ev.message or "",
|
||||
}
|
||||
)
|
||||
|
||||
if not events_data:
|
||||
return ""
|
||||
|
||||
return build_banner_text(events_data, template, timezone_str)
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Failed to build banner text",
|
||||
extra={"banner_id": banner_id},
|
||||
error=str(e),
|
||||
)
|
||||
return ""
|
||||
# #endregion _build_banner_text_for_dashboard
|
||||
|
||||
|
||||
# #region rebuild_banner [C:3] [TYPE Function] [SEMANTICS maintenance, banner, rebuild, text, update]
|
||||
# @BRIEF Rebuild aggregated banner text for a banner and update the Superset chart.
|
||||
# Query all active MaintenanceDashboardState records linked to banner,
|
||||
# build aggregated text via build_banner_text, update markdown chart in Superset.
|
||||
# @PRE banner_id points to an active MaintenanceDashboardBanner with a valid chart_id.
|
||||
# @POST Banner chart in Superset updated with current aggregated text. DB banner_text updated.
|
||||
# @SIDE_EFFECT Modifies Superset chart. Writes DB.
|
||||
# @RELATION DEPENDS_ON -> [build_banner_text]
|
||||
# @RELATION DEPENDS_ON -> [SupersetDashboardsWriteMixin]
|
||||
def rebuild_banner(
|
||||
banner_id: str,
|
||||
db_session: Session,
|
||||
superset_client: SupersetClient,
|
||||
) -> bool:
|
||||
with belief_scope("rebuild_banner", f"banner_id={banner_id}"):
|
||||
app_logger.reason(
|
||||
"Rebuilding banner text",
|
||||
extra={"banner_id": banner_id},
|
||||
)
|
||||
|
||||
banner = db_session.query(MaintenanceDashboardBanner).filter(
|
||||
MaintenanceDashboardBanner.id == banner_id
|
||||
).first()
|
||||
if not banner:
|
||||
app_logger.explore(
|
||||
"Banner not found",
|
||||
extra={"banner_id": banner_id},
|
||||
error="BannerNotFound",
|
||||
)
|
||||
return False
|
||||
|
||||
if banner.status != MaintenanceDashboardBannerStatus.ACTIVE:
|
||||
app_logger.reflect(
|
||||
"Banner not active, skipping",
|
||||
extra={"banner_id": banner_id, "status": banner.status.value},
|
||||
)
|
||||
return False
|
||||
|
||||
# Load settings for template
|
||||
settings = db_session.query(MaintenanceSettings).filter(
|
||||
MaintenanceSettings.id == "default"
|
||||
).first()
|
||||
template = settings.banner_template if settings else ""
|
||||
timezone_str = settings.display_timezone if settings else "UTC"
|
||||
|
||||
# Build aggregated text
|
||||
banner_text = _build_banner_text_for_dashboard(
|
||||
banner_id, db_session, template, timezone_str
|
||||
)
|
||||
|
||||
if not banner_text and banner.chart_id:
|
||||
# No active events — should be handled by end_maintenance, but clean up just in case
|
||||
app_logger.reflect(
|
||||
"No active events for banner, marking as REMOVED",
|
||||
extra={"banner_id": banner_id},
|
||||
)
|
||||
banner.status = MaintenanceDashboardBannerStatus.REMOVED
|
||||
db_session.flush()
|
||||
return True
|
||||
|
||||
# Update banner on dashboard (native MARKDOWN in layout)
|
||||
if banner.chart_id:
|
||||
try:
|
||||
superset_client.update_banner_on_dashboard(
|
||||
banner.dashboard_id, banner.chart_id, banner_text
|
||||
)
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Failed to update banner on dashboard during rebuild",
|
||||
extra={
|
||||
"chart_id": banner.chart_id,
|
||||
"banner_id": banner_id,
|
||||
"dashboard_id": banner.dashboard_id,
|
||||
},
|
||||
error=str(e),
|
||||
)
|
||||
return False
|
||||
|
||||
# Update DB
|
||||
banner.banner_text = banner_text
|
||||
db_session.flush()
|
||||
|
||||
app_logger.reflect(
|
||||
"Banner rebuilt successfully",
|
||||
extra={"banner_id": banner_id},
|
||||
)
|
||||
return True
|
||||
# #endregion rebuild_banner
|
||||
|
||||
# #endregion MaintenanceBannerRenderer
|
||||
359
backend/src/services/maintenance/_chart_manager.py
Normal file
359
backend/src/services/maintenance/_chart_manager.py
Normal file
@@ -0,0 +1,359 @@
|
||||
# #region MaintenanceChartManager [C:3] [TYPE Module] [SEMANTICS maintenance, chart, banner, lifecycle]
|
||||
# @BRIEF Chart operations for maintenance banners: create/get banner charts, process
|
||||
# per-dashboard state transitions for start and end orchestrators.
|
||||
# @LAYER Service
|
||||
# @RELATION DEPENDS_ON -> [SupersetDashboardsWriteMixin]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceModels]
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...core.logger import belief_scope, logger as app_logger
|
||||
from ...core.superset_client import SupersetClient
|
||||
from ...models.maintenance import (
|
||||
MaintenanceDashboardBanner,
|
||||
MaintenanceDashboardBannerStatus,
|
||||
MaintenanceDashboardState,
|
||||
MaintenanceDashboardStateStatus,
|
||||
MaintenanceEvent,
|
||||
MaintenanceEventStatus,
|
||||
MaintenanceSettings,
|
||||
)
|
||||
from ._banner_renderer import _build_banner_text_for_dashboard, rebuild_banner
|
||||
from ._dashboard_scanner import _resolve_dashboard_title
|
||||
|
||||
|
||||
# #region ensure_banner_chart [C:3] [TYPE Function] [SEMANTICS maintenance, banner, chart, creation]
|
||||
# @BRIEF Get or create a MaintenanceDashboardBanner for a dashboard in the target environment.
|
||||
# If an active banner exists in DB, return it. Otherwise, create a markdown chart in Superset,
|
||||
# update dashboard layout, and store the banner row.
|
||||
# @PRE dashboard_id exists in Superset. superset_client has write access.
|
||||
# @POST Returns MaintenanceDashboardBanner with chart_id set. Chart is placed at top of dashboard.
|
||||
# @SIDE_EFFECT Creates chart in Superset, modifies dashboard layout, writes DB row.
|
||||
# @RELATION DEPENDS_ON -> [SupersetDashboardsWriteMixin]
|
||||
def ensure_banner_chart(
|
||||
dashboard_id: int,
|
||||
environment_id: str,
|
||||
superset_client: SupersetClient,
|
||||
db_session: Session,
|
||||
) -> MaintenanceDashboardBanner:
|
||||
with belief_scope(
|
||||
"ensure_banner_chart",
|
||||
f"dashboard_id={dashboard_id}",
|
||||
):
|
||||
app_logger.reason(
|
||||
"Ensuring banner chart exists",
|
||||
extra={"dashboard_id": dashboard_id},
|
||||
)
|
||||
|
||||
# Check for existing active banner
|
||||
existing = (
|
||||
db_session.query(MaintenanceDashboardBanner)
|
||||
.filter(
|
||||
MaintenanceDashboardBanner.dashboard_id == dashboard_id,
|
||||
MaintenanceDashboardBanner.environment_id == environment_id,
|
||||
MaintenanceDashboardBanner.status == MaintenanceDashboardBannerStatus.ACTIVE,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
app_logger.reason(
|
||||
"Found existing active banner, verifying chart in Superset",
|
||||
extra={"banner_id": existing.id, "chart_id": existing.chart_id},
|
||||
)
|
||||
# Verify the chart still exists in Superset
|
||||
try:
|
||||
superset_client.get_chart(existing.chart_id)
|
||||
app_logger.reflect(
|
||||
"Existing banner chart is alive, reusing",
|
||||
extra={"banner_id": existing.id, "chart_id": existing.chart_id},
|
||||
)
|
||||
return existing
|
||||
except Exception:
|
||||
app_logger.explore(
|
||||
"Existing banner chart is stale (not found in Superset), marking as REMOVED",
|
||||
extra={
|
||||
"banner_id": existing.id,
|
||||
"chart_id": existing.chart_id,
|
||||
},
|
||||
)
|
||||
existing.status = MaintenanceDashboardBannerStatus.REMOVED
|
||||
db_session.flush()
|
||||
# Fall through to create a new chart
|
||||
|
||||
# Create markdown chart in Superset
|
||||
try:
|
||||
chart_id = superset_client.create_markdown_chart(
|
||||
dashboard_id, "*Maintenance banner placeholder*"
|
||||
)
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Failed to create markdown chart",
|
||||
extra={"dashboard_id": dashboard_id},
|
||||
error=str(e),
|
||||
)
|
||||
raise
|
||||
|
||||
# Insert chart at top of dashboard layout
|
||||
try:
|
||||
superset_client.update_dashboard_layout(dashboard_id, chart_id)
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Failed to update dashboard layout, deleting chart",
|
||||
extra={"dashboard_id": dashboard_id, "chart_id": chart_id},
|
||||
error=str(e),
|
||||
)
|
||||
# Clean up the chart we just created
|
||||
try:
|
||||
superset_client.delete_chart(chart_id)
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
# Persist banner row
|
||||
banner = MaintenanceDashboardBanner(
|
||||
environment_id=environment_id,
|
||||
dashboard_id=dashboard_id,
|
||||
chart_id=chart_id,
|
||||
banner_text="",
|
||||
status=MaintenanceDashboardBannerStatus.ACTIVE,
|
||||
)
|
||||
db_session.add(banner)
|
||||
db_session.flush()
|
||||
|
||||
app_logger.reflect(
|
||||
"Banner chart created and persisted",
|
||||
extra={
|
||||
"banner_id": banner.id,
|
||||
"chart_id": chart_id,
|
||||
"dashboard_id": dashboard_id,
|
||||
},
|
||||
)
|
||||
return banner
|
||||
# #endregion ensure_banner_chart
|
||||
|
||||
|
||||
# #region _process_dashboards_for_start [C:3] [TYPE Function]
|
||||
# @BRIEF Process each dashboard for a start maintenance event: ensure banner chart,
|
||||
# create dashboard state, build banner text, update chart markdown.
|
||||
# @SIDE_EFFECT Creates/updates Superset charts, writes DB rows.
|
||||
# @RELATION CALLS -> [ensure_banner_chart]
|
||||
# @RELATION CALLS -> [_build_banner_text_for_dashboard]
|
||||
# @RELATION CALLS -> [_resolve_dashboard_title]
|
||||
def _process_dashboards_for_start(
|
||||
dashboard_ids: list[int],
|
||||
event_id: str,
|
||||
environment_id: str,
|
||||
template: str,
|
||||
timezone_str: str,
|
||||
superset_client: SupersetClient,
|
||||
db_session: Session,
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[MaintenanceDashboardBanner]]:
|
||||
successful_dashboards: list[dict[str, Any]] = []
|
||||
failed_dashboards: list[dict[str, Any]] = []
|
||||
banners_created: list[MaintenanceDashboardBanner] = []
|
||||
|
||||
for dash_id in dashboard_ids:
|
||||
app_logger.reason(
|
||||
"Processing dashboard",
|
||||
extra={"dashboard_id": dash_id, "event_id": event_id},
|
||||
)
|
||||
try:
|
||||
# Ensure banner chart exists
|
||||
banner = ensure_banner_chart(
|
||||
dash_id,
|
||||
environment_id,
|
||||
superset_client,
|
||||
db_session,
|
||||
)
|
||||
banners_created.append(banner)
|
||||
|
||||
# Create dashboard state
|
||||
state = MaintenanceDashboardState(
|
||||
event_id=event_id,
|
||||
dashboard_id=dash_id,
|
||||
banner_id=banner.id,
|
||||
status=MaintenanceDashboardStateStatus.ACTIVE,
|
||||
)
|
||||
db_session.add(state)
|
||||
db_session.flush() # Ensure state is visible to subsequent queries
|
||||
|
||||
# Build banner text for this dashboard's active events
|
||||
banner_text = _build_banner_text_for_dashboard(
|
||||
banner.id, db_session, template, timezone_str
|
||||
)
|
||||
|
||||
# Update banner content on dashboard (native MARKDOWN in layout)
|
||||
if banner.chart_id:
|
||||
try:
|
||||
superset_client.update_banner_on_dashboard(
|
||||
dash_id, banner.chart_id, banner_text
|
||||
)
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Failed to update banner on dashboard",
|
||||
extra={
|
||||
"chart_id": banner.chart_id,
|
||||
"dashboard_id": dash_id,
|
||||
},
|
||||
error=str(e),
|
||||
)
|
||||
state.status = MaintenanceDashboardStateStatus.APPLY_FAILED
|
||||
failed_dashboards.append(
|
||||
{"id": dash_id, "title": str(dash_id), "error": str(e)}
|
||||
)
|
||||
continue
|
||||
|
||||
# Update banner text in DB
|
||||
banner.banner_text = banner_text
|
||||
db_session.flush()
|
||||
|
||||
# Resolve dashboard title
|
||||
title = _resolve_dashboard_title(dash_id, superset_client)
|
||||
successful_dashboards.append(
|
||||
{
|
||||
"id": dash_id,
|
||||
"title": title,
|
||||
"chart_id": banner.chart_id,
|
||||
}
|
||||
)
|
||||
app_logger.reflect(
|
||||
"Dashboard processed successfully",
|
||||
extra={"dashboard_id": dash_id},
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Failed to process dashboard",
|
||||
extra={"dashboard_id": dash_id},
|
||||
error=str(e),
|
||||
)
|
||||
failed_dashboards.append(
|
||||
{"id": dash_id, "title": str(dash_id), "error": str(e)}
|
||||
)
|
||||
|
||||
return successful_dashboards, failed_dashboards, banners_created
|
||||
# #endregion _process_dashboards_for_start
|
||||
|
||||
|
||||
# #region _process_states_for_end [C:3] [TYPE Function]
|
||||
# @BRIEF Process each dashboard state for an end maintenance event: if other events
|
||||
# still use same banner → rebuild; if no other events → remove banner (chart + layout).
|
||||
# @SIDE_EFFECT Modifies Superset dashboards (removes charts, updates layouts). Writes DB.
|
||||
# @RELATION CALLS -> [rebuild_banner]
|
||||
def _process_states_for_end(
|
||||
states: list[MaintenanceDashboardState],
|
||||
event_id: str,
|
||||
superset_client: SupersetClient,
|
||||
db_session: Session,
|
||||
) -> tuple[int, list[dict[str, Any]]]:
|
||||
removed_count = 0
|
||||
failed_removals: list[dict[str, Any]] = []
|
||||
|
||||
for state in states:
|
||||
dash_id = state.dashboard_id
|
||||
app_logger.reason(
|
||||
"Processing dashboard state for end",
|
||||
extra={"dashboard_id": dash_id, "state_id": state.id},
|
||||
)
|
||||
|
||||
try:
|
||||
banner_id = state.banner_id
|
||||
if not banner_id:
|
||||
# No banner linked, just mark as removed
|
||||
state.status = MaintenanceDashboardStateStatus.REMOVED
|
||||
removed_count += 1
|
||||
continue
|
||||
|
||||
# Check if other active events still use this banner
|
||||
other_active = db_session.query(MaintenanceDashboardState).filter(
|
||||
MaintenanceDashboardState.banner_id == banner_id,
|
||||
MaintenanceDashboardState.status == MaintenanceDashboardStateStatus.ACTIVE,
|
||||
MaintenanceDashboardState.event_id != event_id,
|
||||
).count()
|
||||
|
||||
banner = db_session.query(MaintenanceDashboardBanner).filter(
|
||||
MaintenanceDashboardBanner.id == banner_id
|
||||
).first()
|
||||
|
||||
if other_active > 0 and banner:
|
||||
# Other events still active — rebuild banner text
|
||||
state.status = MaintenanceDashboardStateStatus.REMOVED
|
||||
db_session.flush()
|
||||
rebuild_banner(banner_id, db_session, superset_client)
|
||||
app_logger.reflect(
|
||||
"Banner rebuilt after removing event",
|
||||
extra={"banner_id": banner_id, "dashboard_id": dash_id},
|
||||
)
|
||||
elif banner:
|
||||
# No other events — remove banner entirely
|
||||
state.status = MaintenanceDashboardStateStatus.REMOVING
|
||||
db_session.flush()
|
||||
|
||||
# Remove chart from layout
|
||||
if banner.chart_id:
|
||||
try:
|
||||
superset_client.remove_chart_from_layout(
|
||||
dash_id, banner.chart_id
|
||||
)
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Failed to remove chart from layout",
|
||||
extra={
|
||||
"dashboard_id": dash_id,
|
||||
"chart_id": banner.chart_id,
|
||||
},
|
||||
error=str(e),
|
||||
)
|
||||
state.status = MaintenanceDashboardStateStatus.REMOVAL_FAILED
|
||||
failed_removals.append(
|
||||
{"id": dash_id, "title": str(dash_id), "error": str(e)}
|
||||
)
|
||||
continue
|
||||
|
||||
# Delete chart
|
||||
try:
|
||||
superset_client.delete_chart(banner.chart_id)
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Failed to delete chart",
|
||||
extra={
|
||||
"chart_id": banner.chart_id,
|
||||
"dashboard_id": dash_id,
|
||||
},
|
||||
error=str(e),
|
||||
)
|
||||
state.status = MaintenanceDashboardStateStatus.REMOVAL_FAILED
|
||||
failed_removals.append(
|
||||
{"id": dash_id, "title": str(dash_id), "error": str(e)}
|
||||
)
|
||||
continue
|
||||
|
||||
# Mark banner as removed
|
||||
banner.status = MaintenanceDashboardBannerStatus.REMOVED
|
||||
state.status = MaintenanceDashboardStateStatus.REMOVED
|
||||
app_logger.reflect(
|
||||
"Banner fully removed",
|
||||
extra={"banner_id": banner_id, "dashboard_id": dash_id},
|
||||
)
|
||||
else:
|
||||
state.status = MaintenanceDashboardStateStatus.REMOVED
|
||||
|
||||
removed_count += 1
|
||||
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Failed to end maintenance for dashboard",
|
||||
extra={"dashboard_id": dash_id},
|
||||
error=str(e),
|
||||
)
|
||||
state.status = MaintenanceDashboardStateStatus.REMOVAL_FAILED
|
||||
failed_removals.append(
|
||||
{"id": dash_id, "title": str(dash_id), "error": str(e)}
|
||||
)
|
||||
|
||||
return removed_count, failed_removals
|
||||
# #endregion _process_states_for_end
|
||||
|
||||
# #endregion MaintenanceChartManager
|
||||
205
backend/src/services/maintenance/_dashboard_scanner.py
Normal file
205
backend/src/services/maintenance/_dashboard_scanner.py
Normal file
@@ -0,0 +1,205 @@
|
||||
# #region MaintenanceDashboardScanner [C:3] [TYPE Module] [SEMANTICS maintenance, dashboard, scanner, tables]
|
||||
# @BRIEF Dashboard discovery and filtering helpers for maintenance banner feature.
|
||||
# Scan Superset datasets, match tables, apply scope/excluded/forced filters.
|
||||
# @LAYER Service
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [SqlTableExtractorModule]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceModels]
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...core.logger import belief_scope, logger as app_logger
|
||||
from ...core.superset_client import SupersetClient
|
||||
from ...models.maintenance import DashboardScope, MaintenanceSettings
|
||||
from ..sql_table_extractor import extract_tables_from_sql
|
||||
|
||||
|
||||
# #region find_affected_dashboards [C:3] [TYPE Function] [SEMANTICS maintenance, dashboard, discovery, tables]
|
||||
# @BRIEF Find all dashboard IDs whose datasets reference any of the given tables.
|
||||
# Supports both table-based (schema, table_name) and virtual SQL datasets (via SqlTableExtractor).
|
||||
# Applies scope/excluded/forced filtering from MaintenanceSettings.
|
||||
# @PRE tables is a non-empty list of "schema.table" strings. superset_client is authenticated.
|
||||
# @POST Returns a deduplicated list of dashboard IDs that should receive banners.
|
||||
# @SIDE_EFFECT Fetches all datasets from Superset (paginated). May be slow with many datasets.
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [SqlTableExtractorModule]
|
||||
def find_affected_dashboards(
|
||||
tables: list[str],
|
||||
superset_client: SupersetClient,
|
||||
settings: MaintenanceSettings | None = None,
|
||||
) -> list[int]:
|
||||
with belief_scope(
|
||||
"find_affected_dashboards",
|
||||
f"tables={tables}",
|
||||
):
|
||||
app_logger.reason(
|
||||
"Finding affected dashboards",
|
||||
extra={"tables": tables},
|
||||
)
|
||||
|
||||
if not tables:
|
||||
app_logger.reflect("No tables provided, returning empty")
|
||||
return []
|
||||
|
||||
# Normalize input tables to lowercase set for matching
|
||||
target_tables = {t.lower().strip() for t in tables if t.strip()}
|
||||
if not target_tables:
|
||||
return []
|
||||
|
||||
# Fetch all datasets from Superset
|
||||
try:
|
||||
_, datasets = superset_client.get_datasets()
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Failed to fetch datasets from Superset",
|
||||
extra={},
|
||||
error=str(e),
|
||||
)
|
||||
raise
|
||||
|
||||
app_logger.reason(
|
||||
f"Scanning {len(datasets)} datasets for table references",
|
||||
extra={"target_tables": list(target_tables)},
|
||||
)
|
||||
|
||||
matched_dashboard_ids: set[int] = set()
|
||||
|
||||
for ds in datasets:
|
||||
ds_schema = (ds.get("schema") or "").lower().strip()
|
||||
ds_table = (ds.get("table_name") or "").lower().strip()
|
||||
is_virtual = ds.get("is_sqllab_view", False) or ds.get("sql", "") not in (
|
||||
None,
|
||||
"",
|
||||
)
|
||||
ds_sql = ds.get("sql") or ""
|
||||
ds_id = ds.get("id")
|
||||
|
||||
# Phase 1: Table-based match
|
||||
if not is_virtual and ds_schema and ds_table:
|
||||
qualified = f"{ds_schema}.{ds_table}"
|
||||
if qualified in target_tables:
|
||||
linked = _get_linked_dashboards(superset_client, ds_id)
|
||||
matched_dashboard_ids.update(linked)
|
||||
continue
|
||||
|
||||
# Phase 2: Virtual/SQL dataset — extract tables from SQL text
|
||||
if is_virtual and ds_sql:
|
||||
extracted = extract_tables_from_sql(ds_sql)
|
||||
if extracted & target_tables:
|
||||
linked = _get_linked_dashboards(superset_client, ds_id)
|
||||
matched_dashboard_ids.update(linked)
|
||||
|
||||
app_logger.reflect(
|
||||
f"Found {len(matched_dashboard_ids)} affected dashboards",
|
||||
extra={"dashboard_ids": list(matched_dashboard_ids)},
|
||||
)
|
||||
|
||||
# Apply filtering from settings
|
||||
result = _apply_dashboard_filters(
|
||||
list(matched_dashboard_ids), superset_client, settings
|
||||
)
|
||||
|
||||
app_logger.reflect(
|
||||
f"After filtering: {len(result)} dashboards",
|
||||
extra={"dashboard_ids": result},
|
||||
)
|
||||
return result
|
||||
# #endregion find_affected_dashboards
|
||||
|
||||
|
||||
# #region _get_linked_dashboards [C:2] [TYPE Function]
|
||||
# @BRIEF Fetch linked dashboards for a dataset from Superset.
|
||||
# @SIDE_EFFECT Calls Superset API.
|
||||
def _get_linked_dashboards(
|
||||
superset_client: SupersetClient,
|
||||
dataset_id: int | None,
|
||||
) -> set[int]:
|
||||
if not dataset_id:
|
||||
return set()
|
||||
try:
|
||||
detail = superset_client.get_dataset_detail(dataset_id)
|
||||
linked = detail.get("linked_dashboards", [])
|
||||
return {int(d["id"]) for d in linked if d.get("id")}
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Failed to fetch linked dashboards",
|
||||
extra={"dataset_id": dataset_id},
|
||||
error=str(e),
|
||||
)
|
||||
return set()
|
||||
# #endregion _get_linked_dashboards
|
||||
|
||||
|
||||
# #region _apply_dashboard_filters [C:2] [TYPE Function]
|
||||
# @BRIEF Apply scope (published/draft/all), excluded, and forced filters from settings.
|
||||
def _apply_dashboard_filters(
|
||||
dashboard_ids: list[int],
|
||||
superset_client: SupersetClient,
|
||||
settings: MaintenanceSettings | None,
|
||||
) -> list[int]:
|
||||
if not settings:
|
||||
return dashboard_ids
|
||||
|
||||
# If forced list exists, include those unconditionally
|
||||
forced_ids = set(settings.forced_dashboard_ids or [])
|
||||
excluded_ids = set(settings.excluded_dashboard_ids or [])
|
||||
|
||||
# Apply scope filtering
|
||||
if settings.dashboard_scope != DashboardScope.ALL:
|
||||
try:
|
||||
_, all_dashboards = superset_client.get_dashboards()
|
||||
scope = settings.dashboard_scope
|
||||
filtered: list[int] = []
|
||||
for did in dashboard_ids:
|
||||
dash = next(
|
||||
(d for d in all_dashboards if d.get("id") == did), None
|
||||
)
|
||||
if dash is None:
|
||||
continue
|
||||
is_published = dash.get("published", True)
|
||||
if scope == DashboardScope.PUBLISHED_ONLY and not is_published:
|
||||
continue
|
||||
if scope == DashboardScope.DRAFT_ONLY and is_published:
|
||||
continue
|
||||
filtered.append(did)
|
||||
dashboard_ids = filtered
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Scope filtering failed, proceeding without it",
|
||||
extra={},
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
# Apply excluded (forced overrides excluded per FR-009-b)
|
||||
result = [
|
||||
did for did in dashboard_ids if did not in excluded_ids or did in forced_ids
|
||||
]
|
||||
|
||||
# Add forced dashboards that are not already included
|
||||
for fid in forced_ids:
|
||||
if fid not in result:
|
||||
result.append(fid)
|
||||
|
||||
return result
|
||||
# #endregion _apply_dashboard_filters
|
||||
|
||||
|
||||
# #region _resolve_dashboard_title [C:2] [TYPE Function]
|
||||
# @BRIEF Resolve dashboard title from Superset by ID.
|
||||
def _resolve_dashboard_title(
|
||||
dashboard_id: int,
|
||||
superset_client: SupersetClient,
|
||||
) -> str:
|
||||
try:
|
||||
_, dashboards = superset_client.get_dashboards()
|
||||
for d in dashboards:
|
||||
if d.get("id") == dashboard_id:
|
||||
return d.get("dashboard_title") or d.get("title") or str(dashboard_id)
|
||||
except Exception:
|
||||
pass
|
||||
return str(dashboard_id)
|
||||
# #endregion _resolve_dashboard_title
|
||||
|
||||
# #endregion MaintenanceDashboardScanner
|
||||
338
backend/src/services/maintenance/_orchestrators.py
Normal file
338
backend/src/services/maintenance/_orchestrators.py
Normal file
@@ -0,0 +1,338 @@
|
||||
# #region MaintenanceOrchestrators [C:4] [TYPE Module] [SEMANTICS maintenance, orchestrator, lifecycle]
|
||||
# @BRIEF C4 orchestrators for maintenance event lifecycle: start, end, end-all.
|
||||
# Private helpers extracted to sub-modules. All orchestrators use belief runtime.
|
||||
# @LAYER Service
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceModels]
|
||||
# @RELATION DEPENDS_ON -> [SupersetClient]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceDashboardScanner]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceBannerRenderer]
|
||||
# @RELATION DEPENDS_ON -> [MaintenanceChartManager]
|
||||
# @INVARIANT C4 methods use belief_scope + REASON/REFLECT/EXPLORE markers.
|
||||
# @INVARIANT Idempotency key: sorted lowercased (tables, start_time, end_time). message NOT in key.
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...core.logger import belief_scope, logger as app_logger
|
||||
from ...core.superset_client import SupersetClient
|
||||
from ...models.maintenance import (
|
||||
MaintenanceDashboardState,
|
||||
MaintenanceEvent,
|
||||
MaintenanceEventStatus,
|
||||
MaintenanceSettings,
|
||||
)
|
||||
from ._chart_manager import _process_dashboards_for_start, _process_states_for_end
|
||||
from ._dashboard_scanner import find_affected_dashboards
|
||||
|
||||
|
||||
# #region build_idempotency_key [C:2] [TYPE Function]
|
||||
# @BRIEF Build idempotency key from (tables, start_time, end_time). Sorted lowercased tables, ISO timestamps.
|
||||
# @PRE tables is a list of strings. start_time is a datetime or None. end_time is a datetime or None.
|
||||
# @POST Returns a tuple of (frozenset(str), str|None, str|None) for DB comparison.
|
||||
# @RATIONALE message is intentionally excluded from key per spec — different message with same tables/times = same event.
|
||||
def build_idempotency_key(
|
||||
tables: list[str],
|
||||
start_time: datetime | None,
|
||||
end_time: datetime | None,
|
||||
) -> tuple[frozenset[str], str | None, str | None]:
|
||||
sorted_tables = sorted(t.strip().lower() for t in tables if t.strip())
|
||||
start_key = start_time.isoformat() if start_time else None
|
||||
end_key = end_time.isoformat() if end_time else None
|
||||
return (frozenset(sorted_tables), start_key, end_key)
|
||||
# #endregion build_idempotency_key
|
||||
|
||||
|
||||
# #region start_maintenance [C:4] [TYPE Function] [SEMANTICS maintenance, start, orchestration, async]
|
||||
# @BRIEF C4 orchestrator for starting a maintenance event. Runs inside a TaskManager task.
|
||||
# Idempotency check with SELECT ... FOR UPDATE. Creates event states, finds dashboards,
|
||||
# ensures banner charts, builds text, updates charts. Event transitions to ACTIVE or PARTIAL.
|
||||
# @PRE event_id points to a valid MaintenanceEvent with status=PENDING.
|
||||
# @POST Event and dashboard states transitioned to ACTIVE/PARTIAL/APPLY_FAILED.
|
||||
# Banner charts created and updated in Superset.
|
||||
# @SIDE_EFFECT Modifies Superset dashboards (creates/updates charts, modifies layouts).
|
||||
# Writes multiple DB rows (dashboard states, banner updates).
|
||||
# @BELIEF Uses belief_scope with REASON/REFLECT/EXPLORE markers.
|
||||
# @RELATION CALLS -> [find_affected_dashboards]
|
||||
# @RELATION CALLS -> [_process_dashboards_for_start]
|
||||
def start_maintenance(
|
||||
event_id: str,
|
||||
db_session: Session,
|
||||
superset_client: SupersetClient,
|
||||
) -> dict[str, Any]:
|
||||
with belief_scope("start_maintenance", f"event_id={event_id}"):
|
||||
app_logger.reason(
|
||||
"Starting maintenance event",
|
||||
extra={"event_id": event_id},
|
||||
)
|
||||
|
||||
# Load event
|
||||
event = db_session.query(MaintenanceEvent).filter(
|
||||
MaintenanceEvent.id == event_id
|
||||
).with_for_update().first()
|
||||
if not event:
|
||||
app_logger.explore(
|
||||
"Event not found",
|
||||
extra={"event_id": event_id},
|
||||
error="EventNotFound",
|
||||
)
|
||||
return {"maintenance_id": event_id, "status": "failed", "error": "Event not found"}
|
||||
|
||||
if event.status != MaintenanceEventStatus.PENDING:
|
||||
app_logger.reflect(
|
||||
"Event already processed",
|
||||
extra={"event_id": event_id, "status": event.status.value},
|
||||
)
|
||||
return {"maintenance_id": event_id, "status": event.status.value}
|
||||
|
||||
# Transition to APPLYING
|
||||
event.status = MaintenanceEventStatus.APPLYING
|
||||
db_session.flush()
|
||||
|
||||
# Load settings
|
||||
settings = db_session.query(MaintenanceSettings).filter(
|
||||
MaintenanceSettings.id == "default"
|
||||
).first()
|
||||
environment_id = settings.target_environment_id if settings else event.environment_id
|
||||
template = settings.banner_template if settings else ""
|
||||
timezone_str = settings.display_timezone if settings else "UTC"
|
||||
|
||||
# Find affected dashboards
|
||||
try:
|
||||
dashboard_ids = find_affected_dashboards(
|
||||
event.tables,
|
||||
superset_client,
|
||||
settings,
|
||||
)
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Dashboard discovery failed",
|
||||
extra={"event_id": event_id},
|
||||
error=str(e),
|
||||
)
|
||||
event.status = MaintenanceEventStatus.FAILED
|
||||
db_session.flush()
|
||||
return {
|
||||
"maintenance_id": event_id,
|
||||
"status": "failed",
|
||||
"error": str(e),
|
||||
"affected_dashboards": 0,
|
||||
"dashboards": [],
|
||||
"failed_dashboards": [],
|
||||
"unmatched_tables": [],
|
||||
}
|
||||
|
||||
if not dashboard_ids:
|
||||
app_logger.reflect(
|
||||
"No matching dashboards found",
|
||||
extra={"event_id": event_id},
|
||||
)
|
||||
event.status = MaintenanceEventStatus.ACTIVE
|
||||
db_session.flush()
|
||||
return {
|
||||
"maintenance_id": event_id,
|
||||
"status": "no_match",
|
||||
"affected_dashboards": 0,
|
||||
"dashboards": [],
|
||||
"failed_dashboards": [],
|
||||
"unmatched_tables": [
|
||||
t for t in event.tables if t not in dashboard_ids
|
||||
],
|
||||
}
|
||||
|
||||
# Process each dashboard
|
||||
successful_dashboards, failed_dashboards, _ = _process_dashboards_for_start(
|
||||
dashboard_ids,
|
||||
event_id,
|
||||
environment_id,
|
||||
template,
|
||||
timezone_str,
|
||||
superset_client,
|
||||
db_session,
|
||||
)
|
||||
|
||||
# Determine final event status
|
||||
if not failed_dashboards:
|
||||
event.status = MaintenanceEventStatus.ACTIVE
|
||||
elif successful_dashboards:
|
||||
event.status = MaintenanceEventStatus.PARTIAL
|
||||
else:
|
||||
event.status = MaintenanceEventStatus.FAILED
|
||||
|
||||
db_session.flush()
|
||||
|
||||
result = {
|
||||
"maintenance_id": event_id,
|
||||
"status": event.status.value,
|
||||
"affected_dashboards": len(successful_dashboards),
|
||||
"dashboards": successful_dashboards,
|
||||
"failed_dashboards": failed_dashboards,
|
||||
"unmatched_tables": [],
|
||||
}
|
||||
|
||||
app_logger.reflect(
|
||||
"Maintenance start completed",
|
||||
extra={
|
||||
"event_id": event_id,
|
||||
"status": event.status.value,
|
||||
"successful": len(successful_dashboards),
|
||||
"failed": len(failed_dashboards),
|
||||
},
|
||||
)
|
||||
return result
|
||||
# #endregion start_maintenance
|
||||
|
||||
|
||||
# #region end_maintenance [C:4] [TYPE Function] [SEMANTICS maintenance, end, orchestration, async, removal]
|
||||
# @BRIEF C4 orchestrator for ending a maintenance event. Runs inside a TaskManager task.
|
||||
# Event → ENDING. Delegates per-state processing to _process_states_for_end.
|
||||
# Event → COMPLETED when done.
|
||||
# @PRE event_id points to a valid active MaintenanceEvent.
|
||||
# @POST Event transitions to COMPLETED. Dashboard states cleaned up. Charts removed when no other events.
|
||||
# @SIDE_EFFECT Modifies Superset dashboards (removes charts, updates layouts). Writes DB.
|
||||
# @BELIEF Uses belief_scope with REASON/REFLECT/EXPLORE markers.
|
||||
# @RELATION CALLS -> [_process_states_for_end]
|
||||
def end_maintenance(
|
||||
event_id: str,
|
||||
db_session: Session,
|
||||
superset_client: SupersetClient,
|
||||
) -> dict[str, Any]:
|
||||
with belief_scope("end_maintenance", f"event_id={event_id}"):
|
||||
app_logger.reason(
|
||||
"Ending maintenance event",
|
||||
extra={"event_id": event_id},
|
||||
)
|
||||
|
||||
# Load event with lock
|
||||
event = db_session.query(MaintenanceEvent).filter(
|
||||
MaintenanceEvent.id == event_id
|
||||
).with_for_update().first()
|
||||
if not event:
|
||||
app_logger.explore(
|
||||
"Event not found",
|
||||
extra={"event_id": event_id},
|
||||
error="EventNotFound",
|
||||
)
|
||||
return {"maintenance_id": event_id, "status": "failed", "error": "Event not found"}
|
||||
|
||||
# Idempotent: already completed
|
||||
if event.status == MaintenanceEventStatus.COMPLETED:
|
||||
app_logger.reflect(
|
||||
"Event already completed, idempotent",
|
||||
extra={"event_id": event_id},
|
||||
)
|
||||
return {"maintenance_id": event_id, "status": "already_completed", "removed_from": 0, "failed_removals": []}
|
||||
|
||||
# Transition to ENDING
|
||||
event.status = MaintenanceEventStatus.ENDING
|
||||
db_session.flush()
|
||||
|
||||
# Find all dashboard states for this event
|
||||
states = db_session.query(MaintenanceDashboardState).filter(
|
||||
MaintenanceDashboardState.event_id == event_id
|
||||
).all()
|
||||
|
||||
# Delegate per-state processing
|
||||
removed_count, failed_removals = _process_states_for_end(
|
||||
states, event_id, superset_client, db_session,
|
||||
)
|
||||
|
||||
# Transition event to COMPLETED (even if some removals failed)
|
||||
event.status = MaintenanceEventStatus.COMPLETED
|
||||
db_session.flush()
|
||||
|
||||
result = {
|
||||
"maintenance_id": event_id,
|
||||
"status": "completed" if not failed_removals else "partial",
|
||||
"removed_from": removed_count,
|
||||
"failed_removals": failed_removals,
|
||||
}
|
||||
|
||||
app_logger.reflect(
|
||||
"Maintenance end completed",
|
||||
extra={
|
||||
"event_id": event_id,
|
||||
"removed": removed_count,
|
||||
"failed": len(failed_removals),
|
||||
},
|
||||
)
|
||||
return result
|
||||
# #endregion end_maintenance
|
||||
|
||||
|
||||
# #region end_all_maintenance [C:4] [TYPE Function] [SEMANTICS maintenance, end-all, orchestration, bulk]
|
||||
# @BRIEF End ALL active maintenance events. Iterates active events, calls end_maintenance for each.
|
||||
# @PRE None.
|
||||
# @POST All events ended. All banners removed or rebuilt (last event removed).
|
||||
# @SIDE_EFFECT Modifies multiple Superset dashboards. Writes multiple DB rows.
|
||||
# @BELIEF Uses belief_scope with REASON/REFLECT/EXPLORE markers.
|
||||
# @RELATION CALLS -> [end_maintenance]
|
||||
def end_all_maintenance(
|
||||
db_session: Session,
|
||||
superset_client: SupersetClient,
|
||||
) -> dict[str, Any]:
|
||||
with belief_scope("end_all_maintenance"):
|
||||
app_logger.reason("Ending all active maintenance events")
|
||||
|
||||
active_events = db_session.query(MaintenanceEvent).filter(
|
||||
MaintenanceEvent.status.in_([
|
||||
MaintenanceEventStatus.ACTIVE,
|
||||
MaintenanceEventStatus.PARTIAL,
|
||||
MaintenanceEventStatus.APPLYING,
|
||||
])
|
||||
).all()
|
||||
|
||||
if not active_events:
|
||||
app_logger.reflect("No active events to end")
|
||||
return {
|
||||
"status": "completed",
|
||||
"closed_events": 0,
|
||||
"removed_from": 0,
|
||||
"failed_removals": [],
|
||||
}
|
||||
|
||||
total_removed = 0
|
||||
all_failed: list[dict[str, Any]] = []
|
||||
closed_count = 0
|
||||
|
||||
for event in active_events:
|
||||
app_logger.reason(
|
||||
"Ending event",
|
||||
extra={"event_id": event.id},
|
||||
)
|
||||
try:
|
||||
result = end_maintenance(
|
||||
event.id, db_session, superset_client
|
||||
)
|
||||
total_removed += result.get("removed_from", 0)
|
||||
all_failed.extend(result.get("failed_removals", []))
|
||||
closed_count += 1
|
||||
except Exception as e:
|
||||
app_logger.explore(
|
||||
"Failed to end event",
|
||||
extra={"event_id": event.id},
|
||||
error=str(e),
|
||||
)
|
||||
all_failed.append(
|
||||
{"id": 0, "title": event.id, "error": str(e)}
|
||||
)
|
||||
|
||||
app_logger.reflect(
|
||||
"All events ended",
|
||||
extra={
|
||||
"closed": closed_count,
|
||||
"removed": total_removed,
|
||||
"failed": len(all_failed),
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "completed" if not all_failed else "partial",
|
||||
"closed_events": closed_count,
|
||||
"removed_from": total_removed,
|
||||
"failed_removals": all_failed,
|
||||
}
|
||||
# #endregion end_all_maintenance
|
||||
|
||||
# #endregion MaintenanceOrchestrators
|
||||
226
backend/src/services/sql_table_extractor.py
Normal file
226
backend/src/services/sql_table_extractor.py
Normal file
@@ -0,0 +1,226 @@
|
||||
# #region SqlTableExtractorModule [C:2] [TYPE Module] [SEMANTICS sql, jinja, table, extraction, parsing]
|
||||
# @BRIEF Three-phase SQL+Jinja table name extractor for virtual datasets as per FR-003.
|
||||
# Phase 1: Detect Jinja spans vs SQL spans
|
||||
# Phase 2: In Jinja spans, extract "schema.table" from string values
|
||||
# Phase 3: In SQL spans, regex pattern + sqlparse filter to reject string literal false positives
|
||||
# @LAYER Service
|
||||
# @RELATION DEPENDS_ON -> [re, sqlparse]
|
||||
# @INVARIANT Only exact schema.table matches (case-insensitive); unqualified references are NOT matched.
|
||||
# @INVARIANT Returns a set[str] of fully-qualified table names (lowercased for case-insensitive matching).
|
||||
|
||||
import re
|
||||
from typing import Iterable
|
||||
|
||||
import sqlparse
|
||||
from sqlparse.sql import Identifier, Parenthesis, Token, TokenList
|
||||
from sqlparse.tokens import Literal, Name, Punctuation, Whitespace
|
||||
|
||||
# ── Phase 1: Jinja span detection ───────────────────────────
|
||||
|
||||
# Minimal Jinja token detection: {% ... %}, {{ ... }}, {# ... #}
|
||||
_JINJA_BLOCK_RE = re.compile(r"{%-?\s*.*?\s*-?%}", re.DOTALL)
|
||||
_JINJA_EXPR_RE = re.compile(r"{{-?\s*.*?\s*-?}}", re.DOTALL)
|
||||
_JINJA_COMMENT_RE = re.compile(r"{#.*?#}", re.DOTALL)
|
||||
|
||||
# ── Phase 3: schema.table regex (case-insensitive) ───────────
|
||||
_SCHEMA_TABLE_RE = re.compile(
|
||||
r"""
|
||||
(?<!['"`\w]) # not preceded by string delimiter or word char
|
||||
\b # word boundary
|
||||
(?: # begin: non-capturing group for full match
|
||||
(?:"([a-zA-Z_][\w]*)")? # optional quoted schema
|
||||
(?:([a-zA-Z_][\w]*)) # schema (unquoted)
|
||||
\.
|
||||
(?:([a-zA-Z_][\w]*)) # table
|
||||
)
|
||||
\b # word boundary
|
||||
(?!['"`]) # not followed by a string delimiter
|
||||
""",
|
||||
re.VERBOSE | re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
# #region detect_jinja_spans [C:2] [TYPE Function]
|
||||
# @BRIEF Phase 1: Split raw SQL text into Jinja spans and SQL spans.
|
||||
# @PRE raw_sql is a string (possibly empty).
|
||||
# @POST Returns a list of (span_type: str, text: str) tuples.
|
||||
# span_type is "jinja" or "sql".
|
||||
def detect_jinja_spans(raw_sql: str) -> list[tuple[str, str]]:
|
||||
"""Split raw SQL+Jinja text into alternating Jinja and SQL spans.
|
||||
|
||||
Phase 1 detects Jinja blocks ({%%}, {{}}, {##}) and returns the
|
||||
remaining text as SQL spans.
|
||||
"""
|
||||
if not raw_sql or not raw_sql.strip():
|
||||
return [("sql", raw_sql or "")]
|
||||
|
||||
spans: list[tuple[str, str]] = []
|
||||
# Collect all Jinja match positions
|
||||
jinja_matches: list[tuple[int, int]] = []
|
||||
for pattern in (_JINJA_BLOCK_RE, _JINJA_EXPR_RE, _JINJA_COMMENT_RE):
|
||||
for m in pattern.finditer(raw_sql):
|
||||
jinja_matches.append((m.start(), m.end()))
|
||||
|
||||
# Merge overlapping Jinja spans
|
||||
if jinja_matches:
|
||||
jinja_matches.sort()
|
||||
merged: list[tuple[int, int]] = [jinja_matches[0]]
|
||||
for start, end in jinja_matches[1:]:
|
||||
if start <= merged[-1][1]:
|
||||
merged[-1] = (merged[-1][0], max(merged[-1][1], end))
|
||||
else:
|
||||
merged.append((start, end))
|
||||
|
||||
# Build alternating sql/jinja spans
|
||||
cursor = 0
|
||||
for start, end in merged:
|
||||
if cursor < start:
|
||||
sql_chunk = raw_sql[cursor:start].strip()
|
||||
if sql_chunk:
|
||||
spans.append(("sql", sql_chunk))
|
||||
jinja_chunk = raw_sql[start:end].strip()
|
||||
if jinja_chunk:
|
||||
spans.append(("jinja", jinja_chunk))
|
||||
cursor = end
|
||||
if cursor < len(raw_sql):
|
||||
remaining = raw_sql[cursor:].strip()
|
||||
if remaining:
|
||||
spans.append(("sql", remaining))
|
||||
else:
|
||||
spans.append(("sql", raw_sql.strip()))
|
||||
|
||||
return spans
|
||||
# #endregion detect_jinja_spans
|
||||
|
||||
|
||||
# #region extract_tables_from_jinja [C:2] [TYPE Function]
|
||||
# @BRIEF Phase 2: Extract "schema.table" references from Jinja string values.
|
||||
# Looks for patterns like "raw.sales" inside Jinja text.
|
||||
# @PRE jinja_text is a string from a Jinja span.
|
||||
# @POST Returns a set of lowercased schema.table strings.
|
||||
def extract_tables_from_jinja(jinja_text: str) -> set[str]:
|
||||
"""Extract ``schema.table`` references from within Jinja template blocks.
|
||||
|
||||
Looks for double-quoted or single-quoted string values that match
|
||||
the ``schema.table`` pattern, e.g. ``"raw.sales"`` or ``'raw.inventory'``.
|
||||
"""
|
||||
tables: set[str] = set()
|
||||
# Match string values: "schema.table" or 'schema.table'
|
||||
string_pattern = re.compile(
|
||||
r"""["']([a-zA-Z_][\w]*\.[a-zA-Z_][\w]*)["']"""
|
||||
)
|
||||
for m in string_pattern.finditer(jinja_text):
|
||||
tables.add(m.group(1).lower())
|
||||
return tables
|
||||
# #endregion extract_tables_from_jinja
|
||||
|
||||
|
||||
# #region is_string_literal [C:1] [TYPE Function]
|
||||
# @BRIEF Check if a sqlparse Token is a string literal (not a schema.table identifier).
|
||||
# @PRE token is a sqlparse Token.
|
||||
# @POST Returns True if the token is a string/Single/Literal.String.
|
||||
def is_string_literal(token: Token) -> bool:
|
||||
"""Return True if token is a string literal type in sqlparse."""
|
||||
return token.ttype is Literal.String.Single or token.ttype is Literal.String
|
||||
# #endregion is_string_literal
|
||||
|
||||
|
||||
# #region extract_tables_from_sql_span [C:2] [TYPE Function]
|
||||
# @BRIEF Phase 3: Extract schema.table references from SQL text using regex + sqlparse filtering.
|
||||
# Uses regex to find all schema.table candidates, then sqlparse to filter out
|
||||
# false positives inside string literals.
|
||||
# @PRE sql_text is a string from a SQL span (non-Jinja).
|
||||
# @POST Returns a set of lowercased schema.table strings.
|
||||
def extract_tables_from_sql_span(sql_text: str) -> set[str]:
|
||||
"""Extract ``schema.table`` references from plain SQL text.
|
||||
|
||||
Uses regex to find all ``schema.table`` candidates, then sqlparse
|
||||
to reject matches that fall inside string literals.
|
||||
"""
|
||||
tables: set[str] = set()
|
||||
raw_matches = _SCHEMA_TABLE_RE.findall(sql_text)
|
||||
if not raw_matches:
|
||||
return tables
|
||||
|
||||
# Use sqlparse to identify string literal positions
|
||||
parsed = sqlparse.parse(sql_text)
|
||||
string_literal_ranges: list[tuple[int, int]] = []
|
||||
|
||||
def walk_tokens(tokens: Iterable[Token], base_offset: int = 0) -> None:
|
||||
offset = base_offset
|
||||
for token in tokens:
|
||||
if isinstance(token, TokenList):
|
||||
walk_tokens(token.flatten(), offset)
|
||||
else:
|
||||
ttype = token.ttype
|
||||
val = token.value
|
||||
if is_string_literal(token):
|
||||
string_literal_ranges.append(
|
||||
(offset, offset + len(val))
|
||||
)
|
||||
offset += len(val)
|
||||
|
||||
for stmt in parsed:
|
||||
if stmt is None:
|
||||
continue
|
||||
walk_tokens(stmt.flatten(), base_offset=0)
|
||||
|
||||
def is_in_string(pos: int) -> bool:
|
||||
for s_start, s_end in string_literal_ranges:
|
||||
if s_start <= pos <= s_end:
|
||||
return True
|
||||
return False
|
||||
|
||||
# Re-scan with full match positions to filter
|
||||
for m in _SCHEMA_TABLE_RE.finditer(sql_text):
|
||||
if not is_in_string(m.start()):
|
||||
# Extract schema and table from match groups
|
||||
schema = m.group(2) or m.group(1) or ""
|
||||
table = m.group(3) or ""
|
||||
# Filter out column aliases (single-char schemas like "a.id", "o.id")
|
||||
if len(schema) < 2 and schema.isalpha() and schema.islower():
|
||||
continue
|
||||
if schema and table:
|
||||
tables.add(f"{schema.lower()}.{table.lower()}")
|
||||
|
||||
return tables
|
||||
# #endregion extract_tables_from_sql_span
|
||||
|
||||
|
||||
# #region extract_tables_from_sql [C:2] [TYPE Function]
|
||||
# @BRIEF Three-phase entry point: extract all schema.table references from raw SQL+Jinja text.
|
||||
# @PRE raw_sql is a string (possibly empty).
|
||||
# @POST Returns a set of lowercased fully-qualified table names (schema.table format).
|
||||
# @RELATION CALLS -> [detect_jinja_spans]
|
||||
# @RELATION CALLS -> [extract_tables_from_jinja]
|
||||
# @RELATION CALLS -> [extract_tables_from_sql_span]
|
||||
def extract_tables_from_sql(raw_sql: str) -> set[str]:
|
||||
"""Extract all ``schema.table`` references from raw SQL + Jinja text.
|
||||
|
||||
Three-phase approach per FR-003:
|
||||
1. Detect Jinja spans vs SQL spans
|
||||
2. In Jinja spans, extract ``schema.table`` from quoted string values
|
||||
3. In SQL spans, regex + sqlparse filter to reject string literal false positives
|
||||
|
||||
Returns a set of lowercased ``schema.table`` strings for case-insensitive matching.
|
||||
|
||||
Example:
|
||||
>>> extract_tables_from_sql("SELECT * FROM raw.sales JOIN raw.inventory ON ...")
|
||||
{'raw.sales', 'raw.inventory'}
|
||||
"""
|
||||
if not raw_sql or not raw_sql.strip():
|
||||
return set()
|
||||
|
||||
all_tables: set[str] = set()
|
||||
spans = detect_jinja_spans(raw_sql)
|
||||
|
||||
for span_type, text in spans:
|
||||
if span_type == "jinja":
|
||||
all_tables.update(extract_tables_from_jinja(text))
|
||||
else:
|
||||
all_tables.update(extract_tables_from_sql_span(text))
|
||||
|
||||
return all_tables
|
||||
# #endregion extract_tables_from_sql
|
||||
|
||||
# #endregion SqlTableExtractorModule
|
||||
@@ -55,7 +55,12 @@ class ValidationRunService:
|
||||
query = self.db.query(ValidationRecord)
|
||||
|
||||
if task_id:
|
||||
query = query.filter(ValidationRecord.task_id == task_id)
|
||||
# Match both spawned task_id (TaskRecord) and policy_id (ValidationPolicy.id)
|
||||
# because the frontend sends the validation policy UUID, but ValidationRecord.task_id
|
||||
# stores the spawned task UUID. Fall back to policy_id for the common case.
|
||||
query = query.filter(
|
||||
(ValidationRecord.task_id == task_id) | (ValidationRecord.policy_id == task_id)
|
||||
)
|
||||
if dashboard_id:
|
||||
query = query.filter(ValidationRecord.dashboard_id == dashboard_id)
|
||||
if environment_id:
|
||||
|
||||
Reference in New Issue
Block a user