feat: LLM Validation section — 9 API endpoints + 3 Svelte pages + Playwright optimization

Backend:
- /api/validation/ CRUD for validation tasks (policies)
- Run history with 6 filters + pagination
- Alembic migrations (provider_id, policy_id)
- 28 pytest tests

Frontend:
- /validation — task list page with status filters + CRUD
- /validation/[id] — task config (Config + History tabs)
- /validation/history — run history with metrics
- Sidebar nav entry under Operations
- i18n en/ru

Playwright (ScreenshotService):
- Removed 25s hardcoded sleeps → conditional waits
- CDP fallback to full_page screenshot
- Total timeout enforcement (120s)
- Debug screenshots → temp dir with cleanup

QA fixes:
- Debug screenshot accumulation in production (tempdir + rmtree)
- Frontend API payload field name mismatch
- History issues field reference fix
- delete_runs scoped by policy_id

Belief protocol:
- @RATIONALE/@REJECTED on 27 C4+ contracts

Closes: VALIDATION-REWORK-2026
This commit is contained in:
2026-05-21 20:29:36 +03:00
parent 36643024cf
commit 48cc02ea0b
22 changed files with 3726 additions and 372 deletions

View File

@@ -37,6 +37,7 @@ __all__ = [
"storage",
"tasks",
"translate",
"validation",
]
# #endregion Route_Group_Contracts

View File

@@ -0,0 +1,270 @@
# #region ValidationRoutes [C:3] [TYPE Module] [SEMANTICS fastapi, validation, api, task, run]
# @BRIEF API routes for validation task management and run history.
# @LAYER API
# @RELATION DEPENDS_ON -> [ValidationTaskService]
# @RELATION DEPENDS_ON -> [ValidationRunService]
# @RELATION DEPENDS_ON -> [TaskManager]
from datetime import date
from fastapi import APIRouter, Depends, HTTPException, Query, status
from sqlalchemy.orm import Session
from ...core.config_manager import ConfigManager
from ...core.database import get_db
from ...core.logger import logger
from ...core.task_manager import TaskManager
from ...dependencies import get_config_manager, get_current_user, get_task_manager, has_permission
from ...schemas.auth import User
from ...schemas.validation import (
TriggerRunResponse,
ValidationRunDetailResponse,
ValidationRunListResponse,
ValidationTaskCreate,
ValidationTaskListResponse,
ValidationTaskResponse,
ValidationTaskUpdate,
)
from ...services.validation_run_service import ValidationRunService
from ...services.validation_service import ValidationTaskService
# #region router [C:1] [TYPE Global]
router = APIRouter(tags=["Validation"])
# #endregion router
# #region _get_task_service [C:1] [TYPE Function]
def _get_task_service(
db: Session = Depends(get_db),
config_manager: ConfigManager = Depends(get_config_manager),
current_user: User = Depends(get_current_user),
) -> ValidationTaskService:
return ValidationTaskService(db=db, config_manager=config_manager, username=current_user.username)
# #endregion _get_task_service
# #region _get_run_service [C:1] [TYPE Function]
def _get_run_service(
db: Session = Depends(get_db),
) -> ValidationRunService:
return ValidationRunService(db=db)
# #endregion _get_run_service
# ============================================================
# Tasks CRUD
# ============================================================
# #region list_tasks [C:4] [TYPE Function]
# @RATIONALE Route handler delegates to service with permission check — keeps API layer thin and testable; auth gate happens before any business logic.
# @REJECTED Business logic in route handler rejected — would duplicate validation logic and make permission checks harder to audit.
@router.get("/tasks", response_model=ValidationTaskListResponse)
async def list_tasks(
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
is_active: bool | None = Query(None),
environment_id: str | None = Query(None),
current_user: User = Depends(get_current_user),
_=Depends(has_permission("validation.task", "VIEW")),
service: ValidationTaskService = Depends(_get_task_service),
):
"""List all validation tasks with pagination and optional filters."""
logger.reason(f"list_tasks — User: {current_user.username}", extra={"src": "validation_routes"})
try:
total, items = service.list_tasks(is_active=is_active, environment_id=environment_id, page=page, page_size=page_size)
return ValidationTaskListResponse(items=items, total=total, page=page, page_size=page_size)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# #endregion list_tasks
# #region create_task [C:4] [TYPE Function]
# @RATIONALE Returns 201 with created resource — follows REST conventions for POST resource creation; signals to clients that a new entity was created.
# @REJECTED Returning 200 with wrapped response rejected — non-standard for POST create; 201 Created signals the correct semantics to HTTP clients and code generators.
@router.post("/tasks", response_model=ValidationTaskResponse, status_code=status.HTTP_201_CREATED)
async def create_task(
payload: ValidationTaskCreate,
current_user: User = Depends(get_current_user),
_=Depends(has_permission("validation.task", "CREATE")),
service: ValidationTaskService = Depends(_get_task_service),
):
"""Create a new validation task with provider and environment validation."""
logger.reason(f"create_task — User: {current_user.username}, name: {payload.name}", extra={"src": "validation_routes"})
try:
return service.create_task(payload)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))
# #endregion create_task
# #region get_task [C:4] [TYPE Function]
# @RATIONALE Joins task detail with recent runs in a single endpoint — reduces frontend API calls from 2 to 1, improving page load performance.
# @REJECTED Separate /tasks/{id}/runs endpoint for recent runs rejected — would require frontend to make 2 sequential API calls, doubling load time on the config page.
@router.get("/tasks/{task_id}", response_model=dict)
async def get_task(
task_id: str,
current_user: User = Depends(get_current_user),
_=Depends(has_permission("validation.task", "VIEW")),
task_service: ValidationTaskService = Depends(_get_task_service),
run_service: ValidationRunService = Depends(_get_run_service),
):
"""Get a validation task with its recent runs."""
logger.reason(f"get_task — Task: {task_id}, User: {current_user.username}", extra={"src": "validation_routes"})
try:
task = task_service.get_task(task_id)
from ...schemas.validation import ValidationRunResponse
# Find recent runs via run_service using dashboard matching
recent = run_service.list_runs(
dashboard_id=task.dashboard_ids[0] if task.dashboard_ids else None,
environment_id=task.environment_id,
page=1,
page_size=10,
)
return {
"task": task.model_dump(),
"recent_runs": [r.model_dump() for r in recent[1]],
}
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# #endregion get_task
# #region update_task [C:4] [TYPE Function]
# @RATIONALE PUT semantics for full resource update — consistent with existing API patterns; Pydantic's exclude_unset handles partial updates while keeping PUT as the endpoint verb.
# @REJECTED PATCH semantics rejected — using PUT with exclude_unset provides the same partial-update flexibility without introducing a second HTTP method for the same resource.
@router.put("/tasks/{task_id}", response_model=ValidationTaskResponse)
async def update_task(
task_id: str,
payload: ValidationTaskUpdate,
current_user: User = Depends(get_current_user),
_=Depends(has_permission("validation.task", "EDIT")),
service: ValidationTaskService = Depends(_get_task_service),
):
"""Update a validation task."""
logger.reason(f"update_task — Task: {task_id}, User: {current_user.username}", extra={"src": "validation_routes"})
try:
return service.update_task(task_id, payload)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# #endregion update_task
# #region delete_task [C:4] [TYPE Function]
# @RATIONALE DELETE returns 204 No Content per REST convention — minimizes response payload for delete operations; supports optional delete_runs query param for cascade control.
# @REJECTED Returning deleted resource in response body rejected — unnecessary data transfer; 204 with no body is the REST standard for delete operations.
@router.delete("/tasks/{task_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_task(
task_id: str,
delete_runs: bool = Query(False, description="Also delete associated validation runs"),
current_user: User = Depends(get_current_user),
_=Depends(has_permission("validation.task", "DELETE")),
service: ValidationTaskService = Depends(_get_task_service),
):
"""Delete a validation task."""
logger.reason(f"delete_task — Task: {task_id}, User: {current_user.username}", extra={"src": "validation_routes"})
try:
service.delete_task(task_id, delete_runs=delete_runs)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# #endregion delete_task
# ============================================================
# Run — trigger immediate validation
# ============================================================
# #region trigger_run [C:4] [TYPE Function]
# @RATIONALE Returns spawned task ID so frontend can poll for completion — enables progress tracking without WebSocket. Async spawn is essential since LLM validation can take 30+ seconds.
# @REJECTED Blocking until run completes rejected — LLM-based validation is slow (30-120s); synchronous wait would timeout HTTP requests and create terrible UX.
@router.post("/tasks/{task_id}/run", response_model=TriggerRunResponse)
async def trigger_run(
task_id: str,
current_user: User = Depends(get_current_user),
_=Depends(has_permission("validation.task", "EXECUTE")),
service: ValidationTaskService = Depends(_get_task_service),
task_manager: TaskManager = Depends(get_task_manager),
):
"""Trigger immediate validation for a task."""
logger.reason(f"trigger_run — Task: {task_id}, User: {current_user.username}", extra={"src": "validation_routes"})
try:
spawned_task_id = await service.trigger_run(task_id, task_manager)
return TriggerRunResponse(task_id=task_id, spawned_task_id=spawned_task_id, status="PENDING", message="Validation task spawned")
except ValueError as e:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))
# #endregion trigger_run
# ============================================================
# Runs — history listing and detail
# ============================================================
# #region list_runs [C:4] [TYPE Function]
# @RATIONALE Exposes 6 query parameters for cross-task run filtering — matches the run service's filter capabilities exactly, enabling flexible history exploration.
# @REJECTED Fixed filter set rejected — would limit the history page's debugging use cases; users need to filter by task, status, dashboard, environment, and date range.
@router.get("/runs", response_model=ValidationRunListResponse)
async def list_runs(
task_id: str | None = Query(None, description="Filter by spawned task ID"),
dashboard_id: str | None = Query(None, description="Filter by dashboard ID"),
environment_id: str | None = Query(None, description="Filter by environment ID"),
status: str | None = Query(None, description="Filter by status: PASS, WARN, FAIL, UNKNOWN"),
date_from: date | None = Query(None, description="Start date (ISO format)"),
date_to: date | None = Query(None, description="End date (ISO format)"),
page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100),
current_user: User = Depends(get_current_user),
_=Depends(has_permission("validation.run", "VIEW")),
service: ValidationRunService = Depends(_get_run_service),
):
"""List validation runs with cross-task filtering and pagination."""
logger.reason(f"list_runs — User: {current_user.username}", extra={"src": "validation_routes"})
try:
total, items = service.list_runs(
task_id=task_id, dashboard_id=dashboard_id, environment_id=environment_id,
status=status, date_from=date_from, date_to=date_to, page=page, page_size=page_size,
)
return ValidationRunListResponse(items=items, total=total, page=page, page_size=page_size)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# #endregion list_runs
# #region get_run_detail [C:4] [TYPE Function]
# @RATIONALE Returns full run detail including issues list and raw response for the report view — single endpoint serves the complete detail page without multiple fetches.
# @REJECTED Paginated issues endpoint rejected — issues per run are typically < 20 items; pagination overhead is unnecessary for such small collections.
@router.get("/runs/{run_id}", response_model=ValidationRunDetailResponse)
async def get_run_detail(
run_id: str,
current_user: User = Depends(get_current_user),
_=Depends(has_permission("validation.run", "VIEW")),
service: ValidationRunService = Depends(_get_run_service),
):
"""Get detailed validation run info including issues and raw response."""
logger.reason(f"get_run_detail — Run: {run_id}, User: {current_user.username}", extra={"src": "validation_routes"})
try:
return service.get_run_detail(run_id)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# #endregion get_run_detail
# #region delete_run [C:3] [TYPE Function]
# @RATIONALE Returns 404 when run not found — explicit error handling rather than silent success; helps frontend distinguish between successful delete and missing resource.
# @REJECTED Silent success on missing run rejected — would mask bugs in the UI that reference already-deleted runs; explicit 404 helps debugging.
@router.delete("/runs/{run_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_run(
run_id: str,
current_user: User = Depends(get_current_user),
_=Depends(has_permission("validation.run", "DELETE")),
service: ValidationRunService = Depends(_get_run_service),
):
"""Delete a validation run record."""
logger.reason(f"delete_run — Run: {run_id}, User: {current_user.username}", extra={"src": "validation_routes"})
if not service.delete_run(run_id):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Run not found")
# #endregion delete_run
# #endregion ValidationRoutes

View File

@@ -0,0 +1,550 @@
# #region ValidationApiTests [C:3] [TYPE Module] [SEMANTICS validation, api, tests, pagination, crud]
# @BRIEF Unit tests for validation task CRUD and run history API endpoints.
# @LAYER: API
# @RELATION BINDS_TO -> [ValidationRoutes]
# @RELATION DEPENDS_ON -> [EXT:FastAPI:TestClient]
# @TEST_CONTRACT: [ValidationTaskCreate|Update|RunRequest] -> [ValidationTaskResponse|RunResponse|TriggerRunResponse]
# @TEST_SCENARIO: list_tasks -> 200 with paginated tasks; filter by is_active/environment_id
# @TEST_SCENARIO: create_task -> 201 with valid payload; 422 with invalid provider/environment
# @TEST_SCENARIO: get_task -> 200 with task+recent_runs; 404 for nonexistent
# @TEST_SCENARIO: update_task -> 200 with updated fields; 404 for nonexistent
# @TEST_SCENARIO: delete_task -> 204 on success; 404 for nonexistent
# @TEST_SCENARIO: trigger_run -> 200 with spawned_task_id; 422 for invalid task
# @TEST_SCENARIO: list_runs -> 200 with 6 filters + pagination
# @TEST_SCENARIO: get_run_detail -> 200 with issues/raw_response; 404 for nonexistent
# @TEST_SCENARIO: delete_run -> 204 on success; 404 for nonexistent
# @TEST_EDGE: missing_field -> 422 when required field omitted from create payload
# @TEST_EDGE: invalid_type -> 422 when provider_id is not multimodal
# @TEST_EDGE: external_fail -> 404/422 when task/run does not exist
# @TEST_EDGE: delete_task_with_runs -> 204 with delete_runs=true flag
# @INVARIANT: Every endpoint requires authentication via has_permission
# @INVARIANT: All list endpoints support pagination (page/page_size) with defaults
# @INVARIANT: provider_id must reference a multimodal LLM provider for creation
# Set required env vars before ANY app imports — crash-early guard for AuthConfig()
import os
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing")
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:test_auth")
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:test_main")
os.environ.setdefault("DEV_MODE", "true")
from datetime import datetime, timezone
import pytest
from unittest.mock import AsyncMock, MagicMock
from fastapi.testclient import TestClient
from src.app import app
from src.dependencies import (
get_current_user,
get_config_manager,
get_task_manager,
)
from src.api.routes.validation import _get_task_service, _get_run_service
from src.schemas.validation import (
ValidationTaskListResponse,
ValidationTaskResponse,
ValidationRunListResponse,
ValidationRunDetailResponse,
TriggerRunResponse,
)
from src.models.auth import User
# ── Shared mocks ───────────────────────────────────────────────────────
mock_user = MagicMock(spec=User)
mock_user.username = "testuser"
mock_user.roles = []
admin_role = MagicMock()
admin_role.name = "Admin"
admin_role.permissions = []
mock_user.roles.append(admin_role)
def _make_now():
"""Return a timezone-aware UTC datetime for Pydantic v2."""
return datetime.now(timezone.utc)
@pytest.fixture(autouse=True)
def mock_all_deps():
"""Override service-level FastAPI dependencies with MagicMock stubs.
Permission checking (has_permission) is NOT overridden — mock_user has
Admin role which passes all permission gates implicitly.
Fixture resets mock_user.roles before each test to prevent cross-test pollution.
"""
# Reset user state — Admin role passes all permission gates
mock_user.roles = [admin_role]
mock_task_service = MagicMock()
mock_run_service = MagicMock()
mock_task_manager = MagicMock()
app.dependency_overrides[_get_task_service] = lambda: mock_task_service
app.dependency_overrides[_get_run_service] = lambda: mock_run_service
app.dependency_overrides[get_config_manager] = lambda: MagicMock()
app.dependency_overrides[get_task_manager] = lambda: mock_task_manager
app.dependency_overrides[get_current_user] = lambda: mock_user
yield {
"task_service": mock_task_service,
"run_service": mock_run_service,
"task_manager": mock_task_manager,
}
app.dependency_overrides.clear()
client = TestClient(app)
# ── Fixtures ────────────────────────────────────────────────────────────
def _make_task_response(**overrides):
"""Build a ValidationTaskResponse for mock returns."""
return ValidationTaskResponse(
id=overrides.get("id", "task-1"),
name=overrides.get("name", "Test Task"),
environment_id=overrides.get("environment_id", "env-1"),
is_active=overrides.get("is_active", True),
dashboard_ids=overrides.get("dashboard_ids", ["dash-1"]),
provider_id=overrides.get("provider_id", "provider-1"),
schedule_days=None,
window_start=None,
window_end=None,
notify_owners=True,
custom_channels=None,
alert_condition="FAIL_ONLY",
created_at=overrides.get("created_at", _make_now()),
updated_at=None,
last_run_status=None,
last_run_at=None,
)
def _make_run_response(**overrides):
"""Build a dict representation of a ValidationRunResponse."""
return {
"id": overrides.get("id", "run-1"),
"task_id": overrides.get("task_id", "task-1"),
"dashboard_id": overrides.get("dashboard_id", "dash-1"),
"environment_id": overrides.get("environment_id", "env-1"),
"status": overrides.get("status", "PASS"),
"summary": overrides.get("summary", ""),
"timestamp": overrides.get("timestamp", _make_now()),
"screenshot_path": None,
"task_name": overrides.get("task_name", "Test Task"),
}
# ── Task CRUD Tests ────────────────────────────────────────────────────
# #region test_list_tasks_success [C:2] [TYPE Function]
# @BRIEF GET /validation/tasks returns paginated task list with filters.
def test_list_tasks_success(mock_all_deps):
task = _make_task_response()
mock_all_deps["task_service"].list_tasks.return_value = (1, [task])
response = client.get("/api/validation/tasks")
assert response.status_code == 200
data = response.json()
assert data["items"][0]["name"] == "Test Task"
assert data["total"] == 1
assert data["page"] == 1
assert data["page_size"] == 20
ValidationTaskListResponse(**data)
# environment_id filter
response = client.get("/api/validation/tasks?environment_id=env-1")
assert response.status_code == 200
mock_all_deps["task_service"].list_tasks.assert_called_with(
is_active=None, environment_id="env-1", page=1, page_size=20
)
# #endregion test_list_tasks_success
# #region test_list_tasks_pagination [C:2] [TYPE Function]
# @BRIEF Pagination query params are validated: page >= 1, page_size 1-100.
def test_list_tasks_pagination(mock_all_deps):
response = client.get("/api/validation/tasks?page=0")
assert response.status_code == 422
response = client.get("/api/validation/tasks?page_size=101")
assert response.status_code == 422
response = client.get("/api/validation/tasks?page_size=0")
assert response.status_code == 422
# #endregion test_list_tasks_pagination
# #region test_list_tasks_service_error [C:2] [TYPE Function]
# @BRIEF Service ValueError becomes 400 on list_tasks.
def test_list_tasks_service_error(mock_all_deps):
mock_all_deps["task_service"].list_tasks.side_effect = ValueError("Bad filter")
response = client.get("/api/validation/tasks?environment_id=nonexistent")
assert response.status_code == 400
assert "Bad filter" in response.json()["detail"]
# #endregion test_list_tasks_service_error
# #region test_create_task_success [C:2] [TYPE Function]
# @BRIEF POST /validation/tasks with valid payload returns 201.
def test_create_task_success(mock_all_deps):
mock_all_deps["task_service"].create_task.return_value = _make_task_response()
payload = {
"name": "My Validation Task",
"environment_id": "env-1",
"dashboard_ids": ["dash-1"],
"provider_id": "provider-1",
"notify_owners": True,
"alert_condition": "FAIL_ONLY",
}
response = client.post("/api/validation/tasks", json=payload)
assert response.status_code == 201
data = response.json()
assert data["name"] == "Test Task"
assert data["id"] == "task-1"
ValidationTaskResponse(**data)
mock_all_deps["task_service"].create_task.assert_called_once()
# #endregion test_create_task_success
# #region test_create_task_missing_fields [C:2] [TYPE Function]
# @BRIEF POST with missing required fields returns 422.
def test_create_task_missing_fields(mock_all_deps):
# Empty payload
response = client.post("/api/validation/tasks", json={})
assert response.status_code == 422
errors = response.json()
assert "detail" in errors
# Missing provider_id
response = client.post("/api/validation/tasks", json={
"name": "Test", "environment_id": "env-1", "dashboard_ids": ["dash-1"],
})
assert response.status_code == 422
# Missing dashboard_ids
response = client.post("/api/validation/tasks", json={
"name": "Test", "environment_id": "env-1", "provider_id": "provider-1",
})
assert response.status_code == 422
# #endregion test_create_task_missing_fields
# #region test_create_task_invalid_provider [C:2] [TYPE Function]
# @BRIEF POST with non-multimodal provider returns 422.
def test_create_task_invalid_provider(mock_all_deps):
mock_all_deps["task_service"].create_task.side_effect = ValueError(
"LLM provider 'GPT-3.5' is not multimodal"
)
payload = {
"name": "Test", "environment_id": "env-1",
"dashboard_ids": ["dash-1"], "provider_id": "provider-2",
}
response = client.post("/api/validation/tasks", json=payload)
assert response.status_code == 422
assert "not multimodal" in response.json()["detail"]
# #endregion test_create_task_invalid_provider
# #region test_get_task_success [C:2] [TYPE Function]
# @BRIEF GET /validation/tasks/{id} returns task with recent_runs.
def test_get_task_success(mock_all_deps):
mock_all_deps["task_service"].get_task.return_value = _make_task_response()
mock_all_deps["run_service"].list_runs.return_value = (0, [])
response = client.get("/api/validation/tasks/task-1")
assert response.status_code == 200
data = response.json()
assert "task" in data
assert "recent_runs" in data
assert data["task"]["name"] == "Test Task"
mock_all_deps["task_service"].get_task.assert_called_with("task-1")
# #endregion test_get_task_success
# #region test_get_task_not_found [C:2] [TYPE Function]
# @BRIEF GET for nonexistent task returns 404.
def test_get_task_not_found(mock_all_deps):
mock_all_deps["task_service"].get_task.side_effect = ValueError("not found")
response = client.get("/api/validation/tasks/nonexistent")
assert response.status_code == 404
# #endregion test_get_task_not_found
# #region test_update_task_success [C:2] [TYPE Function]
# @BRIEF PUT /validation/tasks/{id} updates task and returns updated object.
def test_update_task_success(mock_all_deps):
updated = _make_task_response(name="Updated Name", is_active=False)
mock_all_deps["task_service"].update_task.return_value = updated
payload = {"name": "Updated Name", "is_active": False}
response = client.put("/api/validation/tasks/task-1", json=payload)
assert response.status_code == 200
data = response.json()
assert data["name"] == "Updated Name"
assert data["is_active"] is False
ValidationTaskResponse(**data)
mock_all_deps["task_service"].update_task.assert_called_once()
# #endregion test_update_task_success
# #region test_update_task_not_found [C:2] [TYPE Function]
# @BRIEF PUT for nonexistent task returns 404.
def test_update_task_not_found(mock_all_deps):
mock_all_deps["task_service"].update_task.side_effect = ValueError("not found")
payload = {"name": "Nope"}
response = client.put("/api/validation/tasks/nonexistent", json=payload)
assert response.status_code == 404
# #endregion test_update_task_not_found
# #region test_delete_task_success [C:2] [TYPE Function]
# @BRIEF DELETE /validation/tasks/{id} returns 204.
def test_delete_task_success(mock_all_deps):
mock_all_deps["task_service"].delete_task.return_value = None
response = client.delete("/api/validation/tasks/task-1")
assert response.status_code == 204
mock_all_deps["task_service"].delete_task.assert_called_with("task-1", delete_runs=False)
# #endregion test_delete_task_success
# #region test_delete_task_not_found [C:2] [TYPE Function]
# @BRIEF DELETE for nonexistent task returns 404.
def test_delete_task_not_found(mock_all_deps):
mock_all_deps["task_service"].delete_task.side_effect = ValueError("not found")
response = client.delete("/api/validation/tasks/nonexistent")
assert response.status_code == 404
# #endregion test_delete_task_not_found
# #region test_delete_task_with_runs_flag [C:2] [TYPE Function]
# @BRIEF DELETE with delete_runs=true passes query param to service.
def test_delete_task_with_runs_flag(mock_all_deps):
mock_all_deps["task_service"].delete_task.return_value = None
response = client.delete("/api/validation/tasks/task-1?delete_runs=true")
assert response.status_code == 204
mock_all_deps["task_service"].delete_task.assert_called_with("task-1", delete_runs=True)
# #endregion test_delete_task_with_runs_flag
# #region test_trigger_run_success [C:2] [TYPE Function]
# @BRIEF POST /validation/tasks/{id}/run spawns a validation task.
def test_trigger_run_success(mock_all_deps):
mock_all_deps["task_service"].trigger_run = AsyncMock(return_value="spawned-1")
mock_all_deps["task_manager"].create_task = AsyncMock()
response = client.post("/api/validation/tasks/task-1/run")
assert response.status_code == 200
data = response.json()
assert data["task_id"] == "task-1"
assert data["spawned_task_id"] == "spawned-1"
assert data["status"] == "PENDING"
TriggerRunResponse(**data)
# #endregion test_trigger_run_success
# #region test_trigger_run_not_found [C:2] [TYPE Function]
# @BRIEF POST for nonexistent task returns 422.
def test_trigger_run_not_found(mock_all_deps):
mock_all_deps["task_service"].trigger_run = AsyncMock(
side_effect=ValueError("Validation task 'nonexistent' not found")
)
response = client.post("/api/validation/tasks/nonexistent/run")
assert response.status_code == 422
assert "not found" in response.json()["detail"]
# #endregion test_trigger_run_not_found
# #region test_trigger_run_no_dashboards [C:2] [TYPE Function]
# @BRIEF Trigger fails with 422 when task has no dashboard IDs.
def test_trigger_run_no_dashboards(mock_all_deps):
mock_all_deps["task_service"].trigger_run = AsyncMock(
side_effect=ValueError("has no dashboard IDs configured")
)
response = client.post("/api/validation/tasks/task-1/run")
assert response.status_code == 422
assert "no dashboard IDs" in response.json()["detail"]
# #endregion test_trigger_run_no_dashboards
# ── Run History Tests ──────────────────────────────────────────────────
# #region test_list_runs_success [C:2] [TYPE Function]
# @BRIEF GET /validation/runs returns paginated run list.
def test_list_runs_success(mock_all_deps):
run = _make_run_response()
mock_all_deps["run_service"].list_runs.return_value = (1, [run])
response = client.get("/api/validation/runs")
assert response.status_code == 200
data = response.json()
assert data["items"][0]["id"] == "run-1"
assert data["total"] == 1
ValidationRunListResponse(**data)
# #endregion test_list_runs_success
# #region test_list_runs_filters [C:2] [TYPE Function]
# @BRIEF All 6 filter query params are accepted and forwarded to service.
def test_list_runs_filters(mock_all_deps):
mock_all_deps["run_service"].list_runs.return_value = (0, [])
for query in [
"?task_id=task-1",
"?dashboard_id=dash-1",
"?environment_id=env-1",
"?status=PASS",
"?date_from=2026-01-01",
"?date_to=2026-12-31",
"?status=PASS&dashboard_id=dash-1&date_from=2026-01-01",
]:
response = client.get(f"/api/validation/runs{query}")
assert response.status_code == 200, f"Filter query failed: {query}"
# #endregion test_list_runs_filters
# #region test_list_runs_pagination [C:2] [TYPE Function]
# @BRIEF Pagination boundary checks on runs list.
def test_list_runs_pagination(mock_all_deps):
response = client.get("/api/validation/runs?page=0")
assert response.status_code == 422
response = client.get("/api/validation/runs?page_size=101")
assert response.status_code == 422
# #endregion test_list_runs_pagination
# #region test_list_runs_invalid_status [C:2] [TYPE Function]
# @BRIEF Status is a free string; any value passes through to service.
def test_list_runs_invalid_status(mock_all_deps):
mock_all_deps["run_service"].list_runs.return_value = (0, [])
response = client.get("/api/validation/runs?status=INVALID_STATUS_XYZ")
assert response.status_code == 200
call_kwargs = mock_all_deps["run_service"].list_runs.call_args[1]
assert call_kwargs.get("status") == "INVALID_STATUS_XYZ"
# #endregion test_list_runs_invalid_status
# #region test_get_run_detail_success [C:2] [TYPE Function]
# @BRIEF GET /validation/runs/{id} returns full detail with issues and raw_response.
def test_get_run_detail_success(mock_all_deps):
mock_all_deps["run_service"].get_run_detail.return_value = ValidationRunDetailResponse(
id="run-1",
task_id="task-1",
dashboard_id="dash-1",
environment_id="env-1",
status="FAIL",
summary="Charts failed to render",
timestamp=_make_now(),
screenshot_path="/tmp/screenshots/run-1.png",
issues=[{"severity": "HIGH", "message": "Chart timeout"}],
raw_response='{"status": "error"}',
task_name="Test Task",
)
response = client.get("/api/validation/runs/run-1")
assert response.status_code == 200
data = response.json()
assert data["status"] == "FAIL"
assert len(data["issues"]) == 1
assert data["raw_response"] is not None
ValidationRunDetailResponse(**data)
# #endregion test_get_run_detail_success
# #region test_get_run_detail_not_found [C:2] [TYPE Function]
# @BRIEF GET for nonexistent run returns 404.
def test_get_run_detail_not_found(mock_all_deps):
mock_all_deps["run_service"].get_run_detail.side_effect = ValueError("not found")
response = client.get("/api/validation/runs/nonexistent")
assert response.status_code == 404
# #endregion test_get_run_detail_not_found
# #region test_delete_run_success [C:2] [TYPE Function]
# @BRIEF DELETE /validation/runs/{id} returns 204.
def test_delete_run_success(mock_all_deps):
mock_all_deps["run_service"].delete_run.return_value = True
response = client.delete("/api/validation/runs/run-1")
assert response.status_code == 204
mock_all_deps["run_service"].delete_run.assert_called_with("run-1")
# #endregion test_delete_run_success
# #region test_delete_run_not_found [C:2] [TYPE Function]
# @BRIEF DELETE for nonexistent run returns 404.
def test_delete_run_not_found(mock_all_deps):
mock_all_deps["run_service"].delete_run.return_value = False
response = client.delete("/api/validation/runs/nonexistent")
assert response.status_code == 404
# #endregion test_delete_run_not_found
# ── Auth / Permission Tests ────────────────────────────────────────────
# #region test_endpoints_require_authentication [C:2] [TYPE Function]
# @BRIEF Without current_user dependency, all 9 endpoints return 401.
def test_endpoints_require_authentication(mock_all_deps):
app.dependency_overrides.pop(get_current_user, None)
endpoints = [
("GET", "/api/validation/tasks"),
("POST", "/api/validation/tasks"),
("GET", "/api/validation/tasks/task-1"),
("PUT", "/api/validation/tasks/task-1"),
("DELETE", "/api/validation/tasks/task-1"),
("POST", "/api/validation/tasks/task-1/run"),
("GET", "/api/validation/runs"),
("GET", "/api/validation/runs/run-1"),
("DELETE", "/api/validation/runs/run-1"),
]
for method, path in endpoints:
response = client.request(method, path)
assert response.status_code == 401, f"{method} {path} expected 401 got {response.status_code}"
# #endregion test_endpoints_require_authentication
# #region test_permission_denied_non_admin [C:2] [TYPE Function]
# @BRIEF A non-admin user without validation.* permissions receives 403.
def test_permission_denied_non_admin(mock_all_deps):
"""Remove Admin role so has_permission raises 403."""
mock_user.roles = [] # No Admin, no permissions
response = client.get("/api/validation/tasks")
assert response.status_code == 403
# #endregion test_permission_denied_non_admin
# #region test_permission_denied_all_actions [C:2] [TYPE Function]
# @BRIEF Each action (VIEW, CREATE, EDIT, DELETE, EXECUTE) denied for non-admin.
def test_permission_denied_all_actions(mock_all_deps):
mock_user.roles = []
action_endpoints = [
("GET", "/api/validation/tasks"),
("POST", "/api/validation/tasks"),
("GET", "/api/validation/tasks/task-1"),
("PUT", "/api/validation/tasks/task-1"),
("DELETE", "/api/validation/tasks/task-1"),
("POST", "/api/validation/tasks/task-1/run"),
("GET", "/api/validation/runs"),
("GET", "/api/validation/runs/run-1"),
("DELETE", "/api/validation/runs/run-1"),
]
for method, path in action_endpoints:
response = client.request(method, path)
assert response.status_code == 403, f"{method} {path} expected 403 got {response.status_code}"
# #endregion test_permission_denied_all_actions
# #region test_delete_task_runs_default_false [C:2] [TYPE Function]
# @BRIEF delete_runs query param defaults to False.
def test_delete_task_runs_default_false(mock_all_deps):
mock_all_deps["task_service"].delete_task.return_value = None
response = client.delete("/api/validation/tasks/task-1")
assert response.status_code == 204
mock_all_deps["task_service"].delete_task.assert_called_with("task-1", delete_runs=False)
# #endregion test_delete_task_runs_default_false
# #endregion ValidationApiTests

View File

@@ -44,6 +44,7 @@ from .api.routes import (
storage,
tasks,
translate,
validation,
)
from .core.auth.security import get_password_hash
from .core.cot_logger import seed_trace_id
@@ -217,6 +218,7 @@ app.add_middleware(TraceContextMiddleware)
# @RELATION DEPENDS_ON -> [ReportsRouter]
# @RELATION DEPENDS_ON -> [LlmRoutes]
# @RELATION DEPENDS_ON -> [CleanReleaseV2Api]
# @RELATION DEPENDS_ON -> [ValidationRoutes]
# Include API routes
app.include_router(auth.router)
app.include_router(admin.router)
@@ -239,6 +241,7 @@ app.include_router(profile.router)
app.include_router(dataset_review.router)
app.include_router(health.router)
app.include_router(translate.router)
app.include_router(validation.router, prefix="/api/validation", tags=["Validation"])
# #endregion API_Routes
# #region api.include_routers [C:1] [TYPE Action] [SEMANTICS routes, registration, api]
# @BRIEF Registers all API routers with the FastAPI application.

View File

@@ -16,6 +16,7 @@ def generate_uuid():
# #region ValidationPolicy [TYPE Class]
# @BRIEF Defines a scheduled rule for validating a group of dashboards within an execution window.
# @RELATION DEPENDS_ON -> [LLMProvider]
class ValidationPolicy(Base):
__tablename__ = "validation_policies"
@@ -24,6 +25,7 @@ class ValidationPolicy(Base):
environment_id = Column(String, nullable=False)
is_active = Column(Boolean, default=True)
dashboard_ids = Column(JSON, nullable=False) # Array of dashboard IDs
provider_id = Column(String, nullable=True) # Reference to LLMProvider.id
schedule_days = Column(JSON, nullable=False) # Array of integers (0-6)
window_start = Column(Time, nullable=False)
window_end = Column(Time, nullable=False)
@@ -57,6 +59,7 @@ class ValidationRecord(Base):
id = Column(String, primary_key=True, default=generate_uuid)
task_id = Column(String, nullable=True, index=True) # Reference to TaskRecord
policy_id = Column(String, nullable=True, index=True) # Reference to ValidationPolicy.id
dashboard_id = Column(String, nullable=False, index=True)
environment_id = Column(String, nullable=True, index=True)
timestamp = Column(DateTime, default=datetime.utcnow)

View File

@@ -286,6 +286,7 @@ class DashboardValidationPlugin(PluginBase):
db_record = ValidationRecord(
task_id=context.task_id if context else None,
policy_id=params.get("policy_id"),
dashboard_id=validation_result.dashboard_id,
environment_id=env_id,
status=validation_result.status.value,

View File

@@ -13,6 +13,8 @@ import base64
import io
import json
import os
import shutil
import tempfile
from typing import Any
from urllib.parse import urlsplit
@@ -282,406 +284,508 @@ class ScreenshotService:
return await page.goto(url, wait_until=fallback_wait_until, timeout=timeout)
# endregion ScreenshotService._goto_resilient
# region ScreenshotService.capture_dashboard [TYPE Function]
# region ScreenshotService._wait_for_charts_stabilized [TYPE Function] [C:2]
# @BRIEF Wait until chart elements have non-zero dimensions, with polling.
# @PRE page is a valid Playwright page.
# @POST Waits for chart stabilization or raises on timeout (handled internally).
# @RATIONALE Polls for actual chart rendering dimensions rather than using a fixed delay — charts may load at different speeds depending on dashboard complexity and network conditions.
# @REJECTED Fixed sleep-based wait rejected — would either waste time (too long) or produce blank screenshots (too short); polling for actual canvas/svg dimensions is more reliable.
async def _wait_for_charts_stabilized(self, page, timeout_ms: int = 15000):
"""Wait until chart elements have non-zero dimensions, with polling."""
# Short initial delay for rendering pipeline to start
await asyncio.sleep(0.5)
try:
await page.wait_for_function("""() => {
const charts = document.querySelectorAll('.chart-container canvas, .slice_container svg, .grid-content canvas');
if (charts.length === 0) return true;
return Array.from(charts).some(c => {
if (c.tagName === 'CANVAS') return c.width > 10 && c.height > 10;
if (c.tagName === 'svg') {
const bbox = c.getBoundingClientRect();
return bbox.width > 10 && bbox.height > 10;
}
return false;
});
}""", timeout=timeout_ms)
except Exception:
logger.warning("[ScreenshotService] Chart stabilization wait timed out, proceeding anyway")
# endregion ScreenshotService._wait_for_charts_stabilized
# region ScreenshotService._wait_for_resize_rendered [TYPE Function] [C:2]
# @BRIEF Wait for charts to re-render after viewport resize.
# @PRE page is a valid Playwright page; chart_count_before contains pre-resize element counts.
# @POST Waits for chart content to return or timeout.
# @RATIONALE After viewport resize, Superset triggers lazy chart re-rendering — this function polls for chart elements to reappear before taking the screenshot.
# @REJECTED Single fixed wait after resize rejected — some dashboards re-render instantly while others take seconds; fixed wait is brittle across dashboard types.
async def _wait_for_resize_rendered(self, page, chart_count_before: dict, timeout_ms: int = 10000):
"""Wait for charts to re-render after viewport resize, with polling."""
try:
await page.wait_for_function("""(preCounts) => {
const currentCharts = document.querySelectorAll('.chart-container, .slice_container').length;
const currentCanvases = document.querySelectorAll('canvas').length;
const currentSvgs = document.querySelectorAll('.chart-container svg, .slice_container svg').length;
// At least one chart element must be present
return currentCharts > 0 && (currentCanvases > 0 || currentSvgs > 0);
}""", arg=chart_count_before, timeout=timeout_ms)
except Exception:
logger.warning("[ScreenshotService] Re-render wait timed out after viewport resize, proceeding anyway")
# endregion ScreenshotService._wait_for_resize_rendered
# region ScreenshotService._save_debug_screenshot [TYPE Function] [C:1]
# @BRIEF Save a debug screenshot to a temp directory for diagnostic purposes.
# @PRE debug_dir exists and is writable.
# @POST Returns the debug path or None on failure.
# @RATIONALE Uses tempfile.mkdtemp() to avoid accumulating .png files in production storage. Temp dir is cleaned up on success (rmtree) or preserved on failure for debugging.
# @REJECTED Old approach saved debug .png next to output path — accumulated _debug_failed_login.png, _preresize.png permanently in screenshots dir (F1). In-memory-only debug logging rejected — screenshot state is visual and cannot be captured in logs.
async def _save_debug_screenshot(self, page, debug_dir: str, suffix: str) -> str | None:
debug_path = os.path.join(debug_dir, suffix)
try:
await page.screenshot(path=debug_path)
return debug_path
except Exception:
return None
# endregion ScreenshotService._save_debug_screenshot
# region ScreenshotService.capture_dashboard [TYPE Function] [C:4]
# @PURPOSE Captures a full-page screenshot of a dashboard using Playwright and CDP.
# @PRE dashboard_id is a valid string, output_path is a writable path.
# @POST Returns True if screenshot is saved successfully.
# @SIDE_EFFECT Launches a browser, performs UI login, switches tabs, and writes a PNG file.
# @POST Returns True if screenshot is saved successfully. Debug temp dir cleaned up on success, preserved on failure.
# @SIDE_EFFECT Launches a browser, performs UI login, switches tabs, writes a PNG file; creates and cleans up temp debug directory.
# @UX_STATE [Navigating] -> Loading dashboard UI
# @UX_STATE [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading
# @UX_STATE [CalculatingHeight] -> Determining dashboard dimensions
# @UX_STATE [Capturing] -> Executing CDP screenshot
# @RATIONALE Uses tempfile.mkdtemp() for all debug screenshots; cleans up on success (shutil.rmtree), preserves on failure for diagnostic analysis. Pre-resize screenshot is saved to temp dir and auto-cleaned on success.
# @REJECTED Old approach saved debug .png files alongside output — accumulated permanently (F1). In-memory-only logging rejected — visual state cannot be captured in text.
async def capture_dashboard(self, dashboard_id: str, output_path: str) -> bool:
with belief_scope("capture_dashboard", f"dashboard_id={dashboard_id}"):
logger.info(f"Capturing screenshot for dashboard {dashboard_id}")
async with async_playwright() as p:
browser = await p.chromium.launch(
headless=True,
args=[
"--disable-blink-features=AutomationControlled",
"--disable-infobars",
"--no-sandbox"
]
)
# Set a realistic user agent to avoid 403 Forbidden from OpenResty/WAF
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
# Construct base UI URL from environment (strip /api/v1 suffix)
base_ui_url = self.env.url.rstrip("/")
if base_ui_url.endswith("/api/v1"):
base_ui_url = base_ui_url[:-len("/api/v1")]
debug_dir = tempfile.mkdtemp(prefix="ss_screenshot_debug_")
success = False
try:
# -- begin original body --
with belief_scope("capture_dashboard", f"dashboard_id={dashboard_id}"):
logger.info(f"Capturing screenshot for dashboard {dashboard_id}")
async with async_playwright() as p:
browser = await p.chromium.launch(
headless=True,
args=[
"--disable-blink-features=AutomationControlled",
"--disable-infobars",
"--no-sandbox"
]
)
# Set a realistic user agent to avoid 403 Forbidden from OpenResty/WAF
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
# Construct base UI URL from environment (strip /api/v1 suffix)
base_ui_url = self.env.url.rstrip("/")
if base_ui_url.endswith("/api/v1"):
base_ui_url = base_ui_url[:-len("/api/v1")]
# Create browser context with realistic headers
context = await browser.new_context(
viewport={'width': 1280, 'height': 720},
user_agent=user_agent,
extra_http_headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1"
# Create browser context with realistic headers
context = await browser.new_context(
viewport={'width': 1280, 'height': 720},
user_agent=user_agent,
extra_http_headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1"
}
)
logger.info("Browser context created successfully")
page = await context.new_page()
# Bypass navigator.webdriver detection
await page.add_init_script("delete Object.getPrototypeOf(navigator).webdriver")
# 1. Navigate to login page and authenticate
login_url = f"{base_ui_url.rstrip('/')}/login/"
logger.info(f"[DEBUG] Navigating to login page: {login_url}")
response = await self._goto_resilient(
page,
login_url,
primary_wait_until="domcontentloaded",
fallback_wait_until="load",
timeout=HTTP_REQUEST_TIMEOUT_MS,
)
if response:
logger.info(f"[DEBUG] Login page response status: {response.status}")
# Wait for login form to be ready
await page.wait_for_load_state("domcontentloaded")
# More exhaustive list of selectors for various Superset versions/themes
selectors = {
"username": ['input[name="username"]', 'input#username', 'input[placeholder*="Username"]', 'input[type="text"]'],
"password": ['input[name="password"]', 'input#password', 'input[placeholder*="Password"]', 'input[type="password"]'],
"submit": ['button[type="submit"]', 'button#submit', '.btn-primary', 'input[type="submit"]']
}
)
logger.info("Browser context created successfully")
logger.info("[DEBUG] Attempting to find login form elements...")
page = await context.new_page()
# Bypass navigator.webdriver detection
await page.add_init_script("delete Object.getPrototypeOf(navigator).webdriver")
try:
used_direct_form_login = False
# Find and fill username
username_locator = await self._find_login_field_locator(page, "username")
# 1. Navigate to login page and authenticate
login_url = f"{base_ui_url.rstrip('/')}/login/"
logger.info(f"[DEBUG] Navigating to login page: {login_url}")
if not username_locator:
roots = self._iter_login_roots(page)
logger.info(f"[DEBUG] Found {len(roots)} login roots including child frames")
for root_index, root in enumerate(roots[:5]):
all_inputs = await root.locator('input').all()
logger.info(f"[DEBUG] Root {root_index}: found {len(all_inputs)} input fields")
for i, inp in enumerate(all_inputs[:5]): # Log first 5
inp_type = await inp.get_attribute('type')
inp_name = await inp.get_attribute('name')
inp_id = await inp.get_attribute('id')
logger.info(f"[DEBUG] Root {root_index} input {i}: type={inp_type}, name={inp_name}, id={inp_id}")
used_direct_form_login = await self._submit_login_via_form_post(page, login_url)
if not used_direct_form_login:
raise RuntimeError("Could not find username input field on login page")
username_locator = None
response = await self._goto_resilient(
page,
login_url,
primary_wait_until="domcontentloaded",
fallback_wait_until="load",
timeout=HTTP_REQUEST_TIMEOUT_MS,
)
if response:
logger.info(f"[DEBUG] Login page response status: {response.status}")
if username_locator is not None:
logger.info("[DEBUG] Filling username field")
await username_locator.fill(self.env.username)
# Wait for login form to be ready
await page.wait_for_load_state("domcontentloaded")
# Find and fill password
password_locator = await self._find_login_field_locator(page, "password") if username_locator is not None else None
# More exhaustive list of selectors for various Superset versions/themes
selectors = {
"username": ['input[name="username"]', 'input#username', 'input[placeholder*="Username"]', 'input[type="text"]'],
"password": ['input[name="password"]', 'input#password', 'input[placeholder*="Password"]', 'input[type="password"]'],
"submit": ['button[type="submit"]', 'button#submit', '.btn-primary', 'input[type="submit"]']
}
logger.info("[DEBUG] Attempting to find login form elements...")
if username_locator is not None and not password_locator:
raise RuntimeError("Could not find password input field on login page")
try:
used_direct_form_login = False
# Find and fill username
username_locator = await self._find_login_field_locator(page, "username")
if password_locator is not None:
logger.info("[DEBUG] Filling password field")
await password_locator.fill(self.env.password)
if not username_locator:
roots = self._iter_login_roots(page)
logger.info(f"[DEBUG] Found {len(roots)} login roots including child frames")
for root_index, root in enumerate(roots[:5]):
all_inputs = await root.locator('input').all()
logger.info(f"[DEBUG] Root {root_index}: found {len(all_inputs)} input fields")
for i, inp in enumerate(all_inputs[:5]): # Log first 5
inp_type = await inp.get_attribute('type')
inp_name = await inp.get_attribute('name')
inp_id = await inp.get_attribute('id')
logger.info(f"[DEBUG] Root {root_index} input {i}: type={inp_type}, name={inp_name}, id={inp_id}")
used_direct_form_login = await self._submit_login_via_form_post(page, login_url)
# Click submit
submit_locator = await self._find_submit_locator(page) if username_locator is not None else None
if username_locator is not None and not submit_locator:
raise RuntimeError("Could not find submit button on login page")
if submit_locator is not None:
logger.info("[DEBUG] Clicking submit button")
await submit_locator.click()
# Wait for navigation after login
if not used_direct_form_login:
raise RuntimeError("Could not find username input field on login page")
username_locator = None
try:
await page.wait_for_load_state("load", timeout=PLAYWRIGHT_NAVIGATION_TIMEOUT_MS)
except Exception as load_wait_error:
logger.warning(f"[DEBUG] Login post-submit load wait timed out: {load_wait_error}")
if username_locator is not None:
logger.info("[DEBUG] Filling username field")
await username_locator.fill(self.env.username)
# Check if login was successful
if not used_direct_form_login and "/login" in page.url:
# Check for error messages on page
error_msg = await page.locator(".alert-danger, .error-message").text_content() if await page.locator(".alert-danger, .error-message").count() > 0 else "Unknown error"
logger.error(f"[DEBUG] Login failed. Still on login page. Error: {error_msg}")
debug_path = await self._save_debug_screenshot(page, debug_dir, "failed_login.png")
raise RuntimeError(f"Login failed: {error_msg}. Debug screenshot saved to {debug_path}")
# Find and fill password
password_locator = await self._find_login_field_locator(page, "password") if username_locator is not None else None
logger.info(f"[DEBUG] Login successful. Current URL: {page.url}")
if username_locator is not None and not password_locator:
raise RuntimeError("Could not find password input field on login page")
# Check cookies after successful login
page_cookies = await context.cookies()
logger.info(f"[DEBUG] Cookies after login: {len(page_cookies)}")
for c in page_cookies:
logger.info(f"[DEBUG] Cookie: name={c['name']}, domain={c['domain']}, value={c.get('value', '')[:20]}...")
if password_locator is not None:
logger.info("[DEBUG] Filling password field")
await password_locator.fill(self.env.password)
except Exception as e:
page_title = await page.title()
logger.error(f"UI Login failed. Page title: {page_title}, URL: {page.url}, Error: {e!s}")
debug_path = await self._save_debug_screenshot(page, debug_dir, "failed_login.png")
raise RuntimeError(f"Login failed: {e!s}. Debug screenshot saved to {debug_path}")
# Click submit
submit_locator = await self._find_submit_locator(page) if username_locator is not None else None
# 2. Navigate to dashboard
# @UX_STATE [Navigating] -> Loading dashboard UI
dashboard_url = f"{base_ui_url.rstrip('/')}/superset/dashboard/{dashboard_id}/?standalone=true"
if username_locator is not None and not submit_locator:
raise RuntimeError("Could not find submit button on login page")
if base_ui_url.startswith("https://") and dashboard_url.startswith("http://"):
dashboard_url = dashboard_url.replace("http://", "https://")
if submit_locator is not None:
logger.info("[DEBUG] Clicking submit button")
await submit_locator.click()
logger.info(f"[DEBUG] Navigating to dashboard: {dashboard_url}")
# Wait for navigation after login
if not used_direct_form_login:
try:
await page.wait_for_load_state("load", timeout=PLAYWRIGHT_NAVIGATION_TIMEOUT_MS)
except Exception as load_wait_error:
logger.warning(f"[DEBUG] Login post-submit load wait timed out: {load_wait_error}")
# Check if login was successful
if not used_direct_form_login and "/login" in page.url:
# Check for error messages on page
error_msg = await page.locator(".alert-danger, .error-message").text_content() if await page.locator(".alert-danger, .error-message").count() > 0 else "Unknown error"
logger.error(f"[DEBUG] Login failed. Still on login page. Error: {error_msg}")
debug_path = output_path.replace(".png", "_debug_failed_login.png")
await page.screenshot(path=debug_path)
raise RuntimeError(f"Login failed: {error_msg}. Debug screenshot saved to {debug_path}")
logger.info(f"[DEBUG] Login successful. Current URL: {page.url}")
# Check cookies after successful login
page_cookies = await context.cookies()
logger.info(f"[DEBUG] Cookies after login: {len(page_cookies)}")
for c in page_cookies:
logger.info(f"[DEBUG] Cookie: name={c['name']}, domain={c['domain']}, value={c.get('value', '')[:20]}...")
except Exception as e:
page_title = await page.title()
logger.error(f"UI Login failed. Page title: {page_title}, URL: {page.url}, Error: {e!s}")
debug_path = output_path.replace(".png", "_debug_failed_login.png")
await page.screenshot(path=debug_path)
raise RuntimeError(f"Login failed: {e!s}. Debug screenshot saved to {debug_path}")
# 2. Navigate to dashboard
# @UX_STATE [Navigating] -> Loading dashboard UI
dashboard_url = f"{base_ui_url.rstrip('/')}/superset/dashboard/{dashboard_id}/?standalone=true"
if base_ui_url.startswith("https://") and dashboard_url.startswith("http://"):
dashboard_url = dashboard_url.replace("http://", "https://")
logger.info(f"[DEBUG] Navigating to dashboard: {dashboard_url}")
# Dashboard pages can keep polling/network activity open indefinitely.
response = await self._goto_resilient(
page,
dashboard_url,
primary_wait_until="domcontentloaded",
fallback_wait_until="load",
timeout=HTTP_REQUEST_TIMEOUT_MS,
)
if response:
logger.info(f"[DEBUG] Dashboard navigation response status: {response.status}, URL: {response.url}")
if "/login" in page.url:
debug_path = output_path.replace(".png", "_debug_failed_dashboard_auth.png")
await page.screenshot(path=debug_path)
raise RuntimeError(
f"Dashboard navigation redirected to login page after authentication. Debug screenshot saved to {debug_path}"
# Dashboard pages can keep polling/network activity open indefinitely.
response = await self._goto_resilient(
page,
dashboard_url,
primary_wait_until="domcontentloaded",
fallback_wait_until="load",
timeout=HTTP_REQUEST_TIMEOUT_MS,
)
try:
# Wait for the dashboard grid to be present
await page.wait_for_selector('.dashboard-component, .dashboard-header, [data-test="dashboard-grid"]', timeout=PLAYWRIGHT_NAVIGATION_TIMEOUT_MS)
logger.info("[DEBUG] Dashboard container loaded")
if response:
logger.info(f"[DEBUG] Dashboard navigation response status: {response.status}, URL: {response.url}")
if "/login" in page.url:
debug_path = await self._save_debug_screenshot(page, debug_dir, "failed_dashboard_auth.png")
raise RuntimeError(
f"Dashboard navigation redirected to login page after authentication. Debug screenshot saved to {debug_path}"
)
# Wait for charts to finish loading (Superset uses loading spinners/skeletons)
# We wait until loading indicators disappear or a timeout occurs
try:
# Wait for loading indicators to disappear
await page.wait_for_selector('.loading, .ant-skeleton, .spinner', state="hidden", timeout=HTTP_REQUEST_TIMEOUT_MS)
logger.info("[DEBUG] Loading indicators hidden")
except Exception:
logger.warning("[DEBUG] Timeout waiting for loading indicators to hide")
# Wait for the dashboard grid to be present
await page.wait_for_selector('.dashboard-component, .dashboard-header, [data-test="dashboard-grid"]', timeout=PLAYWRIGHT_NAVIGATION_TIMEOUT_MS)
logger.info("[DEBUG] Dashboard container loaded")
# Wait for charts to actually render their content (e.g., ECharts, NVD3)
# We look for common chart containers that should have content
try:
await page.wait_for_selector('.chart-container canvas, .slice_container svg, .superset-chart-canvas, .grid-content .chart-container', timeout=HTTP_REQUEST_TIMEOUT_MS)
logger.info("[DEBUG] Chart content detected")
except Exception:
logger.warning("[DEBUG] Timeout waiting for chart content")
# Wait for charts to finish loading (Superset uses loading spinners/skeletons)
# We wait until loading indicators disappear or a timeout occurs
try:
# Wait for loading indicators to disappear
await page.wait_for_selector('.loading, .ant-skeleton, .spinner', state="hidden", timeout=HTTP_REQUEST_TIMEOUT_MS)
logger.info("[DEBUG] Loading indicators hidden")
except Exception:
logger.warning("[DEBUG] Timeout waiting for loading indicators to hide")
# Additional check: wait for all chart containers to have non-empty content
logger.info("[DEBUG] Waiting for all charts to have rendered content...")
await page.wait_for_function("""() => {
const charts = document.querySelectorAll('.chart-container, .slice_container');
if (charts.length === 0) return true; // No charts to wait for
# Wait for charts to actually render their content (e.g., ECharts, NVD3)
# We look for common chart containers that should have content
try:
await page.wait_for_selector('.chart-container canvas, .slice_container svg, .superset-chart-canvas, .grid-content .chart-container', timeout=HTTP_REQUEST_TIMEOUT_MS)
logger.info("[DEBUG] Chart content detected")
except Exception:
logger.warning("[DEBUG] Timeout waiting for chart content")
# Additional check: wait for all chart containers to have non-empty content
logger.info("[DEBUG] Waiting for all charts to have rendered content...")
await page.wait_for_function("""() => {
const charts = document.querySelectorAll('.chart-container, .slice_container');
if (charts.length === 0) return true; // No charts to wait for
// Check if all charts have rendered content (canvas, svg, or non-empty div)
return Array.from(charts).every(chart => {
const hasCanvas = chart.querySelector('canvas') !== null;
const hasSvg = chart.querySelector('svg') !== null;
const hasContent = chart.innerText.trim().length > 0 || chart.children.length > 0;
return hasCanvas || hasSvg || hasContent;
});
}""", timeout=HTTP_REQUEST_TIMEOUT_MS)
logger.info("[DEBUG] All charts have rendered content")
// Check if all charts have rendered content (canvas, svg, or non-empty div)
return Array.from(charts).every(chart => {
const hasCanvas = chart.querySelector('canvas') !== null;
const hasSvg = chart.querySelector('svg') !== null;
const hasContent = chart.innerText.trim().length > 0 || chart.children.length > 0;
return hasCanvas || hasSvg || hasContent;
});
}""", timeout=HTTP_REQUEST_TIMEOUT_MS)
logger.info("[DEBUG] All charts have rendered content")
# Scroll to bottom and back to top to trigger lazy loading of all charts
logger.info("[DEBUG] Scrolling to trigger lazy loading...")
await page.evaluate("""async () => {
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
for (let i = 0; i < document.body.scrollHeight; i += 500) {
window.scrollTo(0, i);
await delay(100);
}
window.scrollTo(0, 0);
await delay(500);
}""")
except Exception as e:
logger.warning(f"[DEBUG] Dashboard content wait failed: {e}, proceeding anyway after delay")
# Final stabilization delay - increased for complex dashboards
logger.info("[DEBUG] Final stabilization delay...")
await asyncio.sleep(15)
# Logic to handle tabs and full-page capture
try:
# 1. Handle Tabs (Recursive switching)
# @UX_STATE [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading
processed_tabs = set()
async def switch_tabs(depth=0):
if depth > 3:
return # Limit recursion depth
tab_selectors = [
'.ant-tabs-nav-list .ant-tabs-tab',
'.dashboard-component-tabs .ant-tabs-tab',
'[data-test="dashboard-component-tabs"] .ant-tabs-tab'
]
found_tabs = []
for selector in tab_selectors:
found_tabs = await page.locator(selector).all()
if found_tabs:
break
if found_tabs:
logger.info(f"[DEBUG][TabSwitching] Found {len(found_tabs)} tabs at depth {depth}")
for i, tab in enumerate(found_tabs):
try:
tab_text = (await tab.inner_text()).strip()
tab_id = f"{depth}_{i}_{tab_text}"
if tab_id in processed_tabs:
continue
if await tab.is_visible():
logger.info(f"[DEBUG][TabSwitching] Switching to tab: {tab_text}")
processed_tabs.add(tab_id)
is_active = "ant-tabs-tab-active" in (await tab.get_attribute("class") or "")
if not is_active:
await tab.click()
await asyncio.sleep(2) # Wait for content to render
await switch_tabs(depth + 1)
except Exception as tab_e:
logger.warning(f"[DEBUG][TabSwitching] Failed to process tab {i}: {tab_e}")
try:
first_tab = found_tabs[0]
if "ant-tabs-tab-active" not in (await first_tab.get_attribute("class") or ""):
await first_tab.click()
await asyncio.sleep(1)
except Exception:
pass
await switch_tabs()
# 2. Calculate full height for screenshot
# @UX_STATE [CalculatingHeight] -> Determining dashboard dimensions
full_height = await page.evaluate("""() => {
const body = document.body;
const html = document.documentElement;
const dashboardContent = document.querySelector('.dashboard-content');
return Math.max(
body.scrollHeight, body.offsetHeight,
html.clientHeight, html.scrollHeight, html.offsetHeight,
dashboardContent ? dashboardContent.scrollHeight + 100 : 0
);
}""")
logger.info(f"[DEBUG] Calculated full height: {full_height}")
# DIAGNOSTIC: Count chart elements before resize
chart_count_before = await page.evaluate("""() => {
return {
chartContainers: document.querySelectorAll('.chart-container, .slice_container').length,
canvasElements: document.querySelectorAll('canvas').length,
svgElements: document.querySelectorAll('.chart-container svg, .slice_container svg').length,
visibleCharts: document.querySelectorAll('.chart-container:visible, .slice_container:visible').length
};
}""")
logger.info(f"[DIAGNOSTIC] Chart elements BEFORE viewport resize: {chart_count_before}")
# DIAGNOSTIC: Capture pre-resize screenshot for comparison
pre_resize_path = output_path.replace(".png", "_preresize.png")
try:
await page.screenshot(path=pre_resize_path, full_page=False, timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS)
import os
pre_resize_size = os.path.getsize(pre_resize_path) if os.path.exists(pre_resize_path) else 0
logger.info(f"[DIAGNOSTIC] Pre-resize screenshot saved: {pre_resize_path} ({pre_resize_size} bytes)")
except Exception as pre_e:
logger.warning(f"[DIAGNOSTIC] Failed to capture pre-resize screenshot: {pre_e}")
logger.info(f"[DIAGNOSTIC] Resizing viewport from current to 1920x{int(full_height)}")
await page.set_viewport_size({"width": 1920, "height": int(full_height)})
# DIAGNOSTIC: Increased wait time and log timing
logger.info("[DIAGNOSTIC] Waiting 10 seconds after viewport resize for re-render...")
await asyncio.sleep(10)
logger.info("[DIAGNOSTIC] Wait completed")
# DIAGNOSTIC: Count chart elements after resize and wait
chart_count_after = await page.evaluate("""() => {
return {
chartContainers: document.querySelectorAll('.chart-container, .slice_container').length,
canvasElements: document.querySelectorAll('canvas').length,
svgElements: document.querySelectorAll('.chart-container svg, .slice_container svg').length,
visibleCharts: document.querySelectorAll('.chart-container:visible, .slice_container:visible').length
};
}""")
logger.info(f"[DIAGNOSTIC] Chart elements AFTER viewport resize + wait: {chart_count_after}")
# DIAGNOSTIC: Check if any charts have error states
chart_errors = await page.evaluate("""() => {
const errors = [];
document.querySelectorAll('.chart-container, .slice_container').forEach((chart, i) => {
const errorEl = chart.querySelector('.error, .alert-danger, .ant-alert-error');
if (errorEl) {
errors.push({index: i, text: errorEl.innerText.substring(0, 100)});
# Scroll to bottom and back to top to trigger lazy loading of all charts
logger.info("[DEBUG] Scrolling to trigger lazy loading...")
await page.evaluate("""async () => {
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
for (let i = 0; i < document.body.scrollHeight; i += 500) {
window.scrollTo(0, i);
await delay(100);
}
});
return errors;
}""")
if chart_errors:
logger.warning(f"[DIAGNOSTIC] Charts with error states detected: {chart_errors}")
else:
logger.info("[DIAGNOSTIC] No chart error states detected")
window.scrollTo(0, 0);
await delay(500);
}""")
# 3. Take screenshot using CDP to bypass Playwright's font loading wait
# @UX_STATE [Capturing] -> Executing CDP screenshot
logger.info("[DEBUG] Attempting full-page screenshot via CDP...")
cdp = await page.context.new_cdp_session(page)
except Exception as e:
logger.warning(f"[DEBUG] Dashboard content wait failed: {e}, proceeding anyway after delay")
screenshot_data = await cdp.send("Page.captureScreenshot", {
"format": "png",
"fromSurface": True,
"captureBeyondViewport": True
})
# Final stabilization delay - wait for charts to stabilize
logger.info("[DEBUG] Waiting for charts to stabilize...")
await self._wait_for_charts_stabilized(page)
image_data = base64.b64decode(screenshot_data["data"])
with open(output_path, 'wb') as f:
f.write(image_data)
# DIAGNOSTIC: Verify screenshot file
import os
final_size = os.path.getsize(output_path) if os.path.exists(output_path) else 0
logger.info(f"[DIAGNOSTIC] Final screenshot saved: {output_path}")
logger.info(f"[DIAGNOSTIC] Final screenshot size: {final_size} bytes ({final_size / 1024:.2f} KB)")
# DIAGNOSTIC: Get image dimensions
# Logic to handle tabs and full-page capture
try:
with Image.open(output_path) as final_img:
logger.info(f"[DIAGNOSTIC] Final screenshot dimensions: {final_img.width}x{final_img.height}")
except Exception as img_err:
logger.warning(f"[DIAGNOSTIC] Could not read final image dimensions: {img_err}")
async with asyncio.timeout(SCREENSHOT_SERVICE_TIMEOUT_MS / 1000):
# 1. Handle Tabs (Recursive switching)
# @UX_STATE [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading
processed_tabs = set()
logger.info(f"Full-page screenshot saved to {output_path} (via CDP)")
except Exception as e:
logger.error(f"[DEBUG] Full-page/Tab capture failed: {e}")
try:
await page.screenshot(path=output_path, full_page=True, timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS)
except Exception as e2:
logger.error(f"[DEBUG] Fallback screenshot also failed: {e2}")
await page.screenshot(path=output_path, timeout=PLAYWRIGHT_SHORT_TIMEOUT_MS)
async def switch_tabs(depth=0):
if depth > 3:
return # Limit recursion depth
await browser.close()
return True
tab_selectors = [
'.ant-tabs-nav-list .ant-tabs-tab',
'.dashboard-component-tabs .ant-tabs-tab',
'[data-test="dashboard-component-tabs"] .ant-tabs-tab'
]
found_tabs = []
for selector in tab_selectors:
found_tabs = await page.locator(selector).all()
if found_tabs:
break
if found_tabs:
logger.info(f"[DEBUG][TabSwitching] Found {len(found_tabs)} tabs at depth {depth}")
for i, tab in enumerate(found_tabs):
try:
tab_text = (await tab.inner_text()).strip()
tab_id = f"{depth}_{i}_{tab_text}"
if tab_id in processed_tabs:
continue
if await tab.is_visible():
logger.info(f"[DEBUG][TabSwitching] Switching to tab: {tab_text}")
processed_tabs.add(tab_id)
is_active = "ant-tabs-tab-active" in (await tab.get_attribute("class") or "")
if not is_active:
await tab.click()
# Wait for chart content to render in the newly active tab
try:
await page.wait_for_function("""() => {
const activeTab = document.querySelector('.ant-tabs-tab-active');
if (!activeTab) return true;
const tabPane = activeTab.closest('.ant-tabs')?.querySelector('.ant-tabs-content-holder');
if (!tabPane) return true;
const charts = tabPane.querySelectorAll('canvas, svg');
return charts.length > 0;
}""", timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS)
except Exception:
logger.warning(f"[TabSwitching] Content verification timed out for tab: {tab_text}")
await switch_tabs(depth + 1)
except Exception as tab_e:
logger.warning(f"[DEBUG][TabSwitching] Failed to process tab {i}: {tab_e}")
try:
first_tab = found_tabs[0]
if "ant-tabs-tab-active" not in (await first_tab.get_attribute("class") or ""):
await first_tab.click()
try:
await page.wait_for_function("""() => {
const activeTab = document.querySelector('.ant-tabs-tab-active');
if (!activeTab) return true;
const tabPane = activeTab.closest('.ant-tabs')?.querySelector('.ant-tabs-content-holder');
if (!tabPane) return true;
const charts = tabPane.querySelectorAll('canvas, svg');
return charts.length > 0;
}""", timeout=PLAYWRIGHT_SHORT_TIMEOUT_MS)
except Exception:
logger.warning("[TabSwitching] Content verification timed out for first tab")
except Exception:
pass
await switch_tabs()
# 2. Calculate full height for screenshot
# @UX_STATE [CalculatingHeight] -> Determining dashboard dimensions
full_height = await page.evaluate("""() => {
const body = document.body;
const html = document.documentElement;
const dashboardContent = document.querySelector('.dashboard-content');
return Math.max(
body.scrollHeight, body.offsetHeight,
html.clientHeight, html.scrollHeight, html.offsetHeight,
dashboardContent ? dashboardContent.scrollHeight + 100 : 0
);
}""")
logger.info(f"[DEBUG] Calculated full height: {full_height}")
# DIAGNOSTIC: Count chart elements before resize
chart_count_before = await page.evaluate("""() => {
return {
chartContainers: document.querySelectorAll('.chart-container, .slice_container').length,
canvasElements: document.querySelectorAll('canvas').length,
svgElements: document.querySelectorAll('.chart-container svg, .slice_container svg').length,
visibleCharts: document.querySelectorAll('.chart-container:visible, .slice_container:visible').length
};
}""")
logger.info(f"[DIAGNOSTIC] Chart elements BEFORE viewport resize: {chart_count_before}")
# DIAGNOSTIC: Capture pre-resize screenshot for comparison
pre_resize_path = await self._save_debug_screenshot(page, debug_dir, "preresize.png")
if pre_resize_path:
pre_resize_size = os.path.getsize(pre_resize_path) if os.path.exists(pre_resize_path) else 0
logger.info(f"[DIAGNOSTIC] Pre-resize screenshot saved: {pre_resize_path} ({pre_resize_size} bytes)")
else:
logger.warning("[DIAGNOSTIC] Failed to capture pre-resize screenshot")
logger.info(f"[DIAGNOSTIC] Resizing viewport from current to 1920x{int(full_height)}")
await page.set_viewport_size({"width": 1920, "height": int(full_height)})
# Wait for charts to re-render after viewport resize
logger.info("[DIAGNOSTIC] Waiting for re-render after viewport resize...")
await self._wait_for_resize_rendered(page, chart_count_before)
logger.info("[DIAGNOSTIC] Re-render wait completed")
# DIAGNOSTIC: Count chart elements after resize and wait
chart_count_after = await page.evaluate("""() => {
return {
chartContainers: document.querySelectorAll('.chart-container, .slice_container').length,
canvasElements: document.querySelectorAll('canvas').length,
svgElements: document.querySelectorAll('.chart-container svg, .slice_container svg').length,
visibleCharts: document.querySelectorAll('.chart-container:visible, .slice_container:visible').length
};
}""")
logger.info(f"[DIAGNOSTIC] Chart elements AFTER viewport resize + wait: {chart_count_after}")
# DIAGNOSTIC: Check if any charts have error states
chart_errors = await page.evaluate("""() => {
const errors = [];
document.querySelectorAll('.chart-container, .slice_container').forEach((chart, i) => {
const errorEl = chart.querySelector('.error, .alert-danger, .ant-alert-error');
if (errorEl) {
errors.push({index: i, text: errorEl.innerText.substring(0, 100)});
}
});
return errors;
}""")
if chart_errors:
logger.warning(f"[DIAGNOSTIC] Charts with error states detected: {chart_errors}")
else:
logger.info("[DIAGNOSTIC] No chart error states detected")
# 3. Take screenshot using CDP to bypass Playwright's font loading wait
# @UX_STATE [Capturing] -> Executing CDP screenshot
logger.info("[DEBUG] Attempting full-page screenshot via CDP...")
cdp_success = False
try:
cdp = await page.context.new_cdp_session(page)
screenshot_data = await cdp.send("Page.captureScreenshot", {
"format": "png",
"fromSurface": True,
"captureBeyondViewport": True
})
image_data = base64.b64decode(screenshot_data["data"])
with open(output_path, 'wb') as f:
f.write(image_data)
logger.info("Full-page screenshot saved via CDP")
cdp_success = True
except Exception as cdp_err:
logger.warning(f"CDP screenshot failed: {cdp_err}. Falling back to Playwright full_page.")
await page.screenshot(path=output_path, full_page=True, timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS)
logger.info("Full-page screenshot saved via Playwright fallback")
cdp_success = True
if cdp_success:
# DIAGNOSTIC: Verify screenshot file
final_size = os.path.getsize(output_path) if os.path.exists(output_path) else 0
logger.info(f"[DIAGNOSTIC] Final screenshot saved: {output_path}")
logger.info(f"[DIAGNOSTIC] Final screenshot size: {final_size} bytes ({final_size / 1024:.2f} KB)")
# DIAGNOSTIC: Get image dimensions
try:
with Image.open(output_path) as final_img:
logger.info(f"[DIAGNOSTIC] Final screenshot dimensions: {final_img.width}x{final_img.height}")
except Exception as img_err:
logger.warning(f"[DIAGNOSTIC] Could not read final image dimensions: {img_err}")
except asyncio.TimeoutError:
logger.error(f"Screenshot capture timed out after {SCREENSHOT_SERVICE_TIMEOUT_MS}ms")
try:
await page.screenshot(path=output_path, timeout=PLAYWRIGHT_SHORT_TIMEOUT_MS)
logger.info("Emergency timeout screenshot saved")
except Exception:
pass
except Exception as e:
logger.error(f"[DEBUG] Full-page/Tab capture failed: {e}")
try:
await page.screenshot(path=output_path, full_page=True, timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS)
except Exception as e2:
logger.error(f"[DEBUG] Fallback screenshot also failed: {e2}")
await page.screenshot(path=output_path, timeout=PLAYWRIGHT_SHORT_TIMEOUT_MS)
await browser.close()
success = True
return True
finally:
if success:
shutil.rmtree(debug_dir, ignore_errors=True)
else:
logger.info(f"Debug screenshots preserved in {debug_dir}")
# endregion ScreenshotService.capture_dashboard
# #endregion ScreenshotService
@@ -985,18 +1089,7 @@ class LLMClient:
"issues": [{"severity": "UNKNOWN", "message": "LLM provider returned empty or invalid response"}]
}
# endregion LLMClient.analyze_dashboard
# #endregion LLMClient
# #endregion LLMAnalysisService
return await self.get_json_completion(messages)
except Exception as e:
logger.error(f"[analyze_dashboard] Failed to get analysis: {e!s}")
return {
"status": "UNKNOWN",
"summary": f"Failed to get response from LLM: {e!s}",
"issues": [{"severity": "UNKNOWN", "message": "LLM provider returned empty or invalid response"}]
}
# endregion LLMClient.analyze_dashboard
# #endregion LLMClient
# #endregion LLMAnalysisService

View File

@@ -0,0 +1,139 @@
# #region ValidationSchemas [C:3] [TYPE Module] [SEMANTICS pydantic, validation, schema, task, run]
# @BRIEF Pydantic v2 schemas for validation task management and run history API serialization.
# @LAYER API
# @RELATION DEPENDS_ON -> pydantic
from datetime import datetime
from typing import Any
from pydantic import BaseModel, Field
# #region ValidationTaskCreate [C:1] [TYPE Class]
# @BRIEF Schema for creating a new validation task (policy).
class ValidationTaskCreate(BaseModel):
name: str = Field(..., description="Human-readable task name")
environment_id: str = Field(..., description="Target Superset environment ID")
dashboard_ids: list[str] = Field(..., description="List of dashboard IDs to validate (use single-element array for v1)")
provider_id: str = Field(..., description="LLM provider ID — must be multimodal")
schedule_days: list[int] | None = Field(None, description="Days of week (0-6, 0=Sunday). Omit or empty for manual-only.")
window_start: str | None = Field(None, description="Window start time, HH:MM format")
window_end: str | None = Field(None, description="Window end time, HH:MM format")
notify_owners: bool = Field(True, description="Notify dashboard owners on failure")
custom_channels: list[str] | None = Field(None, description="Additional notification channels")
alert_condition: str = Field("FAIL_ONLY", description="FAIL_ONLY, WARN_AND_FAIL, or ALWAYS")
is_active: bool = Field(True, description="Whether the task is active")
# #endregion ValidationTaskCreate
# #region ValidationTaskUpdate [C:1] [TYPE Class]
# @BRIEF Schema for updating an existing validation task (policy).
class ValidationTaskUpdate(BaseModel):
name: str | None = None
environment_id: str | None = None
dashboard_ids: list[str] | None = None
provider_id: str | None = None
schedule_days: list[int] | None = None
window_start: str | None = None
window_end: str | None = None
notify_owners: bool | None = None
custom_channels: list[str] | None = None
alert_condition: str | None = None
is_active: bool | None = None
# #endregion ValidationTaskUpdate
# #region ValidationTaskResponse [C:1] [TYPE Class]
# @BRIEF Schema for validation task API responses — includes last run status from join.
class ValidationTaskResponse(BaseModel):
id: str
name: str
environment_id: str
is_active: bool
dashboard_ids: list[str]
provider_id: str | None = None
schedule_days: list[int] | None = None
window_start: str | None = None
window_end: str | None = None
notify_owners: bool = True
custom_channels: list[str] | None = None
alert_condition: str = "FAIL_ONLY"
created_at: datetime
updated_at: datetime | None = None
last_run_status: str | None = None
last_run_at: datetime | None = None
class Config:
from_attributes = True
# #endregion ValidationTaskResponse
# #region ValidationTaskListResponse [C:1] [TYPE Class]
# @BRIEF Schema for paginated task list response.
class ValidationTaskListResponse(BaseModel):
items: list[ValidationTaskResponse] = []
total: int = 0
page: int = 1
page_size: int = 20
# #endregion ValidationTaskListResponse
# #region ValidationRunResponse [C:1] [TYPE Class]
# @BRIEF Schema for validation run history listing.
class ValidationRunResponse(BaseModel):
id: str
task_id: str | None = None
dashboard_id: str
environment_id: str | None = None
status: str
summary: str = ""
timestamp: datetime | None = None
screenshot_path: str | None = None
task_name: str | None = None # resolved from ValidationPolicy at query time
class Config:
from_attributes = True
# #endregion ValidationRunResponse
# #region ValidationRunListResponse [C:1] [TYPE Class]
# @BRIEF Schema for paginated run list response.
class ValidationRunListResponse(BaseModel):
items: list[ValidationRunResponse] = []
total: int = 0
page: int = 1
page_size: int = 20
# #endregion ValidationRunListResponse
# #region ValidationRunDetailResponse [C:1] [TYPE Class]
# @BRIEF Schema for full run detail including parsed issues and raw response.
class ValidationRunDetailResponse(BaseModel):
id: str
task_id: str | None = None
dashboard_id: str
environment_id: str | None = None
status: str
summary: str = ""
timestamp: datetime | None = None
screenshot_path: str | None = None
issues: list[dict[str, Any]] = Field(default_factory=list)
raw_response: str | None = None
task_name: str | None = None
class Config:
from_attributes = True
# #endregion ValidationRunDetailResponse
# #region TriggerRunResponse [C:1] [TYPE Class]
# @BRIEF Schema for trigger-run response — returns spawned task ID.
class TriggerRunResponse(BaseModel):
task_id: str
spawned_task_id: str
status: str = "PENDING"
message: str = "Validation task spawned"
# #endregion TriggerRunResponse
# #endregion ValidationSchemas

View File

@@ -0,0 +1,135 @@
# #region ValidationRunService [C:3] [TYPE Module] [SEMANTICS validation, run, service, sqlalchemy]
# @BRIEF Business logic for validation run queries: list, detail, delete.
# @LAYER Service
from datetime import date
from sqlalchemy.orm import Session
from ..core.logger import logger
from ..models.llm import ValidationPolicy, ValidationRecord
from ..schemas.validation import ValidationRunDetailResponse, ValidationRunResponse
# #region ValidationRunService [C:4] [TYPE Class]
# @BRIEF Queries and manages validation run history records.
# @RATIONALE Separates run history queries from task CRUD — run data has different access patterns (filter-heavy, paginated) than task configuration.
# @REJECTED Merging run service into task service rejected — would create a large class with mixed responsibilities and different query patterns.
class ValidationRunService:
def __init__(self, db: Session):
self.db = db
# #region _resolve_task_name [C:2] [TYPE Function]
def _resolve_task_name(self, record: ValidationRecord) -> str | None:
"""Find matching ValidationPolicy name by dashboard + environment."""
if not record.dashboard_id or not record.environment_id:
return None
candidates = (
self.db.query(ValidationPolicy)
.filter(ValidationPolicy.environment_id == record.environment_id)
.all()
)
for policy in candidates:
dids = list(policy.dashboard_ids) if policy.dashboard_ids else []
if record.dashboard_id in dids:
return policy.name
return None
# #endregion _resolve_task_name
# #region list_runs [C:3] [TYPE Function]
# @RATIONALE Supports 6 optional filters + pagination at the query level, enabling flexible cross-task run exploration without application-level filtering.
# @REJECTED Application-level filtering after loading all runs rejected — would be unscalable with large run histories; database-level WHERE clauses are more efficient.
def list_runs(
self,
task_id: str | None = None,
dashboard_id: str | None = None,
environment_id: str | None = None,
status: str | None = None,
date_from: date | None = None,
date_to: date | None = None,
page: int = 1,
page_size: int = 20,
) -> tuple[int, list[ValidationRunResponse]]:
query = self.db.query(ValidationRecord)
if task_id:
query = query.filter(ValidationRecord.task_id == task_id)
if dashboard_id:
query = query.filter(ValidationRecord.dashboard_id == dashboard_id)
if environment_id:
query = query.filter(ValidationRecord.environment_id == environment_id)
if status:
query = query.filter(ValidationRecord.status == status.upper())
if date_from:
query = query.filter(ValidationRecord.timestamp >= date_from)
if date_to:
query = query.filter(ValidationRecord.timestamp <= date_to)
total = query.count()
records = (
query.order_by(ValidationRecord.timestamp.desc())
.offset((page - 1) * page_size)
.limit(page_size)
.all()
)
items = [
ValidationRunResponse(
id=r.id,
task_id=r.task_id,
dashboard_id=r.dashboard_id,
environment_id=r.environment_id,
status=r.status,
summary=r.summary or "",
timestamp=r.timestamp,
screenshot_path=r.screenshot_path,
task_name=self._resolve_task_name(r),
)
for r in records
]
return total, items
# #endregion list_runs
# #region get_run_detail [C:3] [TYPE Function]
# @RATIONALE Returns full join including issues list and raw_response for the report UI — single query avoids N+1 when displaying the run detail page.
# @REJECTED Lazy-loading issues and raw_response separately rejected — would cause N+1 queries on the detail page; both fields are always needed for the report view.
def get_run_detail(self, run_id: str) -> ValidationRunDetailResponse:
record = self.db.query(ValidationRecord).filter(ValidationRecord.id == run_id).first()
if not record:
raise ValueError(f"Validation run '{run_id}' not found")
return ValidationRunDetailResponse(
id=record.id,
task_id=record.task_id,
dashboard_id=record.dashboard_id,
environment_id=record.environment_id,
status=record.status,
summary=record.summary or "",
timestamp=record.timestamp,
screenshot_path=record.screenshot_path,
issues=list(record.issues) if record.issues else [],
raw_response=record.raw_response,
task_name=self._resolve_task_name(record),
)
# #endregion get_run_detail
# #region delete_run [C:2] [TYPE Function]
# @RATIONALE Cleans up the database record only — screenshot cleanup is caller responsibility to maintain separation of concerns between DB and filesystem.
# @REJECTED Automatic screenshot file deletion in service layer rejected — would couple DB cleanup to filesystem operations, complicating testing and error handling.
def delete_run(self, run_id: str) -> bool:
record = self.db.query(ValidationRecord).filter(ValidationRecord.id == run_id).first()
if not record:
return False
self.db.delete(record)
self.db.commit()
logger.info("[ValidationRunService] Deleted run '%s'", run_id)
return True
# #endregion delete_run
# #endregion ValidationRunService
# #endregion ValidationRunService

View File

@@ -0,0 +1,265 @@
# #region ValidationService [C:3] [TYPE Module] [SEMANTICS validation, service, sqlalchemy, task]
# @BRIEF Business logic for validation task CRUD and trigger-run.
# @LAYER Service
from datetime import time
from typing import Any
from sqlalchemy.orm import Session
from ..core.logger import logger
from ..core.task_manager import TaskManager
from ..models.llm import LLMProvider, ValidationPolicy, ValidationRecord
from ..schemas.validation import ValidationTaskCreate, ValidationTaskResponse, ValidationTaskUpdate
from .llm_provider import LLMProviderService
def _parse_time(value: str | None) -> time | None:
"""Parse HH:MM string to datetime.time, or return None."""
if not value:
return None
try:
parts = value.strip().split(":")
return time(int(parts[0]), int(parts[1]))
except (ValueError, IndexError):
return None
def _format_time(value: time | None) -> str | None:
"""Format datetime.time to HH:MM string, or return None."""
if value is None:
return None
return f"{value.hour:02d}:{value.minute:02d}"
# #region ValidationTaskService [C:4] [TYPE Class]
# @BRIEF Validation task (policy) CRUD with provider validation and trigger-run.
# @RATIONALE Service-layer class for validation task CRUD with provider validation and trigger-run. Separates API layer concerns from business logic; enables unit testing without HTTP.
# @REJECTED Embedding CRUD in route handlers rejected — would duplicate validation logic across endpoints; centralized service ensures consistent provider/environment validation before persistence.
class ValidationTaskService:
def __init__(self, db: Session, config_manager=None, username: str | None = None):
self.db = db
self.config_manager = config_manager
self.username = username or "system"
# #region _validate_provider [C:2] [TYPE Function]
# @RATIONALE Centralizes multimodal LLM provider check so all task operations (create, update, trigger) share the same validation gate. Uses LLMProviderService for reuse.
# @REJECTED Duplicating multimodal check in each caller rejected — would create maintenance burden if provider validation rules change or error messages need updating.
def _validate_provider(self, provider_id: str) -> LLMProvider:
if not provider_id or not provider_id.strip():
raise ValueError("provider_id is required")
svc = LLMProviderService(self.db)
provider = svc.get_provider(provider_id)
if not provider:
raise ValueError(f"LLM provider '{provider_id}' not found")
if not bool(provider.is_multimodal):
raise ValueError(
f"LLM provider '{provider.name}' is not multimodal. "
"Dashboard validation requires a multimodal model (image input support)."
)
return provider
# #endregion _validate_provider
# #region _validate_environment [C:1] [TYPE Function]
def _validate_environment(self, environment_id: str) -> None:
if not self.config_manager:
return
env = self.config_manager.get_environment(environment_id)
if not env:
raise ValueError(f"Environment '{environment_id}' not found")
# #endregion _validate_environment
# #region _policy_to_response [C:2] [TYPE Function]
def _policy_to_response(self, policy: ValidationPolicy) -> ValidationTaskResponse:
last_run = None
if policy.dashboard_ids:
last_run = (
self.db.query(ValidationRecord)
.filter(
ValidationRecord.dashboard_id.in_(policy.dashboard_ids),
ValidationRecord.environment_id == policy.environment_id,
)
.order_by(ValidationRecord.timestamp.desc())
.first()
)
return ValidationTaskResponse(
id=policy.id,
name=policy.name,
environment_id=policy.environment_id,
is_active=bool(policy.is_active) if policy.is_active is not None else True,
dashboard_ids=list(policy.dashboard_ids) if policy.dashboard_ids else [],
provider_id=policy.provider_id,
schedule_days=list(policy.schedule_days) if policy.schedule_days else None,
window_start=_format_time(policy.window_start),
window_end=_format_time(policy.window_end),
notify_owners=bool(policy.notify_owners) if policy.notify_owners is not None else True,
custom_channels=list(policy.custom_channels) if policy.custom_channels else None,
alert_condition=policy.alert_condition or "FAIL_ONLY",
created_at=policy.created_at,
updated_at=policy.updated_at,
last_run_status=str(last_run.status) if last_run else None,
last_run_at=last_run.timestamp if last_run else None,
)
# #endregion _policy_to_response
# #region list_tasks [C:3] [TYPE Function]
# @RATIONALE Uses query-time filtering with optional is_active/environment_id params rather than post-filtering results in memory, reducing overhead with large task counts.
# @REJECTED In-memory filtering after loading all rows rejected — would be inefficient with hundreds of tasks; database-level filtering scales better.
def list_tasks(
self,
is_active: bool | None = None,
environment_id: str | None = None,
page: int = 1,
page_size: int = 20,
) -> tuple[int, list[ValidationTaskResponse]]:
query = self.db.query(ValidationPolicy)
if is_active is not None:
query = query.filter(ValidationPolicy.is_active == is_active)
if environment_id:
query = query.filter(ValidationPolicy.environment_id == environment_id)
total = query.count()
policies = (
query.order_by(ValidationPolicy.created_at.desc())
.offset((page - 1) * page_size)
.limit(page_size)
.all()
)
return total, [self._policy_to_response(p) for p in policies]
# #endregion list_tasks
# #region create_task [C:4] [TYPE Function]
# @RATIONALE Validates provider (multimodal check) and environment before persisting the policy, ensuring referential integrity at creation time and preventing silent runtime failures.
# @REJECTED Lazy validation on first run rejected — would allow creation of invalid task configurations with non-multimodal providers that fail only at runtime during dashboard validation.
def create_task(self, payload: ValidationTaskCreate) -> ValidationTaskResponse:
self._validate_provider(payload.provider_id)
self._validate_environment(payload.environment_id)
ws = _parse_time(payload.window_start)
we = _parse_time(payload.window_end)
policy = ValidationPolicy(
name=payload.name,
environment_id=payload.environment_id,
is_active=payload.is_active,
dashboard_ids=payload.dashboard_ids or [],
provider_id=payload.provider_id,
schedule_days=payload.schedule_days or [],
window_start=ws or time(9, 0),
window_end=we or time(18, 0),
notify_owners=payload.notify_owners,
custom_channels=payload.custom_channels,
alert_condition=payload.alert_condition or "FAIL_ONLY",
)
self.db.add(policy)
self.db.commit()
self.db.refresh(policy)
logger.info(
"[ValidationTaskService] Created task '%s' (id=%s)", policy.name, policy.id
)
return self._policy_to_response(policy)
# #endregion create_task
# #region get_task [C:3] [TYPE Function]
# @RATIONALE Returns full task response including last_run_status via joined subquery — gives the UI immediate visibility into task health without a second API call.
# @REJECTED Separate query for last run status rejected — double-query pattern would add latency on every task detail request.
def get_task(self, task_id: str) -> ValidationTaskResponse:
policy = self.db.query(ValidationPolicy).filter(ValidationPolicy.id == task_id).first()
if not policy:
raise ValueError(f"Validation task '{task_id}' not found")
return self._policy_to_response(policy)
# #endregion get_task
# #region update_task [C:4] [TYPE Function]
# @RATIONALE Re-validates provider/environment only when those fields change (via exclude_unset), avoiding unnecessary DB calls on partial updates and allowing other fields to be updated independently.
# @REJECTED Full re-validation on every update rejected — would prevent updating non-critical fields (e.g., schedule, name) when a provider is temporarily unavailable.
def update_task(self, task_id: str, payload: ValidationTaskUpdate) -> ValidationTaskResponse:
policy = self.db.query(ValidationPolicy).filter(ValidationPolicy.id == task_id).first()
if not policy:
raise ValueError(f"Validation task '{task_id}' not found")
update_data = payload.model_dump(exclude_unset=True)
if "provider_id" in update_data and update_data["provider_id"]:
self._validate_provider(update_data["provider_id"])
if "environment_id" in update_data and update_data["environment_id"]:
self._validate_environment(update_data["environment_id"])
if "window_start" in update_data:
ws = _parse_time(update_data.pop("window_start", None))
if ws is not None:
policy.window_start = ws
if "window_end" in update_data:
we = _parse_time(update_data.pop("window_end", None))
if we is not None:
policy.window_end = we
for key, value in update_data.items():
if hasattr(policy, key):
setattr(policy, key, value)
self.db.commit()
self.db.refresh(policy)
logger.info("[ValidationTaskService] Updated task '%s' (id=%s)", policy.name, policy.id)
return self._policy_to_response(policy)
# #endregion update_task
# #region delete_task [C:3] [TYPE Function]
# @RATIONALE Scope run deletion by policy_id instead of dashboard_id+environment_id to prevent deleting runs belonging to other tasks that validate the same dashboard. Requires policy_id column on ValidationRecord, set by the plugin during execution.
# @REJECTED Always-cascade delete rejected would destroy run history unexpectedly. Scoping by dashboard_id+environment_id rejected caused cross-task data loss (F4). Explicit delete_runs flag with policy_id scoping gives users safe control.
def delete_task(self, task_id: str, delete_runs: bool = False) -> None:
policy = self.db.query(ValidationPolicy).filter(ValidationPolicy.id == task_id).first()
if not policy:
raise ValueError(f"Validation task '{task_id}' not found")
if delete_runs:
self.db.query(ValidationRecord).filter(
ValidationRecord.policy_id == policy.id,
).delete(synchronize_session=False)
self.db.delete(policy)
self.db.commit()
logger.info(
"[ValidationTaskService] Deleted task '%s' (id=%s)", policy.name, policy.id
)
# #endregion delete_task
# #region trigger_run [C:4] [TYPE Function]
# @RATIONALE Uses TaskManager.create_task() to spawn validation via the llm_dashboard_validation plugin — reuses existing scheduling, persistence, retry, and notification infrastructure.
# @REJECTED Direct LLM provider call from service layer rejected — would bypass TaskManager's persistence, retry logic, logging pipeline, and WebSocket notifications.
async def trigger_run(self, task_id: str, task_manager: TaskManager) -> str:
policy = self.db.query(ValidationPolicy).filter(ValidationPolicy.id == task_id).first()
if not policy:
raise ValueError(f"Validation task '{task_id}' not found")
if not policy.dashboard_ids:
raise ValueError(f"Validation task '{task_id}' has no dashboard IDs configured")
if policy.provider_id:
self._validate_provider(policy.provider_id)
dashboard_id = policy.dashboard_ids[0]
params: dict[str, Any] = {
"dashboard_id": dashboard_id,
"environment_id": policy.environment_id,
"provider_id": policy.provider_id or "",
"policy_id": task_id,
}
task = await task_manager.create_task(plugin_id="llm_dashboard_validation", params=params)
logger.info(
"[ValidationTaskService] Triggered run for task '%s' dashboard=%s spawned=%s",
policy.name, dashboard_id, task.id,
)
return task.id
# #endregion trigger_run
# #endregion ValidationTaskService
# #endregion ValidationService