feat: LLM Validation section — 9 API endpoints + 3 Svelte pages + Playwright optimization
Backend: - /api/validation/ CRUD for validation tasks (policies) - Run history with 6 filters + pagination - Alembic migrations (provider_id, policy_id) - 28 pytest tests Frontend: - /validation — task list page with status filters + CRUD - /validation/[id] — task config (Config + History tabs) - /validation/history — run history with metrics - Sidebar nav entry under Operations - i18n en/ru Playwright (ScreenshotService): - Removed 25s hardcoded sleeps → conditional waits - CDP fallback to full_page screenshot - Total timeout enforcement (120s) - Debug screenshots → temp dir with cleanup QA fixes: - Debug screenshot accumulation in production (tempdir + rmtree) - Frontend API payload field name mismatch - History issues field reference fix - delete_runs scoped by policy_id Belief protocol: - @RATIONALE/@REJECTED on 27 C4+ contracts Closes: VALIDATION-REWORK-2026
This commit is contained in:
@@ -37,6 +37,7 @@ __all__ = [
|
||||
"storage",
|
||||
"tasks",
|
||||
"translate",
|
||||
"validation",
|
||||
]
|
||||
# #endregion Route_Group_Contracts
|
||||
|
||||
|
||||
270
backend/src/api/routes/validation.py
Normal file
270
backend/src/api/routes/validation.py
Normal file
@@ -0,0 +1,270 @@
|
||||
# #region ValidationRoutes [C:3] [TYPE Module] [SEMANTICS fastapi, validation, api, task, run]
|
||||
# @BRIEF API routes for validation task management and run history.
|
||||
# @LAYER API
|
||||
# @RELATION DEPENDS_ON -> [ValidationTaskService]
|
||||
# @RELATION DEPENDS_ON -> [ValidationRunService]
|
||||
# @RELATION DEPENDS_ON -> [TaskManager]
|
||||
|
||||
from datetime import date
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...core.config_manager import ConfigManager
|
||||
from ...core.database import get_db
|
||||
from ...core.logger import logger
|
||||
from ...core.task_manager import TaskManager
|
||||
from ...dependencies import get_config_manager, get_current_user, get_task_manager, has_permission
|
||||
from ...schemas.auth import User
|
||||
from ...schemas.validation import (
|
||||
TriggerRunResponse,
|
||||
ValidationRunDetailResponse,
|
||||
ValidationRunListResponse,
|
||||
ValidationTaskCreate,
|
||||
ValidationTaskListResponse,
|
||||
ValidationTaskResponse,
|
||||
ValidationTaskUpdate,
|
||||
)
|
||||
from ...services.validation_run_service import ValidationRunService
|
||||
from ...services.validation_service import ValidationTaskService
|
||||
|
||||
# #region router [C:1] [TYPE Global]
|
||||
router = APIRouter(tags=["Validation"])
|
||||
# #endregion router
|
||||
|
||||
|
||||
# #region _get_task_service [C:1] [TYPE Function]
|
||||
def _get_task_service(
|
||||
db: Session = Depends(get_db),
|
||||
config_manager: ConfigManager = Depends(get_config_manager),
|
||||
current_user: User = Depends(get_current_user),
|
||||
) -> ValidationTaskService:
|
||||
return ValidationTaskService(db=db, config_manager=config_manager, username=current_user.username)
|
||||
# #endregion _get_task_service
|
||||
|
||||
|
||||
# #region _get_run_service [C:1] [TYPE Function]
|
||||
def _get_run_service(
|
||||
db: Session = Depends(get_db),
|
||||
) -> ValidationRunService:
|
||||
return ValidationRunService(db=db)
|
||||
# #endregion _get_run_service
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Tasks CRUD
|
||||
# ============================================================
|
||||
|
||||
|
||||
# #region list_tasks [C:4] [TYPE Function]
|
||||
# @RATIONALE Route handler delegates to service with permission check — keeps API layer thin and testable; auth gate happens before any business logic.
|
||||
# @REJECTED Business logic in route handler rejected — would duplicate validation logic and make permission checks harder to audit.
|
||||
@router.get("/tasks", response_model=ValidationTaskListResponse)
|
||||
async def list_tasks(
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(20, ge=1, le=100),
|
||||
is_active: bool | None = Query(None),
|
||||
environment_id: str | None = Query(None),
|
||||
current_user: User = Depends(get_current_user),
|
||||
_=Depends(has_permission("validation.task", "VIEW")),
|
||||
service: ValidationTaskService = Depends(_get_task_service),
|
||||
):
|
||||
"""List all validation tasks with pagination and optional filters."""
|
||||
logger.reason(f"list_tasks — User: {current_user.username}", extra={"src": "validation_routes"})
|
||||
try:
|
||||
total, items = service.list_tasks(is_active=is_active, environment_id=environment_id, page=page, page_size=page_size)
|
||||
return ValidationTaskListResponse(items=items, total=total, page=page, page_size=page_size)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
# #endregion list_tasks
|
||||
|
||||
|
||||
# #region create_task [C:4] [TYPE Function]
|
||||
# @RATIONALE Returns 201 with created resource — follows REST conventions for POST resource creation; signals to clients that a new entity was created.
|
||||
# @REJECTED Returning 200 with wrapped response rejected — non-standard for POST create; 201 Created signals the correct semantics to HTTP clients and code generators.
|
||||
@router.post("/tasks", response_model=ValidationTaskResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def create_task(
|
||||
payload: ValidationTaskCreate,
|
||||
current_user: User = Depends(get_current_user),
|
||||
_=Depends(has_permission("validation.task", "CREATE")),
|
||||
service: ValidationTaskService = Depends(_get_task_service),
|
||||
):
|
||||
"""Create a new validation task with provider and environment validation."""
|
||||
logger.reason(f"create_task — User: {current_user.username}, name: {payload.name}", extra={"src": "validation_routes"})
|
||||
try:
|
||||
return service.create_task(payload)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))
|
||||
# #endregion create_task
|
||||
|
||||
|
||||
# #region get_task [C:4] [TYPE Function]
|
||||
# @RATIONALE Joins task detail with recent runs in a single endpoint — reduces frontend API calls from 2 to 1, improving page load performance.
|
||||
# @REJECTED Separate /tasks/{id}/runs endpoint for recent runs rejected — would require frontend to make 2 sequential API calls, doubling load time on the config page.
|
||||
@router.get("/tasks/{task_id}", response_model=dict)
|
||||
async def get_task(
|
||||
task_id: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
_=Depends(has_permission("validation.task", "VIEW")),
|
||||
task_service: ValidationTaskService = Depends(_get_task_service),
|
||||
run_service: ValidationRunService = Depends(_get_run_service),
|
||||
):
|
||||
"""Get a validation task with its recent runs."""
|
||||
logger.reason(f"get_task — Task: {task_id}, User: {current_user.username}", extra={"src": "validation_routes"})
|
||||
try:
|
||||
task = task_service.get_task(task_id)
|
||||
from ...schemas.validation import ValidationRunResponse
|
||||
# Find recent runs via run_service using dashboard matching
|
||||
recent = run_service.list_runs(
|
||||
dashboard_id=task.dashboard_ids[0] if task.dashboard_ids else None,
|
||||
environment_id=task.environment_id,
|
||||
page=1,
|
||||
page_size=10,
|
||||
)
|
||||
return {
|
||||
"task": task.model_dump(),
|
||||
"recent_runs": [r.model_dump() for r in recent[1]],
|
||||
}
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
|
||||
# #endregion get_task
|
||||
|
||||
|
||||
# #region update_task [C:4] [TYPE Function]
|
||||
# @RATIONALE PUT semantics for full resource update — consistent with existing API patterns; Pydantic's exclude_unset handles partial updates while keeping PUT as the endpoint verb.
|
||||
# @REJECTED PATCH semantics rejected — using PUT with exclude_unset provides the same partial-update flexibility without introducing a second HTTP method for the same resource.
|
||||
@router.put("/tasks/{task_id}", response_model=ValidationTaskResponse)
|
||||
async def update_task(
|
||||
task_id: str,
|
||||
payload: ValidationTaskUpdate,
|
||||
current_user: User = Depends(get_current_user),
|
||||
_=Depends(has_permission("validation.task", "EDIT")),
|
||||
service: ValidationTaskService = Depends(_get_task_service),
|
||||
):
|
||||
"""Update a validation task."""
|
||||
logger.reason(f"update_task — Task: {task_id}, User: {current_user.username}", extra={"src": "validation_routes"})
|
||||
try:
|
||||
return service.update_task(task_id, payload)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
|
||||
# #endregion update_task
|
||||
|
||||
|
||||
# #region delete_task [C:4] [TYPE Function]
|
||||
# @RATIONALE DELETE returns 204 No Content per REST convention — minimizes response payload for delete operations; supports optional delete_runs query param for cascade control.
|
||||
# @REJECTED Returning deleted resource in response body rejected — unnecessary data transfer; 204 with no body is the REST standard for delete operations.
|
||||
@router.delete("/tasks/{task_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_task(
|
||||
task_id: str,
|
||||
delete_runs: bool = Query(False, description="Also delete associated validation runs"),
|
||||
current_user: User = Depends(get_current_user),
|
||||
_=Depends(has_permission("validation.task", "DELETE")),
|
||||
service: ValidationTaskService = Depends(_get_task_service),
|
||||
):
|
||||
"""Delete a validation task."""
|
||||
logger.reason(f"delete_task — Task: {task_id}, User: {current_user.username}", extra={"src": "validation_routes"})
|
||||
try:
|
||||
service.delete_task(task_id, delete_runs=delete_runs)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
|
||||
# #endregion delete_task
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Run — trigger immediate validation
|
||||
# ============================================================
|
||||
|
||||
|
||||
# #region trigger_run [C:4] [TYPE Function]
|
||||
# @RATIONALE Returns spawned task ID so frontend can poll for completion — enables progress tracking without WebSocket. Async spawn is essential since LLM validation can take 30+ seconds.
|
||||
# @REJECTED Blocking until run completes rejected — LLM-based validation is slow (30-120s); synchronous wait would timeout HTTP requests and create terrible UX.
|
||||
@router.post("/tasks/{task_id}/run", response_model=TriggerRunResponse)
|
||||
async def trigger_run(
|
||||
task_id: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
_=Depends(has_permission("validation.task", "EXECUTE")),
|
||||
service: ValidationTaskService = Depends(_get_task_service),
|
||||
task_manager: TaskManager = Depends(get_task_manager),
|
||||
):
|
||||
"""Trigger immediate validation for a task."""
|
||||
logger.reason(f"trigger_run — Task: {task_id}, User: {current_user.username}", extra={"src": "validation_routes"})
|
||||
try:
|
||||
spawned_task_id = await service.trigger_run(task_id, task_manager)
|
||||
return TriggerRunResponse(task_id=task_id, spawned_task_id=spawned_task_id, status="PENDING", message="Validation task spawned")
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))
|
||||
# #endregion trigger_run
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Runs — history listing and detail
|
||||
# ============================================================
|
||||
|
||||
|
||||
# #region list_runs [C:4] [TYPE Function]
|
||||
# @RATIONALE Exposes 6 query parameters for cross-task run filtering — matches the run service's filter capabilities exactly, enabling flexible history exploration.
|
||||
# @REJECTED Fixed filter set rejected — would limit the history page's debugging use cases; users need to filter by task, status, dashboard, environment, and date range.
|
||||
@router.get("/runs", response_model=ValidationRunListResponse)
|
||||
async def list_runs(
|
||||
task_id: str | None = Query(None, description="Filter by spawned task ID"),
|
||||
dashboard_id: str | None = Query(None, description="Filter by dashboard ID"),
|
||||
environment_id: str | None = Query(None, description="Filter by environment ID"),
|
||||
status: str | None = Query(None, description="Filter by status: PASS, WARN, FAIL, UNKNOWN"),
|
||||
date_from: date | None = Query(None, description="Start date (ISO format)"),
|
||||
date_to: date | None = Query(None, description="End date (ISO format)"),
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(20, ge=1, le=100),
|
||||
current_user: User = Depends(get_current_user),
|
||||
_=Depends(has_permission("validation.run", "VIEW")),
|
||||
service: ValidationRunService = Depends(_get_run_service),
|
||||
):
|
||||
"""List validation runs with cross-task filtering and pagination."""
|
||||
logger.reason(f"list_runs — User: {current_user.username}", extra={"src": "validation_routes"})
|
||||
try:
|
||||
total, items = service.list_runs(
|
||||
task_id=task_id, dashboard_id=dashboard_id, environment_id=environment_id,
|
||||
status=status, date_from=date_from, date_to=date_to, page=page, page_size=page_size,
|
||||
)
|
||||
return ValidationRunListResponse(items=items, total=total, page=page, page_size=page_size)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
# #endregion list_runs
|
||||
|
||||
|
||||
# #region get_run_detail [C:4] [TYPE Function]
|
||||
# @RATIONALE Returns full run detail including issues list and raw response for the report view — single endpoint serves the complete detail page without multiple fetches.
|
||||
# @REJECTED Paginated issues endpoint rejected — issues per run are typically < 20 items; pagination overhead is unnecessary for such small collections.
|
||||
@router.get("/runs/{run_id}", response_model=ValidationRunDetailResponse)
|
||||
async def get_run_detail(
|
||||
run_id: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
_=Depends(has_permission("validation.run", "VIEW")),
|
||||
service: ValidationRunService = Depends(_get_run_service),
|
||||
):
|
||||
"""Get detailed validation run info including issues and raw response."""
|
||||
logger.reason(f"get_run_detail — Run: {run_id}, User: {current_user.username}", extra={"src": "validation_routes"})
|
||||
try:
|
||||
return service.get_run_detail(run_id)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
|
||||
# #endregion get_run_detail
|
||||
|
||||
|
||||
# #region delete_run [C:3] [TYPE Function]
|
||||
# @RATIONALE Returns 404 when run not found — explicit error handling rather than silent success; helps frontend distinguish between successful delete and missing resource.
|
||||
# @REJECTED Silent success on missing run rejected — would mask bugs in the UI that reference already-deleted runs; explicit 404 helps debugging.
|
||||
@router.delete("/runs/{run_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_run(
|
||||
run_id: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
_=Depends(has_permission("validation.run", "DELETE")),
|
||||
service: ValidationRunService = Depends(_get_run_service),
|
||||
):
|
||||
"""Delete a validation run record."""
|
||||
logger.reason(f"delete_run — Run: {run_id}, User: {current_user.username}", extra={"src": "validation_routes"})
|
||||
if not service.delete_run(run_id):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Run not found")
|
||||
# #endregion delete_run
|
||||
|
||||
|
||||
# #endregion ValidationRoutes
|
||||
@@ -0,0 +1,550 @@
|
||||
# #region ValidationApiTests [C:3] [TYPE Module] [SEMANTICS validation, api, tests, pagination, crud]
|
||||
# @BRIEF Unit tests for validation task CRUD and run history API endpoints.
|
||||
# @LAYER: API
|
||||
# @RELATION BINDS_TO -> [ValidationRoutes]
|
||||
# @RELATION DEPENDS_ON -> [EXT:FastAPI:TestClient]
|
||||
# @TEST_CONTRACT: [ValidationTaskCreate|Update|RunRequest] -> [ValidationTaskResponse|RunResponse|TriggerRunResponse]
|
||||
# @TEST_SCENARIO: list_tasks -> 200 with paginated tasks; filter by is_active/environment_id
|
||||
# @TEST_SCENARIO: create_task -> 201 with valid payload; 422 with invalid provider/environment
|
||||
# @TEST_SCENARIO: get_task -> 200 with task+recent_runs; 404 for nonexistent
|
||||
# @TEST_SCENARIO: update_task -> 200 with updated fields; 404 for nonexistent
|
||||
# @TEST_SCENARIO: delete_task -> 204 on success; 404 for nonexistent
|
||||
# @TEST_SCENARIO: trigger_run -> 200 with spawned_task_id; 422 for invalid task
|
||||
# @TEST_SCENARIO: list_runs -> 200 with 6 filters + pagination
|
||||
# @TEST_SCENARIO: get_run_detail -> 200 with issues/raw_response; 404 for nonexistent
|
||||
# @TEST_SCENARIO: delete_run -> 204 on success; 404 for nonexistent
|
||||
# @TEST_EDGE: missing_field -> 422 when required field omitted from create payload
|
||||
# @TEST_EDGE: invalid_type -> 422 when provider_id is not multimodal
|
||||
# @TEST_EDGE: external_fail -> 404/422 when task/run does not exist
|
||||
# @TEST_EDGE: delete_task_with_runs -> 204 with delete_runs=true flag
|
||||
# @INVARIANT: Every endpoint requires authentication via has_permission
|
||||
# @INVARIANT: All list endpoints support pagination (page/page_size) with defaults
|
||||
# @INVARIANT: provider_id must reference a multimodal LLM provider for creation
|
||||
|
||||
# Set required env vars before ANY app imports — crash-early guard for AuthConfig()
|
||||
import os
|
||||
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing")
|
||||
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:test_auth")
|
||||
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:test_main")
|
||||
os.environ.setdefault("DEV_MODE", "true")
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from src.app import app
|
||||
from src.dependencies import (
|
||||
get_current_user,
|
||||
get_config_manager,
|
||||
get_task_manager,
|
||||
)
|
||||
from src.api.routes.validation import _get_task_service, _get_run_service
|
||||
from src.schemas.validation import (
|
||||
ValidationTaskListResponse,
|
||||
ValidationTaskResponse,
|
||||
ValidationRunListResponse,
|
||||
ValidationRunDetailResponse,
|
||||
TriggerRunResponse,
|
||||
)
|
||||
from src.models.auth import User
|
||||
|
||||
# ── Shared mocks ───────────────────────────────────────────────────────
|
||||
|
||||
mock_user = MagicMock(spec=User)
|
||||
mock_user.username = "testuser"
|
||||
mock_user.roles = []
|
||||
admin_role = MagicMock()
|
||||
admin_role.name = "Admin"
|
||||
admin_role.permissions = []
|
||||
mock_user.roles.append(admin_role)
|
||||
|
||||
|
||||
def _make_now():
|
||||
"""Return a timezone-aware UTC datetime for Pydantic v2."""
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_all_deps():
|
||||
"""Override service-level FastAPI dependencies with MagicMock stubs.
|
||||
|
||||
Permission checking (has_permission) is NOT overridden — mock_user has
|
||||
Admin role which passes all permission gates implicitly.
|
||||
Fixture resets mock_user.roles before each test to prevent cross-test pollution.
|
||||
"""
|
||||
# Reset user state — Admin role passes all permission gates
|
||||
mock_user.roles = [admin_role]
|
||||
|
||||
mock_task_service = MagicMock()
|
||||
mock_run_service = MagicMock()
|
||||
mock_task_manager = MagicMock()
|
||||
|
||||
app.dependency_overrides[_get_task_service] = lambda: mock_task_service
|
||||
app.dependency_overrides[_get_run_service] = lambda: mock_run_service
|
||||
app.dependency_overrides[get_config_manager] = lambda: MagicMock()
|
||||
app.dependency_overrides[get_task_manager] = lambda: mock_task_manager
|
||||
app.dependency_overrides[get_current_user] = lambda: mock_user
|
||||
|
||||
yield {
|
||||
"task_service": mock_task_service,
|
||||
"run_service": mock_run_service,
|
||||
"task_manager": mock_task_manager,
|
||||
}
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
# ── Fixtures ────────────────────────────────────────────────────────────
|
||||
|
||||
def _make_task_response(**overrides):
|
||||
"""Build a ValidationTaskResponse for mock returns."""
|
||||
return ValidationTaskResponse(
|
||||
id=overrides.get("id", "task-1"),
|
||||
name=overrides.get("name", "Test Task"),
|
||||
environment_id=overrides.get("environment_id", "env-1"),
|
||||
is_active=overrides.get("is_active", True),
|
||||
dashboard_ids=overrides.get("dashboard_ids", ["dash-1"]),
|
||||
provider_id=overrides.get("provider_id", "provider-1"),
|
||||
schedule_days=None,
|
||||
window_start=None,
|
||||
window_end=None,
|
||||
notify_owners=True,
|
||||
custom_channels=None,
|
||||
alert_condition="FAIL_ONLY",
|
||||
created_at=overrides.get("created_at", _make_now()),
|
||||
updated_at=None,
|
||||
last_run_status=None,
|
||||
last_run_at=None,
|
||||
)
|
||||
|
||||
|
||||
def _make_run_response(**overrides):
|
||||
"""Build a dict representation of a ValidationRunResponse."""
|
||||
return {
|
||||
"id": overrides.get("id", "run-1"),
|
||||
"task_id": overrides.get("task_id", "task-1"),
|
||||
"dashboard_id": overrides.get("dashboard_id", "dash-1"),
|
||||
"environment_id": overrides.get("environment_id", "env-1"),
|
||||
"status": overrides.get("status", "PASS"),
|
||||
"summary": overrides.get("summary", ""),
|
||||
"timestamp": overrides.get("timestamp", _make_now()),
|
||||
"screenshot_path": None,
|
||||
"task_name": overrides.get("task_name", "Test Task"),
|
||||
}
|
||||
|
||||
|
||||
# ── Task CRUD Tests ────────────────────────────────────────────────────
|
||||
|
||||
# #region test_list_tasks_success [C:2] [TYPE Function]
|
||||
# @BRIEF GET /validation/tasks returns paginated task list with filters.
|
||||
def test_list_tasks_success(mock_all_deps):
|
||||
task = _make_task_response()
|
||||
mock_all_deps["task_service"].list_tasks.return_value = (1, [task])
|
||||
|
||||
response = client.get("/api/validation/tasks")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["items"][0]["name"] == "Test Task"
|
||||
assert data["total"] == 1
|
||||
assert data["page"] == 1
|
||||
assert data["page_size"] == 20
|
||||
ValidationTaskListResponse(**data)
|
||||
|
||||
# environment_id filter
|
||||
response = client.get("/api/validation/tasks?environment_id=env-1")
|
||||
assert response.status_code == 200
|
||||
mock_all_deps["task_service"].list_tasks.assert_called_with(
|
||||
is_active=None, environment_id="env-1", page=1, page_size=20
|
||||
)
|
||||
# #endregion test_list_tasks_success
|
||||
|
||||
|
||||
# #region test_list_tasks_pagination [C:2] [TYPE Function]
|
||||
# @BRIEF Pagination query params are validated: page >= 1, page_size 1-100.
|
||||
def test_list_tasks_pagination(mock_all_deps):
|
||||
response = client.get("/api/validation/tasks?page=0")
|
||||
assert response.status_code == 422
|
||||
|
||||
response = client.get("/api/validation/tasks?page_size=101")
|
||||
assert response.status_code == 422
|
||||
|
||||
response = client.get("/api/validation/tasks?page_size=0")
|
||||
assert response.status_code == 422
|
||||
# #endregion test_list_tasks_pagination
|
||||
|
||||
|
||||
# #region test_list_tasks_service_error [C:2] [TYPE Function]
|
||||
# @BRIEF Service ValueError becomes 400 on list_tasks.
|
||||
def test_list_tasks_service_error(mock_all_deps):
|
||||
mock_all_deps["task_service"].list_tasks.side_effect = ValueError("Bad filter")
|
||||
response = client.get("/api/validation/tasks?environment_id=nonexistent")
|
||||
assert response.status_code == 400
|
||||
assert "Bad filter" in response.json()["detail"]
|
||||
# #endregion test_list_tasks_service_error
|
||||
|
||||
|
||||
# #region test_create_task_success [C:2] [TYPE Function]
|
||||
# @BRIEF POST /validation/tasks with valid payload returns 201.
|
||||
def test_create_task_success(mock_all_deps):
|
||||
mock_all_deps["task_service"].create_task.return_value = _make_task_response()
|
||||
|
||||
payload = {
|
||||
"name": "My Validation Task",
|
||||
"environment_id": "env-1",
|
||||
"dashboard_ids": ["dash-1"],
|
||||
"provider_id": "provider-1",
|
||||
"notify_owners": True,
|
||||
"alert_condition": "FAIL_ONLY",
|
||||
}
|
||||
response = client.post("/api/validation/tasks", json=payload)
|
||||
assert response.status_code == 201
|
||||
data = response.json()
|
||||
assert data["name"] == "Test Task"
|
||||
assert data["id"] == "task-1"
|
||||
ValidationTaskResponse(**data)
|
||||
mock_all_deps["task_service"].create_task.assert_called_once()
|
||||
# #endregion test_create_task_success
|
||||
|
||||
|
||||
# #region test_create_task_missing_fields [C:2] [TYPE Function]
|
||||
# @BRIEF POST with missing required fields returns 422.
|
||||
def test_create_task_missing_fields(mock_all_deps):
|
||||
# Empty payload
|
||||
response = client.post("/api/validation/tasks", json={})
|
||||
assert response.status_code == 422
|
||||
errors = response.json()
|
||||
assert "detail" in errors
|
||||
|
||||
# Missing provider_id
|
||||
response = client.post("/api/validation/tasks", json={
|
||||
"name": "Test", "environment_id": "env-1", "dashboard_ids": ["dash-1"],
|
||||
})
|
||||
assert response.status_code == 422
|
||||
|
||||
# Missing dashboard_ids
|
||||
response = client.post("/api/validation/tasks", json={
|
||||
"name": "Test", "environment_id": "env-1", "provider_id": "provider-1",
|
||||
})
|
||||
assert response.status_code == 422
|
||||
# #endregion test_create_task_missing_fields
|
||||
|
||||
|
||||
# #region test_create_task_invalid_provider [C:2] [TYPE Function]
|
||||
# @BRIEF POST with non-multimodal provider returns 422.
|
||||
def test_create_task_invalid_provider(mock_all_deps):
|
||||
mock_all_deps["task_service"].create_task.side_effect = ValueError(
|
||||
"LLM provider 'GPT-3.5' is not multimodal"
|
||||
)
|
||||
payload = {
|
||||
"name": "Test", "environment_id": "env-1",
|
||||
"dashboard_ids": ["dash-1"], "provider_id": "provider-2",
|
||||
}
|
||||
response = client.post("/api/validation/tasks", json=payload)
|
||||
assert response.status_code == 422
|
||||
assert "not multimodal" in response.json()["detail"]
|
||||
# #endregion test_create_task_invalid_provider
|
||||
|
||||
|
||||
# #region test_get_task_success [C:2] [TYPE Function]
|
||||
# @BRIEF GET /validation/tasks/{id} returns task with recent_runs.
|
||||
def test_get_task_success(mock_all_deps):
|
||||
mock_all_deps["task_service"].get_task.return_value = _make_task_response()
|
||||
mock_all_deps["run_service"].list_runs.return_value = (0, [])
|
||||
|
||||
response = client.get("/api/validation/tasks/task-1")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "task" in data
|
||||
assert "recent_runs" in data
|
||||
assert data["task"]["name"] == "Test Task"
|
||||
mock_all_deps["task_service"].get_task.assert_called_with("task-1")
|
||||
# #endregion test_get_task_success
|
||||
|
||||
|
||||
# #region test_get_task_not_found [C:2] [TYPE Function]
|
||||
# @BRIEF GET for nonexistent task returns 404.
|
||||
def test_get_task_not_found(mock_all_deps):
|
||||
mock_all_deps["task_service"].get_task.side_effect = ValueError("not found")
|
||||
response = client.get("/api/validation/tasks/nonexistent")
|
||||
assert response.status_code == 404
|
||||
# #endregion test_get_task_not_found
|
||||
|
||||
|
||||
# #region test_update_task_success [C:2] [TYPE Function]
|
||||
# @BRIEF PUT /validation/tasks/{id} updates task and returns updated object.
|
||||
def test_update_task_success(mock_all_deps):
|
||||
updated = _make_task_response(name="Updated Name", is_active=False)
|
||||
mock_all_deps["task_service"].update_task.return_value = updated
|
||||
|
||||
payload = {"name": "Updated Name", "is_active": False}
|
||||
response = client.put("/api/validation/tasks/task-1", json=payload)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["name"] == "Updated Name"
|
||||
assert data["is_active"] is False
|
||||
ValidationTaskResponse(**data)
|
||||
mock_all_deps["task_service"].update_task.assert_called_once()
|
||||
# #endregion test_update_task_success
|
||||
|
||||
|
||||
# #region test_update_task_not_found [C:2] [TYPE Function]
|
||||
# @BRIEF PUT for nonexistent task returns 404.
|
||||
def test_update_task_not_found(mock_all_deps):
|
||||
mock_all_deps["task_service"].update_task.side_effect = ValueError("not found")
|
||||
payload = {"name": "Nope"}
|
||||
response = client.put("/api/validation/tasks/nonexistent", json=payload)
|
||||
assert response.status_code == 404
|
||||
# #endregion test_update_task_not_found
|
||||
|
||||
|
||||
# #region test_delete_task_success [C:2] [TYPE Function]
|
||||
# @BRIEF DELETE /validation/tasks/{id} returns 204.
|
||||
def test_delete_task_success(mock_all_deps):
|
||||
mock_all_deps["task_service"].delete_task.return_value = None
|
||||
response = client.delete("/api/validation/tasks/task-1")
|
||||
assert response.status_code == 204
|
||||
mock_all_deps["task_service"].delete_task.assert_called_with("task-1", delete_runs=False)
|
||||
# #endregion test_delete_task_success
|
||||
|
||||
|
||||
# #region test_delete_task_not_found [C:2] [TYPE Function]
|
||||
# @BRIEF DELETE for nonexistent task returns 404.
|
||||
def test_delete_task_not_found(mock_all_deps):
|
||||
mock_all_deps["task_service"].delete_task.side_effect = ValueError("not found")
|
||||
response = client.delete("/api/validation/tasks/nonexistent")
|
||||
assert response.status_code == 404
|
||||
# #endregion test_delete_task_not_found
|
||||
|
||||
|
||||
# #region test_delete_task_with_runs_flag [C:2] [TYPE Function]
|
||||
# @BRIEF DELETE with delete_runs=true passes query param to service.
|
||||
def test_delete_task_with_runs_flag(mock_all_deps):
|
||||
mock_all_deps["task_service"].delete_task.return_value = None
|
||||
response = client.delete("/api/validation/tasks/task-1?delete_runs=true")
|
||||
assert response.status_code == 204
|
||||
mock_all_deps["task_service"].delete_task.assert_called_with("task-1", delete_runs=True)
|
||||
# #endregion test_delete_task_with_runs_flag
|
||||
|
||||
|
||||
# #region test_trigger_run_success [C:2] [TYPE Function]
|
||||
# @BRIEF POST /validation/tasks/{id}/run spawns a validation task.
|
||||
def test_trigger_run_success(mock_all_deps):
|
||||
mock_all_deps["task_service"].trigger_run = AsyncMock(return_value="spawned-1")
|
||||
mock_all_deps["task_manager"].create_task = AsyncMock()
|
||||
|
||||
response = client.post("/api/validation/tasks/task-1/run")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["task_id"] == "task-1"
|
||||
assert data["spawned_task_id"] == "spawned-1"
|
||||
assert data["status"] == "PENDING"
|
||||
TriggerRunResponse(**data)
|
||||
# #endregion test_trigger_run_success
|
||||
|
||||
|
||||
# #region test_trigger_run_not_found [C:2] [TYPE Function]
|
||||
# @BRIEF POST for nonexistent task returns 422.
|
||||
def test_trigger_run_not_found(mock_all_deps):
|
||||
mock_all_deps["task_service"].trigger_run = AsyncMock(
|
||||
side_effect=ValueError("Validation task 'nonexistent' not found")
|
||||
)
|
||||
response = client.post("/api/validation/tasks/nonexistent/run")
|
||||
assert response.status_code == 422
|
||||
assert "not found" in response.json()["detail"]
|
||||
# #endregion test_trigger_run_not_found
|
||||
|
||||
|
||||
# #region test_trigger_run_no_dashboards [C:2] [TYPE Function]
|
||||
# @BRIEF Trigger fails with 422 when task has no dashboard IDs.
|
||||
def test_trigger_run_no_dashboards(mock_all_deps):
|
||||
mock_all_deps["task_service"].trigger_run = AsyncMock(
|
||||
side_effect=ValueError("has no dashboard IDs configured")
|
||||
)
|
||||
response = client.post("/api/validation/tasks/task-1/run")
|
||||
assert response.status_code == 422
|
||||
assert "no dashboard IDs" in response.json()["detail"]
|
||||
# #endregion test_trigger_run_no_dashboards
|
||||
|
||||
|
||||
# ── Run History Tests ──────────────────────────────────────────────────
|
||||
|
||||
# #region test_list_runs_success [C:2] [TYPE Function]
|
||||
# @BRIEF GET /validation/runs returns paginated run list.
|
||||
def test_list_runs_success(mock_all_deps):
|
||||
run = _make_run_response()
|
||||
mock_all_deps["run_service"].list_runs.return_value = (1, [run])
|
||||
|
||||
response = client.get("/api/validation/runs")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["items"][0]["id"] == "run-1"
|
||||
assert data["total"] == 1
|
||||
ValidationRunListResponse(**data)
|
||||
# #endregion test_list_runs_success
|
||||
|
||||
|
||||
# #region test_list_runs_filters [C:2] [TYPE Function]
|
||||
# @BRIEF All 6 filter query params are accepted and forwarded to service.
|
||||
def test_list_runs_filters(mock_all_deps):
|
||||
mock_all_deps["run_service"].list_runs.return_value = (0, [])
|
||||
|
||||
for query in [
|
||||
"?task_id=task-1",
|
||||
"?dashboard_id=dash-1",
|
||||
"?environment_id=env-1",
|
||||
"?status=PASS",
|
||||
"?date_from=2026-01-01",
|
||||
"?date_to=2026-12-31",
|
||||
"?status=PASS&dashboard_id=dash-1&date_from=2026-01-01",
|
||||
]:
|
||||
response = client.get(f"/api/validation/runs{query}")
|
||||
assert response.status_code == 200, f"Filter query failed: {query}"
|
||||
# #endregion test_list_runs_filters
|
||||
|
||||
|
||||
# #region test_list_runs_pagination [C:2] [TYPE Function]
|
||||
# @BRIEF Pagination boundary checks on runs list.
|
||||
def test_list_runs_pagination(mock_all_deps):
|
||||
response = client.get("/api/validation/runs?page=0")
|
||||
assert response.status_code == 422
|
||||
|
||||
response = client.get("/api/validation/runs?page_size=101")
|
||||
assert response.status_code == 422
|
||||
# #endregion test_list_runs_pagination
|
||||
|
||||
|
||||
# #region test_list_runs_invalid_status [C:2] [TYPE Function]
|
||||
# @BRIEF Status is a free string; any value passes through to service.
|
||||
def test_list_runs_invalid_status(mock_all_deps):
|
||||
mock_all_deps["run_service"].list_runs.return_value = (0, [])
|
||||
response = client.get("/api/validation/runs?status=INVALID_STATUS_XYZ")
|
||||
assert response.status_code == 200
|
||||
call_kwargs = mock_all_deps["run_service"].list_runs.call_args[1]
|
||||
assert call_kwargs.get("status") == "INVALID_STATUS_XYZ"
|
||||
# #endregion test_list_runs_invalid_status
|
||||
|
||||
|
||||
# #region test_get_run_detail_success [C:2] [TYPE Function]
|
||||
# @BRIEF GET /validation/runs/{id} returns full detail with issues and raw_response.
|
||||
def test_get_run_detail_success(mock_all_deps):
|
||||
mock_all_deps["run_service"].get_run_detail.return_value = ValidationRunDetailResponse(
|
||||
id="run-1",
|
||||
task_id="task-1",
|
||||
dashboard_id="dash-1",
|
||||
environment_id="env-1",
|
||||
status="FAIL",
|
||||
summary="Charts failed to render",
|
||||
timestamp=_make_now(),
|
||||
screenshot_path="/tmp/screenshots/run-1.png",
|
||||
issues=[{"severity": "HIGH", "message": "Chart timeout"}],
|
||||
raw_response='{"status": "error"}',
|
||||
task_name="Test Task",
|
||||
)
|
||||
|
||||
response = client.get("/api/validation/runs/run-1")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "FAIL"
|
||||
assert len(data["issues"]) == 1
|
||||
assert data["raw_response"] is not None
|
||||
ValidationRunDetailResponse(**data)
|
||||
# #endregion test_get_run_detail_success
|
||||
|
||||
|
||||
# #region test_get_run_detail_not_found [C:2] [TYPE Function]
|
||||
# @BRIEF GET for nonexistent run returns 404.
|
||||
def test_get_run_detail_not_found(mock_all_deps):
|
||||
mock_all_deps["run_service"].get_run_detail.side_effect = ValueError("not found")
|
||||
response = client.get("/api/validation/runs/nonexistent")
|
||||
assert response.status_code == 404
|
||||
# #endregion test_get_run_detail_not_found
|
||||
|
||||
|
||||
# #region test_delete_run_success [C:2] [TYPE Function]
|
||||
# @BRIEF DELETE /validation/runs/{id} returns 204.
|
||||
def test_delete_run_success(mock_all_deps):
|
||||
mock_all_deps["run_service"].delete_run.return_value = True
|
||||
response = client.delete("/api/validation/runs/run-1")
|
||||
assert response.status_code == 204
|
||||
mock_all_deps["run_service"].delete_run.assert_called_with("run-1")
|
||||
# #endregion test_delete_run_success
|
||||
|
||||
|
||||
# #region test_delete_run_not_found [C:2] [TYPE Function]
|
||||
# @BRIEF DELETE for nonexistent run returns 404.
|
||||
def test_delete_run_not_found(mock_all_deps):
|
||||
mock_all_deps["run_service"].delete_run.return_value = False
|
||||
response = client.delete("/api/validation/runs/nonexistent")
|
||||
assert response.status_code == 404
|
||||
# #endregion test_delete_run_not_found
|
||||
|
||||
|
||||
# ── Auth / Permission Tests ────────────────────────────────────────────
|
||||
|
||||
# #region test_endpoints_require_authentication [C:2] [TYPE Function]
|
||||
# @BRIEF Without current_user dependency, all 9 endpoints return 401.
|
||||
def test_endpoints_require_authentication(mock_all_deps):
|
||||
app.dependency_overrides.pop(get_current_user, None)
|
||||
endpoints = [
|
||||
("GET", "/api/validation/tasks"),
|
||||
("POST", "/api/validation/tasks"),
|
||||
("GET", "/api/validation/tasks/task-1"),
|
||||
("PUT", "/api/validation/tasks/task-1"),
|
||||
("DELETE", "/api/validation/tasks/task-1"),
|
||||
("POST", "/api/validation/tasks/task-1/run"),
|
||||
("GET", "/api/validation/runs"),
|
||||
("GET", "/api/validation/runs/run-1"),
|
||||
("DELETE", "/api/validation/runs/run-1"),
|
||||
]
|
||||
for method, path in endpoints:
|
||||
response = client.request(method, path)
|
||||
assert response.status_code == 401, f"{method} {path} expected 401 got {response.status_code}"
|
||||
# #endregion test_endpoints_require_authentication
|
||||
|
||||
|
||||
# #region test_permission_denied_non_admin [C:2] [TYPE Function]
|
||||
# @BRIEF A non-admin user without validation.* permissions receives 403.
|
||||
def test_permission_denied_non_admin(mock_all_deps):
|
||||
"""Remove Admin role so has_permission raises 403."""
|
||||
mock_user.roles = [] # No Admin, no permissions
|
||||
response = client.get("/api/validation/tasks")
|
||||
assert response.status_code == 403
|
||||
# #endregion test_permission_denied_non_admin
|
||||
|
||||
|
||||
# #region test_permission_denied_all_actions [C:2] [TYPE Function]
|
||||
# @BRIEF Each action (VIEW, CREATE, EDIT, DELETE, EXECUTE) denied for non-admin.
|
||||
def test_permission_denied_all_actions(mock_all_deps):
|
||||
mock_user.roles = []
|
||||
action_endpoints = [
|
||||
("GET", "/api/validation/tasks"),
|
||||
("POST", "/api/validation/tasks"),
|
||||
("GET", "/api/validation/tasks/task-1"),
|
||||
("PUT", "/api/validation/tasks/task-1"),
|
||||
("DELETE", "/api/validation/tasks/task-1"),
|
||||
("POST", "/api/validation/tasks/task-1/run"),
|
||||
("GET", "/api/validation/runs"),
|
||||
("GET", "/api/validation/runs/run-1"),
|
||||
("DELETE", "/api/validation/runs/run-1"),
|
||||
]
|
||||
for method, path in action_endpoints:
|
||||
response = client.request(method, path)
|
||||
assert response.status_code == 403, f"{method} {path} expected 403 got {response.status_code}"
|
||||
# #endregion test_permission_denied_all_actions
|
||||
|
||||
|
||||
# #region test_delete_task_runs_default_false [C:2] [TYPE Function]
|
||||
# @BRIEF delete_runs query param defaults to False.
|
||||
def test_delete_task_runs_default_false(mock_all_deps):
|
||||
mock_all_deps["task_service"].delete_task.return_value = None
|
||||
response = client.delete("/api/validation/tasks/task-1")
|
||||
assert response.status_code == 204
|
||||
mock_all_deps["task_service"].delete_task.assert_called_with("task-1", delete_runs=False)
|
||||
# #endregion test_delete_task_runs_default_false
|
||||
|
||||
# #endregion ValidationApiTests
|
||||
Reference in New Issue
Block a user