feat(037): verification pipeline automation + GET read-API (T080-T081)
Close the 037 pipeline-automation and read-API gaps found in the audit:
deploy/release hooks did not create VerificationRun, and GET endpoints for
history/detail were absent even though 039 UI and client call them.
T080 - _release_routes.py: create_release now fires best-effort
_trigger_release_verification -> VerificationRun with trigger=release_create
(metric+structure); verification scheduling failures never roll back the
release transaction.
T081 - verification.py: add GET /verification/history (dashboard_id +
environment_id filters, newest-first) and GET /verification/{run_id}
(404 RUN_NOT_FOUND); reuse _record_to_response.
- verification_run.py + alembic migration p2q3r4s5t6u7: nullable indexed
dashboard_id populated from structure/visual/metric category_params.
- verification_service.py: _derive_dashboard_id helper.
Verification: release routes (32) + verification API (8) + persistence (21)
= 53 passed; ruff clean for changed code (pre-existing RUF012/UP017 on old
lines left untouched).
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
# #region Alembic.VerificationRunDashboardId [C:2] [TYPE Module] [SEMANTICS alembic,verification,dashboard,history]
|
||||
# @ingroup Alembic
|
||||
# @BRIEF Add nullable verification_runs.dashboard_id for 037 T081 history filtering by dashboard.
|
||||
# @LAYER Database
|
||||
# @RELATION DEPENDS_ON -> [Models.VerificationRun]
|
||||
# @INVARIANT The column is nullable and indexed — existing runs remain valid and history
|
||||
# filtering by dashboard is a soft filter (runs without dashboard_id excluded only
|
||||
# when the caller filters on dashboard_id).
|
||||
# @RATIONALE VerificationRunRecord previously carried repository/release/environment but no
|
||||
# dashboard identity; frontend getVerificationHistory(dashboardId, envId) needs a
|
||||
# dashboard-scoped query. dashboard_id is populated from category_params at persist.
|
||||
# @REJECTED Encoding dashboard identity into environment_id or release_id was rejected — it is
|
||||
# a distinct dimension and would corrupt existing environment/release semantics.
|
||||
"""add verification_runs.dashboard_id
|
||||
|
||||
Revision ID: p2q3r4s5t6u7
|
||||
Revises: o1p2q3r4s5t6
|
||||
Create Date: 2026-08-07 14:00:00.000000
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "p2q3r4s5t6u7"
|
||||
down_revision: str | Sequence[str] | None = "o1p2q3r4s5t6"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add nullable, indexed verification_runs.dashboard_id."""
|
||||
op.add_column("verification_runs", sa.Column("dashboard_id", sa.Integer(), nullable=True))
|
||||
op.create_index("ix_verification_runs_dashboard", "verification_runs", ["dashboard_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Drop the dashboard_id index and column."""
|
||||
op.drop_index("ix_verification_runs_dashboard", table_name="verification_runs")
|
||||
op.drop_column("verification_runs", "dashboard_id")
|
||||
# #endregion Alembic.VerificationRunDashboardId
|
||||
@@ -1,5 +1,5 @@
|
||||
#region Api.DashboardTesting.VerificationRuns [C:3] [TYPE Module] [SEMANTICS baseline,api,verification-runs,feature-037]
|
||||
# @defgroup Api Verification runs API route — POST /dashboard-testing/verification-runs.
|
||||
# @defgroup Api Verification runs API route — POST create + GET history/detail (037 T081).
|
||||
# @LAYER API
|
||||
# @RELATION DEPENDS_ON -> [BaselineEngine.Verification.Service]
|
||||
# @INVARIANT No SQL, raw endpoint, or raw query_context in request schema.
|
||||
@@ -12,15 +12,20 @@ from sqlalchemy.orm import Session
|
||||
from src.core.database import get_db
|
||||
from src.dependencies import has_permission
|
||||
from src.models.auth import User
|
||||
from src.models.verification_run import VerificationRunRecord
|
||||
from src.schemas.dashboard_testing import (
|
||||
VerificationRun,
|
||||
VerificationRunRequest,
|
||||
)
|
||||
from src.services.dashboard_testing.verification_service import create_verification_run_async
|
||||
from src.services.dashboard_testing.verification_service import (
|
||||
_record_to_response,
|
||||
create_verification_run_async,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/dashboard-testing", tags=["Dashboard-Testing"])
|
||||
|
||||
_WRITE_PERMISSION = Depends(has_permission("dashboard:testing", "WRITE"))
|
||||
_READ_PERMISSION = Depends(has_permission("dashboard:testing", "READ"))
|
||||
_DB_DEPENDENCY = Depends(get_db)
|
||||
|
||||
|
||||
@@ -60,4 +65,48 @@ async def create_verification_run_endpoint(
|
||||
) from err
|
||||
# #endregion Api.DashboardTesting.CreateVerificationRun
|
||||
|
||||
|
||||
# #region Api.DashboardTesting.VerificationHistory [C:3] [TYPE Function] [SEMANTICS baseline,api,verification,history]
|
||||
# @ingroup Api
|
||||
# @BRIEF List verification runs chronologically, optionally filtered by dashboard_id/environment_id (037 T081).
|
||||
# @POST Returns VerificationRun[] ordered by created_at desc; matches frontend getVerificationHistory().
|
||||
# @RELATION CALLS -> [BaselineEngine.Verification.RecordToResponse]
|
||||
# @TEST_EDGE dashboard_filter -> only runs for that dashboard returned.
|
||||
# @TEST_EDGE env_filter -> only runs for that environment returned.
|
||||
@router.get("/verification/history", response_model=list[VerificationRun])
|
||||
def list_verification_history(
|
||||
dashboard_id: int | None = None,
|
||||
environment_id: str | None = None,
|
||||
limit: int = 100,
|
||||
db: Session = _DB_DEPENDENCY, # type: ignore[assignment]
|
||||
_current_user: User = _READ_PERMISSION,
|
||||
) -> list[VerificationRun]:
|
||||
query = db.query(VerificationRunRecord)
|
||||
if dashboard_id is not None:
|
||||
query = query.filter(VerificationRunRecord.dashboard_id == dashboard_id)
|
||||
if environment_id:
|
||||
query = query.filter(VerificationRunRecord.environment_id == environment_id)
|
||||
records = query.order_by(VerificationRunRecord.created_at.desc()).limit(max(1, min(int(limit), 500))).all()
|
||||
return [_record_to_response(r) for r in records]
|
||||
# #endregion Api.DashboardTesting.VerificationHistory
|
||||
|
||||
|
||||
# #region Api.DashboardTesting.VerificationDetail [C:3] [TYPE Function] [SEMANTICS baseline,api,verification,detail]
|
||||
# @ingroup Api
|
||||
# @BRIEF Return a single verification run by id (037 T081).
|
||||
# @POST Returns VerificationRun; 404 when the run does not exist.
|
||||
# @RELATION CALLS -> [BaselineEngine.Verification.RecordToResponse]
|
||||
# @TEST_EDGE missing_run -> 404.
|
||||
@router.get("/verification/{run_id}", response_model=VerificationRun)
|
||||
def get_verification_run(
|
||||
run_id: str,
|
||||
db: Session = _DB_DEPENDENCY, # type: ignore[assignment]
|
||||
_current_user: User = _READ_PERMISSION,
|
||||
) -> VerificationRun:
|
||||
record = db.get(VerificationRunRecord, run_id)
|
||||
if record is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail={"code": "RUN_NOT_FOUND"})
|
||||
return _record_to_response(record)
|
||||
# #endregion Api.DashboardTesting.VerificationDetail
|
||||
|
||||
#endregion Api.DashboardTesting.VerificationRuns
|
||||
|
||||
@@ -6,18 +6,20 @@
|
||||
# @RATIONALE Named releases make the user-visible publication decision auditable and independent of branch names.
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from src.api.routes.git_schemas import (
|
||||
DashboardReleaseSchema,
|
||||
DeployRequest,
|
||||
ReleaseApprovalRequest,
|
||||
ReleaseCreateRequest,
|
||||
ReleasePolicySchema,
|
||||
DeployRequest,
|
||||
)
|
||||
from src.core.database import get_db
|
||||
from src.core.logger import logger
|
||||
from src.dependencies import get_config_manager, get_current_user, has_permission
|
||||
from src.models.auth import User
|
||||
from src.models.dashboard_release import DashboardRelease
|
||||
@@ -191,9 +193,48 @@ async def create_release(
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=409, detail="Release version already exists for this dashboard") from error
|
||||
db.refresh(release)
|
||||
# 037 T080: release_create trigger spawns an automated verification run. Best-effort —
|
||||
# a verification failure must not roll back the release transaction.
|
||||
await _trigger_release_verification(db, repository, release, preprod_environment.id, current_user.username)
|
||||
return release
|
||||
|
||||
|
||||
# #region Api.ReleaseRoutes.TriggerReleaseVerification [C:4] [TYPE Function] [SEMANTICS git,release,verification,t080]
|
||||
# @ingroup Api
|
||||
# @BRIEF Create a VerificationRun with trigger=release_create for the just-created release (037 T080).
|
||||
# @POST Best-effort: on any error logs EXPLORE and returns without raising — release stays valid.
|
||||
# @SIDE_EFFECT Persists a VerificationRunRecord (metric category, structure when evidence absent).
|
||||
# @REJECTED Failing the release on verification-scheduling errors was rejected — verification is
|
||||
# advisory at release_create time; publish gates remain the enforcement point.
|
||||
async def _trigger_release_verification(
|
||||
db: Session,
|
||||
repository: GitRepository,
|
||||
release: DashboardRelease,
|
||||
environment_id: str,
|
||||
created_by: str,
|
||||
) -> None:
|
||||
try:
|
||||
from src.schemas.dashboard_testing import VerificationRunRequest
|
||||
from src.services.dashboard_testing.verification_service import create_verification_run_async
|
||||
|
||||
request = VerificationRunRequest(
|
||||
repository_id=UUID(str(repository.id)),
|
||||
release_id=UUID(str(release.id)),
|
||||
trigger="release_create",
|
||||
environment_id=environment_id,
|
||||
categories=["metric", "structure"],
|
||||
)
|
||||
await create_verification_run_async(db, request, created_by=created_by)
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.explore(
|
||||
"Release-create verification scheduling failed (best-effort)",
|
||||
payload={"release_id": str(release.id)},
|
||||
error=str(exc),
|
||||
)
|
||||
# #endregion Api.ReleaseRoutes.TriggerReleaseVerification
|
||||
|
||||
|
||||
# #endregion Api.ReleaseRoutes.CreateRelease
|
||||
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ from __future__ import annotations
|
||||
from datetime import UTC, datetime
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import JSON, Column, DateTime, ForeignKey, Index, String, Text
|
||||
from sqlalchemy import JSON, Column, DateTime, ForeignKey, Index, Integer, String, Text
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from .dashboard_release import DashboardRelease # noqa: F401 — used by relationship() string lookup
|
||||
@@ -75,6 +75,10 @@ class VerificationRunRecord(Base):
|
||||
)
|
||||
trigger = Column(String, nullable=False)
|
||||
environment_id = Column(String, nullable=False)
|
||||
# Additive 2026-08-07 (037 T081): nullable dashboard_id enables history filtering by
|
||||
# dashboard. Populated from structure/visual category_params at persist; null for runs
|
||||
# that predate the column or that carry no dashboard context (manual text runs).
|
||||
dashboard_id = Column(Integer, nullable=True, index=True)
|
||||
|
||||
# Fan-out plan linkage (041 dataset-lineage-blast-radius, R5/R8): null = ordinary run.
|
||||
# Additive amendment; trigger value "dataset_updated" is data-level (plain String column).
|
||||
|
||||
@@ -109,8 +109,12 @@ class VerificationRunOrchestrator:
|
||||
# ── Derive overall status ──
|
||||
overall_status = _derive_overall_status(outcomes)
|
||||
|
||||
# ── Derive dashboard_id from category params (037 T081) ──
|
||||
dashboard_id = _derive_dashboard_id(category_params)
|
||||
|
||||
record = _persist_verification_run(
|
||||
self.db, request, outcomes, overall_status, agent_run_id, release_id, created_by,
|
||||
dashboard_id=dashboard_id,
|
||||
)
|
||||
log("BaselineEngine.Verification.Orchestrator.Execute", "REFLECT",
|
||||
"Verification run completed",
|
||||
@@ -246,12 +250,27 @@ def _derive_overall_status(outcomes: list[CategoryOutcome]) -> str:
|
||||
# #endregion BaselineEngine.Verification.DeriveStatus
|
||||
|
||||
|
||||
# #region BaselineEngine.Verification.DeriveDashboardId [C:2] [TYPE Function] [SEMANTICS verification,dashboard,derive]
|
||||
# @BRIEF Extract a nullable dashboard_id from category_params (037 T081).
|
||||
# @POST Returns the first non-null dashboard_id found across structure/visual/metric category
|
||||
# params; None when no category carries one.
|
||||
def _derive_dashboard_id(category_params: dict[str, Any]) -> int | None:
|
||||
for key in ("structure", "visual", "metric"):
|
||||
params = category_params.get(key) or {}
|
||||
did = params.get("dashboard_id")
|
||||
if isinstance(did, int) and did > 0:
|
||||
return did
|
||||
return None
|
||||
# #endregion BaselineEngine.Verification.DeriveDashboardId
|
||||
|
||||
|
||||
# #region BaselineEngine.Verification.PersistRun [C:2] [TYPE Function] [SEMANTICS verification,persistence,record]
|
||||
# @BRIEF Build VerificationRunRecord, persist atomically with rollback on failure.
|
||||
# @SIDE_EFFECT DB commit; rolls back on failure and re-raises.
|
||||
def _persist_verification_run(
|
||||
db: Session, request: VerificationRunRequest, outcomes: list[CategoryOutcome],
|
||||
overall_status: str, agent_run_id: str | None, release_id: str | None, created_by: str,
|
||||
dashboard_id: int | None = None,
|
||||
) -> VerificationRunRecord:
|
||||
"""Build record, persist with commit rollback on failure."""
|
||||
categories_run = [o.category for o in outcomes]
|
||||
@@ -261,6 +280,7 @@ def _persist_verification_run(
|
||||
id=None,
|
||||
agent_run_id=agent_run_id, repository_id=str(request.repository_id),
|
||||
release_id=release_id, trigger=request.trigger, environment_id=request.environment_id,
|
||||
dashboard_id=dashboard_id,
|
||||
categories_run=categories_run,
|
||||
category_outcomes=[o.model_dump(mode="json") for o in outcomes],
|
||||
overall_status=overall_status, summary=summary,
|
||||
|
||||
@@ -166,6 +166,77 @@ class TestVerificationRunApi:
|
||||
assert outcome["status"] == "blocked"
|
||||
assert "evidence_refs" in outcome["summary"]
|
||||
# #endregion Test.Api.DashboardTesting.VerificationApi.TestStructureBlocked
|
||||
|
||||
# #region Test.Api.DashboardTesting.VerificationApi.TestHistoryGet [C:2] [TYPE Function]
|
||||
# @BRIEF GET /verification/history returns persisted runs ordered newest-first (037 T081).
|
||||
def test_history_get_returns_runs(
|
||||
self, dashboard_testing_client, dashboard_testing_verification_repository_id: str
|
||||
):
|
||||
from src.core.database import SessionLocal
|
||||
from src.models.verification_run import VerificationRunRecord
|
||||
|
||||
setup = SessionLocal()
|
||||
try:
|
||||
r1 = VerificationRunRecord(
|
||||
repository_id=dashboard_testing_verification_repository_id,
|
||||
trigger="scheduled", environment_id="dev", dashboard_id=42,
|
||||
category_outcomes=[], overall_status="pass", summary="s",
|
||||
created_by="system",
|
||||
)
|
||||
r2 = VerificationRunRecord(
|
||||
repository_id=dashboard_testing_verification_repository_id,
|
||||
trigger="scheduled", environment_id="dev", dashboard_id=42,
|
||||
category_outcomes=[], overall_status="fail", summary="s",
|
||||
created_by="system",
|
||||
)
|
||||
setup.add_all([r1, r2])
|
||||
setup.commit()
|
||||
ids = {r1.id, r2.id}
|
||||
finally:
|
||||
setup.close()
|
||||
|
||||
response = dashboard_testing_client.get(
|
||||
"/api/dashboard-testing/verification/history",
|
||||
params={"dashboard_id": 42, "environment_id": "dev"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
runs = response.json()
|
||||
assert len(runs) == 2
|
||||
returned = {r["id"] for r in runs}
|
||||
assert returned == ids
|
||||
# newest-first ordering
|
||||
assert runs[0]["created_at"] >= runs[1]["created_at"]
|
||||
# #endregion Test.Api.DashboardTesting.VerificationApi.TestHistoryGet
|
||||
|
||||
# #region Test.Api.DashboardTesting.VerificationApi.TestDetailGet [C:2] [TYPE Function]
|
||||
# @BRIEF GET /verification/{run_id} returns a single run; missing run -> 404 (037 T081).
|
||||
def test_detail_get_and_missing_404(
|
||||
self, dashboard_testing_client, dashboard_testing_verification_repository_id: str
|
||||
):
|
||||
from src.core.database import SessionLocal
|
||||
from src.models.verification_run import VerificationRunRecord
|
||||
|
||||
setup = SessionLocal()
|
||||
try:
|
||||
r = VerificationRunRecord(
|
||||
repository_id=dashboard_testing_verification_repository_id,
|
||||
trigger="scheduled", environment_id="dev", dashboard_id=42,
|
||||
category_outcomes=[], overall_status="pass", summary="s",
|
||||
created_by="system",
|
||||
)
|
||||
setup.add(r)
|
||||
setup.commit()
|
||||
run_id = r.id
|
||||
finally:
|
||||
setup.close()
|
||||
|
||||
resp = dashboard_testing_client.get(f"/api/dashboard-testing/verification/{run_id}")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["id"] == run_id
|
||||
|
||||
missing = dashboard_testing_client.get("/api/dashboard-testing/verification/does-not-exist")
|
||||
assert missing.status_code == 404
|
||||
# #endregion Test.Api.DashboardTesting.VerificationApi.TestDetailGet
|
||||
# #endregion Test.Api.DashboardTesting.VerificationApi.Create
|
||||
|
||||
# #endregion Test.Api.DashboardTesting.VerificationApi
|
||||
|
||||
@@ -13,15 +13,13 @@ os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:")
|
||||
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-tests")
|
||||
os.environ.setdefault("DEV_MODE", "true")
|
||||
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import BaseModel
|
||||
|
||||
_src = str(Path(__file__).resolve().parent.parent.parent / "src")
|
||||
if _src not in sys.path:
|
||||
@@ -122,9 +120,10 @@ def _make_config_manager(overrides: dict | None = None) -> MagicMock:
|
||||
|
||||
def _make_mock_user(is_admin: bool = True) -> MagicMock:
|
||||
"""Build a user mock with an admin or regular role."""
|
||||
from src.schemas.auth import RoleSchema, User as UserSchema
|
||||
from datetime import datetime
|
||||
|
||||
from src.schemas.auth import RoleSchema, User as UserSchema
|
||||
|
||||
admin_role = RoleSchema(
|
||||
id="r1",
|
||||
name="Admin",
|
||||
@@ -299,7 +298,6 @@ class TestListReleases:
|
||||
def test_success_with_releases(self):
|
||||
"""Returns releases ordered by created_at desc."""
|
||||
repo = _make_mock_repository()
|
||||
from datetime import timezone
|
||||
release = _make_mock_release()
|
||||
|
||||
releases = [release]
|
||||
@@ -405,6 +403,48 @@ class TestCreateRelease:
|
||||
assert data["status"] == "ready_to_publish"
|
||||
assert data["created_by"] == "admin"
|
||||
|
||||
def test_create_release_triggers_verification_run(self):
|
||||
"""037 T080: creating a release must spawn a VerificationRun with trigger=release_create."""
|
||||
repo_id = "11111111-2222-3333-4444-555555555555"
|
||||
repo = _make_mock_repository(id=repo_id)
|
||||
deployment = _make_mock_deployment()
|
||||
cm = _make_config_manager()
|
||||
cm.get_config.return_value.settings.git_release.require_prod_approval = False
|
||||
|
||||
db_mock = MagicMock()
|
||||
self._make_query_chain(db_mock, first_result=repo, candidate_result=deployment)
|
||||
|
||||
release_id = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
|
||||
# Override the chain's refresh side-effect so the created release gets a valid UUID id
|
||||
# (real DashboardRelease.id is a String(36) UUID; the default "auto-id-42" is not).
|
||||
def _refresh_uuid(instance):
|
||||
if hasattr(instance, "_sa_instance_state"):
|
||||
if not getattr(instance, "id", None):
|
||||
instance.id = release_id
|
||||
if not getattr(instance, "created_at", None):
|
||||
instance.created_at = datetime.now()
|
||||
|
||||
db_mock.refresh.side_effect = _refresh_uuid
|
||||
|
||||
create_run = AsyncMock(return_value=MagicMock())
|
||||
|
||||
with (
|
||||
patch("src.api.routes.git._resolve_dashboard_id_from_ref", AsyncMock(return_value=42)),
|
||||
patch("src.api.routes.git._release_routes._resolve_stage_environment") as mock_resolve_env,
|
||||
patch("src.api.routes.git._release_routes._probe_drift", AsyncMock(return_value=("in_sync", "c0ffee42"))),
|
||||
patch("src.services.dashboard_testing.verification_service.create_verification_run_async", create_run),
|
||||
):
|
||||
mock_resolve_env.return_value = MagicMock(id="preprod-1")
|
||||
client = _make_client(db_mock=db_mock, config_manager_mock=cm)
|
||||
resp = client.post("/repositories/test-dash/releases", json=self.CREATE_PAYLOAD)
|
||||
|
||||
assert resp.status_code == 201
|
||||
# Verification hook fired with the release_create trigger
|
||||
create_run.assert_awaited_once()
|
||||
request = create_run.await_args.args[1] # db, request, created_by
|
||||
assert request.trigger == "release_create"
|
||||
assert str(request.repository_id) == repo_id
|
||||
|
||||
def test_success_with_approval_required(self):
|
||||
"""Creates a release with awaiting_approval when policy requires prod approval."""
|
||||
repo = _make_mock_repository()
|
||||
|
||||
Reference in New Issue
Block a user