busya
81de959ef7
Close the 037 pipeline-automation and read-API gaps found in the audit:
deploy/release hooks did not create VerificationRun, and GET endpoints for
history/detail were absent even though 039 UI and client call them.
T080 - _release_routes.py: create_release now fires best-effort
_trigger_release_verification -> VerificationRun with trigger=release_create
(metric+structure); verification scheduling failures never roll back the
release transaction.
T081 - verification.py: add GET /verification/history (dashboard_id +
environment_id filters, newest-first) and GET /verification/{run_id}
(404 RUN_NOT_FOUND); reuse _record_to_response.
- verification_run.py + alembic migration p2q3r4s5t6u7: nullable indexed
dashboard_id populated from structure/visual/metric category_params.
- verification_service.py: _derive_dashboard_id helper.
Verification: release routes (32) + verification API (8) + persistence (21)
= 53 passed; ruff clean for changed code (pre-existing RUF012/UP017 on old
lines left untouched).
243 lines
11 KiB
Python
243 lines
11 KiB
Python
# #region Test.Api.DashboardTesting.VerificationApi [C:3] [TYPE Module] [SEMANTICS test,api,dashboard-testing,verification-runs,http,persistence]
|
|
# @defgroup Verification-run HTTP tests with real repository fixtures.
|
|
# @LAYER Test
|
|
# @RELATION BINDS_TO -> [Api.DashboardTesting.CreateVerificationRun]
|
|
# @TEST_FIXTURE: verification_repository -> INLINE_JSON
|
|
# @TEST_EDGE: valid_repository -> API persists a run only for an existing repository.
|
|
# @TEST_EDGE: invalid_repository -> API returns 422 without weakening production validation.
|
|
from __future__ import annotations
|
|
|
|
from src.models.agent_run import AgentRun
|
|
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationApi.Create [C:3] [TYPE Class] [SEMANTICS test,api,verification-runs,http,persistence]
|
|
# @BRIEF HTTP-level verification-run creation exercises the API database fixture.
|
|
# @RELATION VERIFIES -> [Api.DashboardTesting.CreateVerificationRun]
|
|
class TestVerificationRunApi:
|
|
# #region Test.Api.DashboardTesting.VerificationApi.Test201Persistence [C:2] [TYPE Function]
|
|
# @BRIEF POST with real repository and agent-run foreign keys persists a verification record.
|
|
def test_create_verification_run_201_and_persistence(
|
|
self,
|
|
dashboard_testing_client,
|
|
dashboard_testing_verification_repository_id: str,
|
|
):
|
|
from src.core.database import SessionLocal
|
|
from src.models.verification_run import VerificationRunRecord
|
|
|
|
setup_session = SessionLocal()
|
|
try:
|
|
run = AgentRun(
|
|
user_id="test-user",
|
|
intent="dashboard_scenario_build",
|
|
trigger="manual",
|
|
dashboard_id="42",
|
|
environment_id="dev",
|
|
context_snapshot={"test": True},
|
|
)
|
|
setup_session.add(run)
|
|
setup_session.commit()
|
|
run_id = run.id
|
|
finally:
|
|
setup_session.close()
|
|
|
|
response = dashboard_testing_client.post(
|
|
"/api/dashboard-testing/verification-runs",
|
|
json={
|
|
"repository_id": dashboard_testing_verification_repository_id,
|
|
"trigger": "manual",
|
|
"environment_id": "dev",
|
|
"categories": ["structure"],
|
|
"evidence_refs": {"structure": ["ev://s/diff-abc"]},
|
|
"agent_run_id": run_id,
|
|
},
|
|
)
|
|
assert response.status_code == 201, response.text
|
|
data = response.json()
|
|
assert data["overall_status"] == "inconclusive"
|
|
outcome = data["category_outcomes"][0]
|
|
assert outcome["category"] == "structure"
|
|
assert outcome["status"] == "inconclusive"
|
|
assert outcome["evidence_refs"] == ["ev://s/diff-abc"]
|
|
|
|
verify_session = SessionLocal()
|
|
try:
|
|
record = verify_session.get(VerificationRunRecord, data["id"])
|
|
assert record is not None
|
|
assert record.repository_id == dashboard_testing_verification_repository_id
|
|
assert record.agent_run_id == run_id
|
|
finally:
|
|
verify_session.close()
|
|
# #endregion Test.Api.DashboardTesting.VerificationApi.Test201Persistence
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationApi.Test422BadTrigger [C:2] [TYPE Function]
|
|
# @BRIEF Invalid request literals are rejected by request validation before persistence.
|
|
def test_create_verification_run_422_bad_trigger(
|
|
self, dashboard_testing_client, dashboard_testing_verification_repository_id: str
|
|
):
|
|
response = dashboard_testing_client.post(
|
|
"/api/dashboard-testing/verification-runs",
|
|
json={
|
|
"repository_id": dashboard_testing_verification_repository_id,
|
|
"trigger": "not_a_valid_trigger",
|
|
"environment_id": "dev",
|
|
"categories": ["metric"],
|
|
},
|
|
)
|
|
assert response.status_code == 422
|
|
# #endregion Test.Api.DashboardTesting.VerificationApi.Test422BadTrigger
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationApi.Test422MissingAgentRun [C:2] [TYPE Function]
|
|
# @BRIEF Nonexistent agent-run references return the production validation error.
|
|
def test_create_verification_run_422_invalid_agent_run(
|
|
self, dashboard_testing_client, dashboard_testing_verification_repository_id: str
|
|
):
|
|
response = dashboard_testing_client.post(
|
|
"/api/dashboard-testing/verification-runs",
|
|
json={
|
|
"repository_id": dashboard_testing_verification_repository_id,
|
|
"trigger": "manual",
|
|
"environment_id": "dev",
|
|
"categories": ["metric"],
|
|
"evidence_refs": {"metric": ["ev://m/1"]},
|
|
"agent_run_id": "00000000-0000-0000-0000-000000000000",
|
|
},
|
|
)
|
|
assert response.status_code == 422
|
|
assert "agent_run_id" in response.json()["detail"]
|
|
# #endregion Test.Api.DashboardTesting.VerificationApi.Test422MissingAgentRun
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationApi.TestBlockedUnsupported [C:2] [TYPE Function]
|
|
# @BRIEF Unsupported categories remain blocked after repository validation succeeds.
|
|
def test_blocked_unsupported_category_via_api(
|
|
self, dashboard_testing_client, dashboard_testing_verification_repository_id: str
|
|
):
|
|
response = dashboard_testing_client.post(
|
|
"/api/dashboard-testing/verification-runs",
|
|
json={
|
|
"repository_id": dashboard_testing_verification_repository_id,
|
|
"trigger": "scheduled",
|
|
"environment_id": "prod",
|
|
"categories": ["content_integrity"],
|
|
},
|
|
)
|
|
assert response.status_code == 201, response.text
|
|
outcome = response.json()["category_outcomes"][0]
|
|
assert outcome["status"] == "blocked"
|
|
assert "no executor" in outcome["summary"].lower()
|
|
# #endregion Test.Api.DashboardTesting.VerificationApi.TestBlockedUnsupported
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationApi.TestEvidenceOnly [C:2] [TYPE Function]
|
|
# @BRIEF Evidence-only categories are inconclusive rather than fabricated passes.
|
|
def test_evidence_only_inconclusive_via_api(
|
|
self, dashboard_testing_client, dashboard_testing_verification_repository_id: str
|
|
):
|
|
response = dashboard_testing_client.post(
|
|
"/api/dashboard-testing/verification-runs",
|
|
json={
|
|
"repository_id": dashboard_testing_verification_repository_id,
|
|
"trigger": "release_publish",
|
|
"environment_id": "staging",
|
|
"categories": ["xlsx"],
|
|
"evidence_refs": {"xlsx": ["s3://bucket/report.xlsx"]},
|
|
},
|
|
)
|
|
assert response.status_code == 201, response.text
|
|
outcome = response.json()["category_outcomes"][0]
|
|
assert outcome["status"] == "inconclusive"
|
|
assert outcome["evidence_refs"] == ["s3://bucket/report.xlsx"]
|
|
# #endregion Test.Api.DashboardTesting.VerificationApi.TestEvidenceOnly
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationApi.TestStructureBlocked [C:2] [TYPE Function]
|
|
# @BRIEF Structure execution without evidence or parameters remains blocked.
|
|
def test_structure_blocked_without_evidence_via_api(
|
|
self, dashboard_testing_client, dashboard_testing_verification_repository_id: str
|
|
):
|
|
response = dashboard_testing_client.post(
|
|
"/api/dashboard-testing/verification-runs",
|
|
json={
|
|
"repository_id": dashboard_testing_verification_repository_id,
|
|
"trigger": "deploy_to_preprod",
|
|
"environment_id": "dev",
|
|
"categories": ["structure"],
|
|
},
|
|
)
|
|
assert response.status_code == 201, response.text
|
|
outcome = response.json()["category_outcomes"][0]
|
|
assert outcome["status"] == "blocked"
|
|
assert "evidence_refs" in outcome["summary"]
|
|
# #endregion Test.Api.DashboardTesting.VerificationApi.TestStructureBlocked
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationApi.TestHistoryGet [C:2] [TYPE Function]
|
|
# @BRIEF GET /verification/history returns persisted runs ordered newest-first (037 T081).
|
|
def test_history_get_returns_runs(
|
|
self, dashboard_testing_client, dashboard_testing_verification_repository_id: str
|
|
):
|
|
from src.core.database import SessionLocal
|
|
from src.models.verification_run import VerificationRunRecord
|
|
|
|
setup = SessionLocal()
|
|
try:
|
|
r1 = VerificationRunRecord(
|
|
repository_id=dashboard_testing_verification_repository_id,
|
|
trigger="scheduled", environment_id="dev", dashboard_id=42,
|
|
category_outcomes=[], overall_status="pass", summary="s",
|
|
created_by="system",
|
|
)
|
|
r2 = VerificationRunRecord(
|
|
repository_id=dashboard_testing_verification_repository_id,
|
|
trigger="scheduled", environment_id="dev", dashboard_id=42,
|
|
category_outcomes=[], overall_status="fail", summary="s",
|
|
created_by="system",
|
|
)
|
|
setup.add_all([r1, r2])
|
|
setup.commit()
|
|
ids = {r1.id, r2.id}
|
|
finally:
|
|
setup.close()
|
|
|
|
response = dashboard_testing_client.get(
|
|
"/api/dashboard-testing/verification/history",
|
|
params={"dashboard_id": 42, "environment_id": "dev"},
|
|
)
|
|
assert response.status_code == 200, response.text
|
|
runs = response.json()
|
|
assert len(runs) == 2
|
|
returned = {r["id"] for r in runs}
|
|
assert returned == ids
|
|
# newest-first ordering
|
|
assert runs[0]["created_at"] >= runs[1]["created_at"]
|
|
# #endregion Test.Api.DashboardTesting.VerificationApi.TestHistoryGet
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationApi.TestDetailGet [C:2] [TYPE Function]
|
|
# @BRIEF GET /verification/{run_id} returns a single run; missing run -> 404 (037 T081).
|
|
def test_detail_get_and_missing_404(
|
|
self, dashboard_testing_client, dashboard_testing_verification_repository_id: str
|
|
):
|
|
from src.core.database import SessionLocal
|
|
from src.models.verification_run import VerificationRunRecord
|
|
|
|
setup = SessionLocal()
|
|
try:
|
|
r = VerificationRunRecord(
|
|
repository_id=dashboard_testing_verification_repository_id,
|
|
trigger="scheduled", environment_id="dev", dashboard_id=42,
|
|
category_outcomes=[], overall_status="pass", summary="s",
|
|
created_by="system",
|
|
)
|
|
setup.add(r)
|
|
setup.commit()
|
|
run_id = r.id
|
|
finally:
|
|
setup.close()
|
|
|
|
resp = dashboard_testing_client.get(f"/api/dashboard-testing/verification/{run_id}")
|
|
assert resp.status_code == 200, resp.text
|
|
assert resp.json()["id"] == run_id
|
|
|
|
missing = dashboard_testing_client.get("/api/dashboard-testing/verification/does-not-exist")
|
|
assert missing.status_code == 404
|
|
# #endregion Test.Api.DashboardTesting.VerificationApi.TestDetailGet
|
|
# #endregion Test.Api.DashboardTesting.VerificationApi.Create
|
|
|
|
# #endregion Test.Api.DashboardTesting.VerificationApi
|