ss-tools/backend/tests/api/test_dashboard_testing_verification_api.py

# #region Test.Api.DashboardTesting.VerificationApi [C:3] [TYPE Module] [SEMANTICS test,api,dashboard-testing,verification-runs,http,persistence]
# @defgroup Verification-run HTTP tests with real repository fixtures.
# @LAYER Test
# @RELATION BINDS_TO -> [Api.DashboardTesting.CreateVerificationRun]
# @TEST_FIXTURE: verification_repository -> INLINE_JSON
# @TEST_EDGE: valid_repository -> API persists a run only for an existing repository.
# @TEST_EDGE: invalid_repository -> API returns 422 without weakening production validation.
from __future__ import annotations

from src.models.agent_run import AgentRun


# #region Test.Api.DashboardTesting.VerificationApi.Create [C:3] [TYPE Class] [SEMANTICS test,api,verification-runs,http,persistence]
# @BRIEF HTTP-level verification-run creation exercises the API database fixture.
# @RELATION VERIFIES -> [Api.DashboardTesting.CreateVerificationRun]
class TestVerificationRunApi:
    # #region Test.Api.DashboardTesting.VerificationApi.Test201Persistence [C:2] [TYPE Function]
    # @BRIEF POST with real repository and agent-run foreign keys persists a verification record.
    def test_create_verification_run_201_and_persistence(
        self,
        dashboard_testing_client,
        dashboard_testing_verification_repository_id: str,
    ):
        from src.core.database import SessionLocal
        from src.models.verification_run import VerificationRunRecord

        setup_session = SessionLocal()
        try:
            run = AgentRun(
                user_id="test-user",
                intent="dashboard_scenario_build",
                trigger="manual",
                dashboard_id="42",
                environment_id="dev",
                context_snapshot={"test": True},
            )
            setup_session.add(run)
            setup_session.commit()
            run_id = run.id
        finally:
            setup_session.close()

        response = dashboard_testing_client.post(
            "/api/dashboard-testing/verification-runs",
            json={
                "repository_id": dashboard_testing_verification_repository_id,
                "trigger": "manual",
                "environment_id": "dev",
                "categories": ["structure"],
                "evidence_refs": {"structure": ["ev://s/diff-abc"]},
                "agent_run_id": run_id,
            },
        )
        assert response.status_code == 201, response.text
        data = response.json()
        assert data["overall_status"] == "inconclusive"
        outcome = data["category_outcomes"][0]
        assert outcome["category"] == "structure"
        assert outcome["status"] == "inconclusive"
        assert outcome["evidence_refs"] == ["ev://s/diff-abc"]

        verify_session = SessionLocal()
        try:
            record = verify_session.get(VerificationRunRecord, data["id"])
            assert record is not None
            assert record.repository_id == dashboard_testing_verification_repository_id
            assert record.agent_run_id == run_id
        finally:
            verify_session.close()
    # #endregion Test.Api.DashboardTesting.VerificationApi.Test201Persistence

    # #region Test.Api.DashboardTesting.VerificationApi.Test422BadTrigger [C:2] [TYPE Function]
    # @BRIEF Invalid request literals are rejected by request validation before persistence.
    def test_create_verification_run_422_bad_trigger(
        self, dashboard_testing_client, dashboard_testing_verification_repository_id: str
    ):
        response = dashboard_testing_client.post(
            "/api/dashboard-testing/verification-runs",
            json={
                "repository_id": dashboard_testing_verification_repository_id,
                "trigger": "not_a_valid_trigger",
                "environment_id": "dev",
                "categories": ["metric"],
            },
        )
        assert response.status_code == 422
    # #endregion Test.Api.DashboardTesting.VerificationApi.Test422BadTrigger

    # #region Test.Api.DashboardTesting.VerificationApi.Test422MissingAgentRun [C:2] [TYPE Function]
    # @BRIEF Nonexistent agent-run references return the production validation error.
    def test_create_verification_run_422_invalid_agent_run(
        self, dashboard_testing_client, dashboard_testing_verification_repository_id: str
    ):
        response = dashboard_testing_client.post(
            "/api/dashboard-testing/verification-runs",
            json={
                "repository_id": dashboard_testing_verification_repository_id,
                "trigger": "manual",
                "environment_id": "dev",
                "categories": ["metric"],
                "evidence_refs": {"metric": ["ev://m/1"]},
                "agent_run_id": "00000000-0000-0000-0000-000000000000",
            },
        )
        assert response.status_code == 422
        assert "agent_run_id" in response.json()["detail"]
    # #endregion Test.Api.DashboardTesting.VerificationApi.Test422MissingAgentRun

    # #region Test.Api.DashboardTesting.VerificationApi.TestBlockedUnsupported [C:2] [TYPE Function]
    # @BRIEF Unsupported categories remain blocked after repository validation succeeds.
    def test_blocked_unsupported_category_via_api(
        self, dashboard_testing_client, dashboard_testing_verification_repository_id: str
    ):
        response = dashboard_testing_client.post(
            "/api/dashboard-testing/verification-runs",
            json={
                "repository_id": dashboard_testing_verification_repository_id,
                "trigger": "scheduled",
                "environment_id": "prod",
                "categories": ["content_integrity"],
            },
        )
        assert response.status_code == 201, response.text
        outcome = response.json()["category_outcomes"][0]
        assert outcome["status"] == "blocked"
        assert "no executor" in outcome["summary"].lower()
    # #endregion Test.Api.DashboardTesting.VerificationApi.TestBlockedUnsupported

    # #region Test.Api.DashboardTesting.VerificationApi.TestEvidenceOnly [C:2] [TYPE Function]
    # @BRIEF Evidence-only categories are inconclusive rather than fabricated passes.
    def test_evidence_only_inconclusive_via_api(
        self, dashboard_testing_client, dashboard_testing_verification_repository_id: str
    ):
        response = dashboard_testing_client.post(
            "/api/dashboard-testing/verification-runs",
            json={
                "repository_id": dashboard_testing_verification_repository_id,
                "trigger": "release_publish",
                "environment_id": "staging",
                "categories": ["xlsx"],
                "evidence_refs": {"xlsx": ["s3://bucket/report.xlsx"]},
            },
        )
        assert response.status_code == 201, response.text
        outcome = response.json()["category_outcomes"][0]
        assert outcome["status"] == "inconclusive"
        assert outcome["evidence_refs"] == ["s3://bucket/report.xlsx"]
    # #endregion Test.Api.DashboardTesting.VerificationApi.TestEvidenceOnly

    # #region Test.Api.DashboardTesting.VerificationApi.TestStructureBlocked [C:2] [TYPE Function]
    # @BRIEF Structure execution without evidence or parameters remains blocked.
    def test_structure_blocked_without_evidence_via_api(
        self, dashboard_testing_client, dashboard_testing_verification_repository_id: str
    ):
        response = dashboard_testing_client.post(
            "/api/dashboard-testing/verification-runs",
            json={
                "repository_id": dashboard_testing_verification_repository_id,
                "trigger": "deploy_to_preprod",
                "environment_id": "dev",
                "categories": ["structure"],
            },
        )
        assert response.status_code == 201, response.text
        outcome = response.json()["category_outcomes"][0]
        assert outcome["status"] == "blocked"
        assert "evidence_refs" in outcome["summary"]
    # #endregion Test.Api.DashboardTesting.VerificationApi.TestStructureBlocked
# #endregion Test.Api.DashboardTesting.VerificationApi.Create

# #endregion Test.Api.DashboardTesting.VerificationApi