ss-tools/backend/tests/api/test_dashboard_testing_feature037.py

# #region Test.Api.DashboardTesting.Feature037 [C:4] [TYPE Module] [SEMANTICS testing,api,dashboard-testing,feature-037,verification-runs,orchestrator]
# @defgroup Orchestrator tests for feature-037 verification runs using real foreign-key fixtures.
# @LAYER Test
# @RELATION BINDS_TO -> [BaselineEngine.Verification.Service]
# @TEST_FIXTURE: verification_repository -> INLINE_JSON
# @TEST_EDGE: existing_repository -> Repository validation permits only a persisted GitRepository.
# @TEST_EDGE: missing_repository -> Repository validation raises ValueError.
# @TEST_EDGE: invalid_type -> Invalid trigger is rejected by schema validation in the API suite.
# @TEST_EDGE: external_fail -> Commit exceptions roll back in verification-persistence tests.
from __future__ import annotations

from datetime import datetime
import pytest
from uuid import UUID, uuid4

from sqlalchemy import create_engine, event
from sqlalchemy.orm import Session

from src.models.agent_run import AgentRun
from src.models.dashboard_release import DashboardRelease
from src.models.git import DeploymentEnvironment, GitRepository, GitServerConfig
from src.models.mapping import Base
from src.schemas.dashboard_testing import VerificationRunRequest
from src.services.dashboard_testing.verification_service import create_verification_run

_REPOSITORY_ID = "550e8400-e29b-41d4-a716-446655440000"
_ENGINE = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False})
event.listen(_ENGINE, "connect", lambda connection, _: connection.execute("PRAGMA foreign_keys=ON"))
Base.metadata.create_all(bind=_ENGINE)


# #region Test.Api.DashboardTesting.Feature037.DbSession [C:2] [TYPE Function] [SEMANTICS test,verification,fixture]
# @BRIEF Supply an isolated session sharing the module FK-enabled database schema.
@pytest.fixture
def db_session() -> Session:
    connection = _ENGINE.connect()
    transaction = connection.begin()
    session = Session(bind=connection)
    try:
        yield session
    finally:
        session.close()
        transaction.rollback()
        connection.close()
# #endregion Test.Api.DashboardTesting.Feature037.DbSession


# #region Test.Api.DashboardTesting.Feature037.Repository [C:2] [TYPE Function] [SEMANTICS test,verification,repository,fixture]
# @BRIEF Create the minimal persisted GitServerConfig and GitRepository required by validation.
@pytest.fixture(autouse=True)
def sample_repository(db_session: Session) -> GitRepository:
    server = GitServerConfig(
        id=str(uuid4()),
        name="verification-test-server",
        provider="GITHUB",
        url="https://git.example.test",
        pat="verification-test-token",
    )
    db_session.add(server)
    db_session.flush()
    repository = GitRepository(
        id=_REPOSITORY_ID,
        dashboard_id=999,
        config_id=server.id,
        remote_url="https://git.example.test/org/verification.git",
        local_path="/tmp/verification-test-repository",
    )
    db_session.add(repository)
    db_session.commit()
    return repository
# #endregion Test.Api.DashboardTesting.Feature037.Repository


# #region Test.Api.DashboardTesting.Feature037.Request [C:1] [TYPE Function]
def _request(
    categories: list[str] | None = None,
    *,
    agent_run_id: str | None = None,
    release_id: str | None = None,
    evidence: dict[str, list[str]] | None = None,
    trigger: str = "manual",
    environment_id: str = "dev",
    repository_id: str = _REPOSITORY_ID,
) -> VerificationRunRequest:
    return VerificationRunRequest(
        repository_id=UUID(repository_id),
        trigger=trigger,  # type: ignore[arg-type]
        environment_id=environment_id,
        categories=categories or ["metric"],  # type: ignore[list-item]
        agent_run_id=UUID(agent_run_id) if agent_run_id else None,
        release_id=UUID(release_id) if release_id else None,
        evidence_refs=evidence,
    )
# #endregion Test.Api.DashboardTesting.Feature037.Request


# #region Test.Api.DashboardTesting.Feature037.AgentRun [C:1] [TYPE Function]
def _agent_run(session: Session) -> AgentRun:
    run = AgentRun(
        id=str(uuid4()),
        user_id="test-user",
        intent="dashboard_scenario_build",
        trigger="manual",
        dashboard_id="42",
        environment_id="dev",
        context_snapshot={"test": True},
        status="CREATED",
    )
    session.add(run)
    session.commit()
    return run
# #endregion Test.Api.DashboardTesting.Feature037.AgentRun


# #region Test.Api.DashboardTesting.Feature037.Release [C:1] [TYPE Function]
def _release(session: Session) -> DashboardRelease:
    repository = session.get(GitRepository, _REPOSITORY_ID)
    assert repository is not None
    environment = DeploymentEnvironment(
        id=str(uuid4()),
        name="verification-release-env",
        superset_url="https://superset.example.test",
        superset_token="verification-release-token",
    )
    session.add(environment)
    session.flush()
    from src.models.deployment import DeploymentRecord

    deployment = DeploymentRecord(
        repository_id=repository.id,
        environment_id=environment.id,
        commit_hash="a" * 40,
        content_hash="b" * 64,
        deployed_by="qa",
    )
    session.add(deployment)
    session.flush()
    release = DashboardRelease(
        id=str(uuid4()),
        repository_id=repository.id,
        deployment_id=deployment.id,
        name="v1.0.0",
        version="1.0.0",
        notes="Verification fixture release",
        commit_hash="a" * 40,
        content_hash="b" * 64,
        created_by="qa",
    )
    session.add(release)
    session.commit()
    return release
# #endregion Test.Api.DashboardTesting.Feature037.Release


# #region Test.Feature037.VerificationRunService [C:3] [TYPE Class] [SEMANTICS test,verification,service,orchestrator]
# @BRIEF Orchestrator behaviour uses real persistence and Git FK fixture records.
# @RELATION VERIFIES -> [BaselineEngine.Verification.Service]
class TestVerificationRunOrchestrator:
    # #region Test.Feature037.VerificationRunService.Evidence [C:2] [TYPE Function]
    # @BRIEF Evidence-only unsupported categories yield inconclusive, never a fabricated pass.
    def test_category_with_evidence_is_inconclusive_not_pass(self, db_session: Session):
        result = create_verification_run(
            db_session, _request(["xlsx"], evidence={"xlsx": ["s3://bucket/report.xlsx"]})
        )
        outcome = result.category_outcomes[0]
        assert outcome.category == "xlsx"
        assert outcome.status == "inconclusive"
        assert outcome.evidence_refs == ["s3://bucket/report.xlsx"]
        assert result.overall_status == "inconclusive"
    # #endregion Test.Feature037.VerificationRunService.Evidence

    # #region Test.Feature037.VerificationRunService.StructureEvidence [C:2] [TYPE Function]
    # @BRIEF Structure category evidence is inconclusive when no executable diff parameters exist.
    def test_structure_executor_with_evidence(self, db_session: Session):
        result = create_verification_run(
            db_session,
            _request(["structure"], evidence={"structure": ["ev://structure/diff-1"]}),
        )
        outcome = result.category_outcomes[0]
        assert outcome.category == "structure"
        assert outcome.status == "inconclusive"
        assert outcome.evidence_refs == ["ev://structure/diff-1"]
    # #endregion Test.Feature037.VerificationRunService.StructureEvidence

    # #region Test.Feature037.VerificationRunService.Blocked [C:2] [TYPE Function]
    # @BRIEF Unsupported categories without evidence remain blocked.
    def test_category_without_evidence_is_blocked(self, db_session: Session):
        result = create_verification_run(db_session, _request(["content_integrity"]))
        assert result.category_outcomes[0].status == "blocked"
        assert result.overall_status == "blocked"
    # #endregion Test.Feature037.VerificationRunService.Blocked

    # #region Test.Feature037.VerificationRunService.UniqueIds [C:2] [TYPE Function]
    # @BRIEF Identical valid requests create distinct UUID records.
    def test_identical_requests_produce_unique_ids(self, db_session: Session):
        request = _request(evidence={"metric": ["ev://metric/1"]})
        assert create_verification_run(db_session, request).id != create_verification_run(db_session, request).id
    # #endregion Test.Feature037.VerificationRunService.UniqueIds

    # #region Test.Feature037.VerificationRunService.Persistence [C:2] [TYPE Function]
    # @BRIEF A committed run can be reloaded with its hardcoded request fields.
    def test_persistence_and_reload(self, db_session: Session):
        from src.models.verification_run import VerificationRunRecord

        result = create_verification_run(
            db_session,
            _request(
                ["structure"],
                evidence={"structure": ["ev://structure/diff-1"]},
                trigger="deploy_to_preprod",
                environment_id="staging",
            ),
        )
        record = db_session.get(VerificationRunRecord, str(result.id))
        assert record is not None
        assert (record.trigger, record.environment_id, record.overall_status) == (
            "deploy_to_preprod", "staging", "inconclusive"
        )
        assert record.category_outcomes[0]["category"] == "structure"
    # #endregion Test.Feature037.VerificationRunService.Persistence

    # #region Test.Feature037.VerificationRunService.InvalidLinks [C:2] [TYPE Function]
    # @BRIEF Missing optional parent references retain their explicit validation errors.
    def test_invalid_links_raise_value_error(self, db_session: Session):
        with pytest.raises(ValueError, match=r"agent_run_id.*not found"):
            create_verification_run(db_session, _request(agent_run_id=str(uuid4())))
        with pytest.raises(ValueError, match=r"release_id.*not found"):
            create_verification_run(db_session, _request(release_id=str(uuid4())))
    # #endregion Test.Feature037.VerificationRunService.InvalidLinks

    # #region Test.Feature037.VerificationRunService.ValidLinks [C:2] [TYPE Function]
    # @BRIEF Existing agent-run and release references persist when repository IDs agree.
    def test_valid_links_succeed(self, db_session: Session):
        agent_run = _agent_run(db_session)
        release = _release(db_session)
        result = create_verification_run(
            db_session,
            _request(
                ["metric"],
                agent_run_id=agent_run.id,
                release_id=release.id,
                evidence={"metric": ["ev://metric/abc"]},
            ),
        )
        assert str(result.agent_run_id) == agent_run.id
        assert str(result.release_id) == release.id
    # #endregion Test.Feature037.VerificationRunService.ValidLinks

    # #region Test.Feature037.VerificationRunService.InvalidRepository [C:2] [TYPE Function]
    # @BRIEF Unknown repository IDs are rejected before any verification record persists.
    def test_invalid_repository_raises_value_error(self, db_session: Session):
        with pytest.raises(ValueError, match=r"repository_id.*not found"):
            create_verification_run(
                db_session,
                _request(repository_id="00000000-0000-0000-0000-000000000000"),
            )
    # #endregion Test.Feature037.VerificationRunService.InvalidRepository

    # #region Test.Feature037.VerificationRunService.Status [C:2] [TYPE Function]
    # @BRIEF Blocked outcomes take priority over inconclusive outcomes.
    def test_overall_status_priority(self, db_session: Session):
        result = create_verification_run(
            db_session,
            _request(
                ["structure", "content_integrity"],
                evidence={"structure": ["ev://structure/diff-1"]},
            ),
        )
        assert result.overall_status == "blocked"
        assert [outcome.status for outcome in result.category_outcomes] == ["inconclusive", "blocked"]
    # #endregion Test.Feature037.VerificationRunService.Status

    # #region Test.Feature037.VerificationRunService.ImmutabilityViolation [C:2] [TYPE Function] [SEMANTICS test,verification,immutability-violation]
    # @BRIEF Pydantic schema accepts immutability_violation in CategoryOutcome and VerificationRun.
    # @TEST_EDGE: immutability_violation_status -> CategoryOutcome and VerificationRun Pydantic schemas
    #             accept immutability_violation as valid overall_status.
    def test_immutability_violation_overall_status(self):
        """Pydantic schemas accept immutability_violation as overall_status."""
        from datetime import UTC, datetime
        from uuid import uuid4

        from src.schemas.dashboard_testing import CategoryOutcome, VerificationRun

        # CategoryOutcome accepts immutability_violation
        co = CategoryOutcome(
            category="metric", status="immutability_violation",
            summary="CRITICAL: immutability violation detected",
        )
        assert co.status == "immutability_violation"

        # VerificationRun accepts immutability_violation as overall_status
        run = VerificationRun(
            id=uuid4(),
            repository_id=uuid4(),
            trigger="manual",
            environment_id="prod",
            overall_status="immutability_violation",
            created_at=datetime.now(UTC),
            category_outcomes=[co],
        )
        assert run.overall_status == "immutability_violation"

        # Derive overall_status from immutability_violation CategoryOutcome
        from src.services.dashboard_testing.verification_service import VerificationRunOrchestrator
        derived = VerificationRunOrchestrator._derive_overall_status([co])
        assert derived == "immutability_violation"
    # #endregion Test.Feature037.VerificationRunService.ImmutabilityViolation

    # #region Test.Feature037.VerificationRunService.Response [C:2] [TYPE Function]
    # @BRIEF Response includes the persisted repository and audit metadata.
    def test_full_run_response_properties(self, db_session: Session):
        result = create_verification_run(
            db_session,
            _request(
                evidence={"metric": ["ev://metric/perf-1"]},
                trigger="release_publish",
                environment_id="prod",
            ),
        )
        assert result.repository_id == UUID(_REPOSITORY_ID)
        assert result.trigger == "release_publish"
        assert result.environment_id == "prod"
        assert isinstance(result.created_at, datetime)
        assert "release_publish" in result.summary
    # #endregion Test.Feature037.VerificationRunService.Response
# #endregion Test.Feature037.VerificationRunService

# #endregion Test.Api.DashboardTesting.Feature037