# #region Test.Api.DashboardTesting.VerificationPersistence [C:4] [TYPE Module] [SEMANTICS testing,api,dashboard-testing,verification,persistence,repository-fk] # @defgroup VerificationRun persistence, real executor, and transaction rollback tests. # @LAYER Test # @RELATION BINDS_TO -> [BaselineEngine.Verification.Service] # @TEST_FIXTURE executor_params -> INLINE_JSON # @TEST_EDGE metric_outcome -> Hardcoded matching values yield pass through the real comparison service. # @TEST_EDGE visual_outcome -> Matching image hashes yield pass through the real visual service. # @TEST_EDGE structure_outcome -> Matching persisted snapshots yield pass through the real diff service. # @TEST_EDGE fk_set_null -> Deleting linked AgentRun or DashboardRelease preserves the verification record. # @TEST_EDGE repository_fk_set_null -> Deleting GitRepository sets verification_runs.repository_id to NULL. # @TEST_EDGE invalid_repository -> Referencing non-existent GitRepository raises ValueError. # @TEST_EDGE commit_failure -> A commit exception rolls back the unpersisted verification record. # @TEST_EDGE independent_evidence_same_run -> Actual evidence from same AgentRun as baseline is BLOCKED. # @TEST_EDGE independent_evidence_separate_run -> Actual evidence from different AgentRun passes invariant. from __future__ import annotations import hashlib from pathlib import Path import pytest from uuid import uuid4 from sqlalchemy import create_engine, event from sqlalchemy.orm import Session import yaml from src.models.agent_run import AgentRun, DraftArtifact from src.models.dashboard_release import DashboardRelease from src.models.deployment import DeploymentRecord from src.models.git import DeploymentEnvironment, GitRepository, GitServerConfig from src.models.mapping import Base from src.models.verification_run import VerificationRunRecord from src.schemas.dashboard_testing import VerificationRunRequest from src.services.dashboard_testing.structure_diff_service import set_snapshot_base_path from src.services.dashboard_testing.verification_service import create_verification_run _ENGINE = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}) event.listen(_ENGINE, "connect", lambda connection, _: connection.execute("PRAGMA foreign_keys=ON")) Base.metadata.create_all(bind=_ENGINE) _FIXTURE_BASE = Path(__file__).parents[1] / "fixtures" / "structure_diff" _REPOSITORY_ID = "550e8400-e29b-41d4-a716-446655440000" def _ensure_repository() -> GitRepository: """Create the default GitRepository in the shared engine if not present.""" with Session(_ENGINE) as session: existing = session.query(GitRepository).filter(GitRepository.id == _REPOSITORY_ID).first() if existing: return existing server = GitServerConfig( id=str(uuid4()), name="auto-server", provider="GITHUB", url="https://auto.test", pat="token", ) session.add(server) session.flush() # Ensure server is persisted before repo references it repo = GitRepository( id=_REPOSITORY_ID, dashboard_id=9999, config_id=server.id, remote_url="https://auto.test/repo.git", local_path="/tmp/auto", ) session.add(repo) session.commit() return repo # Ensure the default repository exists at module load time _ensure_repository() # #region Test.Api.DashboardTesting.VerificationPersistence.DbSession [C:2] [TYPE Function] [SEMANTICS test,verification,fixture] # @BRIEF Supply an isolated SQLite session with foreign-key enforcement. @pytest.fixture def db_session() -> Session: connection = _ENGINE.connect() transaction = connection.begin() session = Session(bind=connection) try: yield session finally: session.close() if transaction.is_active: transaction.rollback() connection.close() # #endregion Test.Api.DashboardTesting.VerificationPersistence.DbSession # #region Test.Api.DashboardTesting.VerificationPersistence.Request [C:1] [TYPE Function] [SEMANTICS test,verification,fixture] def _request(category: str, params: dict, **links: str) -> VerificationRunRequest: return VerificationRunRequest( repository_id=links.get("repository_id", _REPOSITORY_ID), trigger="manual", environment_id="ss-preprod", categories=[category], category_params={category: params}, agent_run_id=links.get("agent_run_id"), release_id=links.get("release_id"), ) # #endregion Test.Api.DashboardTesting.VerificationPersistence.Request # #region Test.Api.DashboardTesting.VerificationPersistence.VisualRequest [C:1] [TYPE Function] [SEMANTICS test,verification,visual,evidence-refs] def _visual_request( category: str, params: dict, evidence_refs: list[str] | None = None, **links: str, ) -> VerificationRunRequest: """Create a VerificationRunRequest with evidence_refs for visual/testing categories.""" return VerificationRunRequest( repository_id=links.get("repository_id", _REPOSITORY_ID), trigger="manual", environment_id=links.get("environment_id", "ss-preprod"), categories=[category], category_params={category: params}, evidence_refs={category: evidence_refs} if evidence_refs else None, agent_run_id=links.get("agent_run_id"), release_id=links.get("release_id"), ) # #endregion Test.Api.DashboardTesting.VerificationPersistence.VisualRequest # #region Test.Api.DashboardTesting.VerificationPersistence.LinkedRelease [C:1] [TYPE Function] [SEMANTICS test,verification,release,fixture] def _linked_release(session: Session) -> DashboardRelease: # Use the pre-existing repository (created at module load) repository = session.query(GitRepository).filter(GitRepository.id == _REPOSITORY_ID).first() assert repository is not None, "Default repository must exist" environment = DeploymentEnvironment(id=str(uuid4()), name="test", superset_url="https://superset.test", superset_token="token") session.add(environment) session.flush() deployment = DeploymentRecord(repository_id=repository.id, environment_id=environment.id, commit_hash="a" * 40, content_hash="b" * 64, deployed_by="qa") session.add(deployment) session.flush() release = DashboardRelease(id=str(uuid4()), repository_id=repository.id, deployment_id=deployment.id, name="v1.0.0", version="v1.0.0", notes="test", commit_hash="a" * 40, content_hash="b" * 64, created_by="qa") session.add(release) session.commit() return release # #endregion Test.Api.DashboardTesting.VerificationPersistence.LinkedRelease # #region Test.Api.DashboardTesting.VerificationPersistence.RealExecutors [C:3] [TYPE Class] [SEMANTICS test,verification,executor,real] class TestRealExecutorOutcomes: """Verification categories must derive statuses from their real domain services.""" # #region Test.Api.DashboardTesting.VerificationPersistence.RealExecutors.Metric [C:2] [TYPE Function] [SEMANTICS test,verification,metric] # @BRIEF Matching hardcoded normalized values yield a real metric pass. def test_metric_executor_reports_real_pass(self, db_session: Session): result = create_verification_run(db_session, _request("metric", { "comparisons": [{ "actual": {"kind": "integer", "canonical_value": "7"}, "expected": {"kind": "integer", "canonical_value": "7"}, "policy": {"type": "exact"}, }], })) assert result.overall_status == "pass" assert result.category_outcomes[0].status == "pass" assert result.category_outcomes[0].details["comparisons"][0]["status"] == "pass" # #endregion Test.Api.DashboardTesting.VerificationPersistence.RealExecutors.Metric # #region Test.Api.DashboardTesting.VerificationPersistence.RealExecutors.Visual [C:2] [TYPE Function] [SEMANTICS test,verification,visual,security] # @BRIEF Caller-supplied expected_image_sha256 without catalog is now BLOCKED (security fix). # The executor no longer accepts hashes/policy from params — it requires catalog + evidence. def test_visual_executor_rejects_caller_hashes_without_catalog(self, db_session: Session): image_hash = "c" * 64 # agent_run_id and release_id are required by VerificationRunRequest for visual category visual_release = _linked_release(db_session) visual_release_id = visual_release.id # Create a valid AgentRun record from src.schemas.agent_run import CreateAgentRunRequest, UIContextV2 from src.services.agent_runs.service import create_agent_run agent_run = create_agent_run(db_session, CreateAgentRunRequest( context=UIContextV2(objectType="dashboard", objectId="42", envId="ss-preprod", route="/dashboards/42", contextVersion=2, intent="build_dashboard_test_scenario"), ), user_id="qa") result = create_verification_run(db_session, _request("visual", { "actual_image_sha256": image_hash, "expected_image_sha256": image_hash, "policy": {"type": "visual_exact"}, }, agent_run_id=agent_run.id, release_id=visual_release_id)) assert result.overall_status == "blocked", ( f"Expected blocked (caller hashes rejected without catalog), " f"got {result.overall_status}: {result.category_outcomes[0].summary}" ) assert result.category_outcomes[0].status == "blocked" # #endregion Test.Api.DashboardTesting.VerificationPersistence.RealExecutors.Visual # #region Test.Api.DashboardTesting.VerificationPersistence.RealExecutors.Structure [C:2] [TYPE Function] [SEMANTICS test,verification,structure] # @BRIEF Identical persisted snapshots yield a real structure pass without mocking the diff service. def test_structure_executor_reports_real_pass(self, db_session: Session): set_snapshot_base_path(_FIXTURE_BASE) try: result = create_verification_run(db_session, _request("structure", { "dashboard_id": 42, "release_version_from": "v1.0.0", "release_version_to": "v1.0.0", })) finally: set_snapshot_base_path(None) assert result.overall_status == "pass" assert result.category_outcomes[0].status == "pass" # #endregion Test.Api.DashboardTesting.VerificationPersistence.RealExecutors.Structure # #endregion Test.Api.DashboardTesting.VerificationPersistence.RealExecutors # #region Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys [C:4] [TYPE Class] [SEMANTICS test,verification,fk,ondelete,repository-set-null] class TestVerificationRunForeignKeys: """The verification record retains audit history when optional parent rows are deleted.""" # #region Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys.AgentRunSetNull [C:2] [TYPE Function] [SEMANTICS test,verification,agent-run,fk] # @BRIEF Deleting a linked AgentRun sets verification_runs.agent_run_id to NULL. def test_agent_run_delete_sets_link_to_null(self, db_session: Session): agent_run = AgentRun(id=str(uuid4()), user_id="qa", intent="dashboard_scenario_build", trigger="manual", dashboard_id="42", environment_id="ss-preprod", context_snapshot={}, status="CREATED") db_session.add(agent_run) db_session.commit() result = create_verification_run(db_session, _request("metric", {"comparisons": []}, agent_run_id=agent_run.id)) db_session.delete(agent_run) db_session.commit() record = db_session.get(VerificationRunRecord, str(result.id)) assert record is not None assert record.agent_run_id is None # #endregion Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys.AgentRunSetNull # #region Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys.ReleaseSetNull [C:2] [TYPE Function] [SEMANTICS test,verification,release,fk] # @BRIEF Deleting a linked release sets verification_runs.release_id to NULL. def test_release_delete_sets_link_to_null(self, db_session: Session): release = _linked_release(db_session) result = create_verification_run(db_session, _request("metric", {"comparisons": []}, repository_id=release.repository_id, release_id=release.id)) db_session.delete(release) db_session.commit() record = db_session.get(VerificationRunRecord, str(result.id)) assert record is not None assert record.release_id is None # #endregion Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys.ReleaseSetNull # #region Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys.RepositorySetNull [C:2] [TYPE Function] [SEMANTICS test,verification,repository,fk,set-null] # @BRIEF Deleting a linked GitRepository sets verification_runs.repository_id to NULL. def test_repository_delete_sets_link_to_null(self, db_session: Session): # Create a dedicated repository for this test (not the shared _REPOSITORY_ID) repo_id = str(uuid4()) server = GitServerConfig(id=str(uuid4()), name="fk-test-server", provider="GITHUB", url="https://fk.test", pat="token") db_session.add(server) db_session.flush() repo = GitRepository(id=repo_id, dashboard_id=7002, config_id=server.id, remote_url="https://fk.test/repo.git", local_path="/tmp/fk") db_session.add(repo) db_session.commit() # Create verification run referencing this repository result = create_verification_run(db_session, _request("metric", {"comparisons": []}, repository_id=repo_id)) # Delete the repository db_session.delete(repo) db_session.commit() record = db_session.get(VerificationRunRecord, str(result.id)) assert record is not None, "VerificationRunRecord should survive repository deletion" assert record.repository_id is None, \ f"repository_id should be NULL after parent delete, got {record.repository_id!r}" # #endregion Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys.RepositorySetNull # #endregion Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys # #region Test.Api.DashboardTesting.VerificationPersistence.CommitRollback [C:2] [TYPE Function] [SEMANTICS test,verification,rollback] # @BRIEF A database commit exception leaves no partially persisted VerificationRunRecord. def test_commit_failure_rolls_back_verification_record(db_session: Session): def reject_commit(_session: Session) -> None: raise RuntimeError("forced commit failure") event.listen(db_session, "before_commit", reject_commit, once=True) with pytest.raises(RuntimeError, match="forced commit failure"): create_verification_run(db_session, _request("metric", {"comparisons": []})) assert db_session.query(VerificationRunRecord).count() == 0 # #endregion Test.Api.DashboardTesting.VerificationPersistence.CommitRollback # #region Test.Api.DashboardTesting.VerificationPersistence.RepositoryValidation [C:2] [TYPE Class] [SEMANTICS test,verification,repository,validation,invalid-fk] class TestVerificationRepositoryValidation: """Verification service validates repository existence before creating a run.""" # #region Test.Api.DashboardTesting.VerificationPersistence.RepositoryValidation.Missing [C:2] [TYPE Function] # @BRIEF Referencing a non-existent GitRepository raises ValueError. def test_missing_repository_raises_value_error(self, db_session: Session): """T047: create_verification_run raises ValueError when repository does not exist.""" bogus_repo_id = "00000000-0000-0000-0000-000000000000" with pytest.raises(ValueError, match=r"repository_id.*not found"): create_verification_run( db_session, _request("metric", {"comparisons": []}, repository_id=bogus_repo_id), ) # #endregion Test.Api.DashboardTesting.VerificationPersistence.RepositoryValidation.Missing # #region Test.Api.DashboardTesting.VerificationPersistence.RepositoryValidation.ValidSucceeds [C:2] [TYPE Function] # @BRIEF Referencing an existing GitRepository succeeds. def test_valid_repository_succeeds(self, db_session: Session): """T047: create_verification_run succeeds when repository exists.""" result = create_verification_run( db_session, _request("metric", {"comparisons": []}, repository_id=_REPOSITORY_ID), ) assert result is not None assert str(result.repository_id) == _REPOSITORY_ID # #endregion Test.Api.DashboardTesting.VerificationPersistence.RepositoryValidation.ValidSucceeds # #endregion Test.Api.DashboardTesting.VerificationPersistence.RepositoryValidation # #region Test.Api.DashboardTesting.VerificationPersistence.ReleaseConsistency [C:2] [TYPE Class] [SEMANTICS test,verification,release,consistency,repository-mismatch] class TestVerificationReleaseConsistency: """Verification service validates release belongs to the claimed repository.""" # #region Test.Api.DashboardTesting.VerificationPersistence.ReleaseConsistency.Mismatch [C:2] [TYPE Function] # @BRIEF Referencing a release that belongs to a different repository raises ValueError. def test_release_repository_mismatch_raises(self, db_session: Session): """T047: Creating a run with release_id from a different repository raises ValueError.""" release = _linked_release(db_session) # Use a different (but existing) repository other_repo_id = str(uuid4()) server = GitServerConfig(id=str(uuid4()), name="other-server", provider="GITHUB", url="https://other.test", pat="token") db_session.add(server) db_session.flush() other_repo = GitRepository(id=other_repo_id, dashboard_id=8000, config_id=server.id, remote_url="https://other.test/repo.git", local_path="/tmp/other") db_session.add(other_repo) db_session.commit() with pytest.raises(ValueError, match="belongs to repository"): create_verification_run( db_session, _request("metric", {"comparisons": []}, repository_id=other_repo_id, release_id=release.id), ) # #endregion Test.Api.DashboardTesting.VerificationPersistence.ReleaseConsistency.Mismatch # #region Test.Api.DashboardTesting.VerificationPersistence.ReleaseConsistency.Valid [C:2] [TYPE Function] # @BRIEF Referencing a release that belongs to the same repository succeeds. def test_release_repository_consistent_succeeds(self, db_session: Session): """T047: Creating a run with consistent release+repository succeeds.""" release = _linked_release(db_session) result = create_verification_run( db_session, _request("metric", {"comparisons": []}, repository_id=release.repository_id, release_id=release.id), ) assert result is not None assert str(result.release_id) == release.id # #endregion Test.Api.DashboardTesting.VerificationPersistence.ReleaseConsistency.Valid # #endregion Test.Api.DashboardTesting.VerificationPersistence.ReleaseConsistency # #region Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence [C:3] [TYPE Class] [SEMANTICS test,verification,visual,independent-evidence,draft-artifact,fk,api-persisted] class TestVisualIndependentEvidence: """Verify independent-evidence invariant: actual evidence AgentRun MUST differ from VisualBaselineEntry.provenance.agent_run_id. Same-run actual/expected is blocked.""" # #region Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence.VisualEnv [C:3] [TYPE Function] [SEMANTICS test,verification,visual,fixture,catalog,draft,fk] # @BRIEF Create catalog + repository + release + AgentRuns + FK-enforced DraftArtifacts for visual testing. # @SIDE_EFFECT Creates temp catalog YAML on filesystem. # @SIDE_EFFECT Resets DraftStorage singleton for test isolation. @pytest.fixture def _visual_env(self, db_session: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict: """Set up full visual verification environment with FK-enforced DraftArtifacts.""" from src.services.agent_runs.artifacts import get_draft_storage # ── Temp paths ────────────────────────────────────── drafts_root = tmp_path / "drafts" drafts_root.mkdir() # Reset DraftStorage singleton for test isolation monkeypatch.setattr("src.services.agent_runs.artifacts._draft_storage", None) monkeypatch.setenv("DRAFT_STORAGE_ROOT", str(drafts_root)) # Override safe_path default base to tmp_path so catalog uses test-relative path monkeypatch.setattr( "src.services.dashboard_testing.safe_path._DEFAULT_BASE", tmp_path.resolve(), ) # ── Baseline AgentRun (captured expected screenshot) ── baseline_run = AgentRun( id=str(uuid4()), user_id="qa", intent="dashboard_scenario_build", trigger="manual", dashboard_id="12345", environment_id="ss-preprod", context_snapshot={}, status="CREATED", ) db_session.add(baseline_run) # ── Evidence AgentRun (provides actual screenshot) ──── evidence_run = AgentRun( id=str(uuid4()), user_id="qa", intent="dashboard_scenario_build", trigger="manual", dashboard_id="12345", environment_id="ss-preprod", context_snapshot={}, status="CREATED", ) db_session.add(evidence_run) # ── Repository ────────────────────────────────────── repo_key = "test-vis-repo" dash_key = "dash_12345" server = GitServerConfig( id=str(uuid4()), name="vis-test-server", provider="GITHUB", url="https://vis-test.test", pat="token", ) db_session.add(server) db_session.flush() repo = GitRepository( id=str(uuid4()), dashboard_id=12345, config_id=server.id, remote_url="https://vis-test.test/repo.git", local_path=f"/tmp/{repo_key}", ) db_session.add(repo) db_session.flush() # ── Release with approved status ──────────────────── env = DeploymentEnvironment( id=str(uuid4()), name="test", superset_url="https://superset.test", superset_token="token", ) db_session.add(env) db_session.flush() deployment = DeploymentRecord( repository_id=repo.id, environment_id=env.id, commit_hash="a" * 40, content_hash="b" * 64, deployed_by="qa", ) db_session.add(deployment) db_session.flush() release = DashboardRelease( id=str(uuid4()), repository_id=repo.id, deployment_id=deployment.id, name="v1.0.0", version="v1.0.0", notes="test", commit_hash="a" * 40, content_hash="b" * 64, created_by="qa", status="approved", ) db_session.add(release) db_session.flush() # ── Expected screenshot DraftArtifact (baseline run) ─ storage = get_draft_storage() expected_bytes = b"expected_screenshot_png_bytes" expected_hash = hashlib.sha256(expected_bytes).hexdigest() expected_content_ref = storage.store(baseline_run.id, expected_hash, expected_bytes) expected_draft = DraftArtifact( id=str(uuid4()), run_id=baseline_run.id, kind="visual", name="expected_screenshot.png", intended_path="/tmp/expected.png", content_ref=expected_content_ref, sha256=expected_hash, ) db_session.add(expected_draft) # ── Actual screenshot DraftArtifact (evidence run) ─── actual_bytes = b"actual_screenshot_png_bytes" actual_hash = hashlib.sha256(actual_bytes).hexdigest() actual_content_ref = storage.store(evidence_run.id, actual_hash, actual_bytes) actual_draft = DraftArtifact( id=str(uuid4()), run_id=evidence_run.id, kind="screenshot_evidence", name="actual_screenshot.png", intended_path="/tmp/actual.png", content_ref=actual_content_ref, sha256=actual_hash, ) db_session.add(actual_draft) # ── Catalog YAML at expected path ──────────────────── catalog_dir = tmp_path / "git_repos" / repo_key / "dashboard_tests" / dash_key catalog_dir.mkdir(parents=True) catalog_yaml = { "schema_version": 1, "dashboard": {"id": 12345}, "entries": [{ "schema_version": 1, "baseline_id": str(uuid4()), "kind": "visual", "release_version": "v1.0.0", "release_commit_hash": "a" * 40, "dashboard_id": 12345, "tab_identifier": "tab1", "expected_image_sha256": expected_hash, "expected_image_content_ref": expected_content_ref, "source_response_hash": "d" * 64, "captured_at": "2026-01-01T00:00:00Z", "policy": {"type": "exact"}, "status": "approved", "fingerprints": { "layout": "f" * 64, "query": "f" * 64, "dataset": "f" * 64, "filter": "f" * 64, }, "provenance": { "environment": "test", "actor": "test", "agent_run_id": baseline_run.id, }, "approval": { "by": "test", "at": "2026-01-01T00:00:00Z", }, "normalized_filters": { "filters": [], "filters_hash": "f" * 64, }, "created_at": "2026-01-01T00:00:00Z", }], } (catalog_dir / "baselines.yaml").write_text(yaml.safe_dump(catalog_yaml)) db_session.commit() return { "repo": repo, "release": release, "env": env, "baseline_run": baseline_run, "evidence_run": evidence_run, "actual_draft": actual_draft, "expected_draft": expected_draft, } # #endregion Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence.VisualEnv # #region Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence.SameRunBlocked [C:2] [TYPE Function] [SEMANTICS test,verification,visual,independent-evidence,same-run-blocked] # @BRIEF Actual evidence from same AgentRun as baseline provenance is BLOCKED by independent-evidence invariant. # @TEST_EDGE same_run_evidence -> Using the same agent_run_id as the baseline capture yields blocked with # "independent-evidence invariant" in the outcome summary. # @INVARIANT The visual executor MUST reject same-run actual/expected before resolving any artifacts. def test_same_run_blocked(self, db_session: Session, _visual_env: dict): """Same-run actual evidence is BLOCKED by independent-evidence invariant.""" env = _visual_env result = create_verification_run(db_session, _visual_request( "visual", {"dashboard_id": 12345, "tab_identifier": "tab1"}, evidence_refs=[env["actual_draft"].id], agent_run_id=env["baseline_run"].id, release_id=env["release"].id, repository_id=env["repo"].id, environment_id=env["env"].id, )) assert result.overall_status == "blocked", ( f"Expected blocked for same-run evidence, " f"got {result.overall_status}: {result.category_outcomes[0].summary}" ) outcome = result.category_outcomes[0] assert outcome.status == "blocked" assert "independent-evidence invariant" in outcome.summary.lower(), ( f"Expected independent-evidence invariant message, got: {outcome.summary}" ) # Verify the run IS persisted (immutable audit record) record = db_session.get(VerificationRunRecord, str(result.id)) assert record is not None, "Blocked verification run MUST be persisted" assert record.overall_status == "blocked" # #endregion Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence.SameRunBlocked # #region Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence.SeparateRunAllowed [C:2] [TYPE Function] [SEMANTICS test,verification,visual,independent-evidence,separate-run-allowed] # @BRIEF Actual evidence from a different AgentRun than baseline passes the independent-evidence invariant check. # The verification proceeds (may be blocked at later stages like Superset connectivity). # @TEST_EDGE separate_run_evidence -> Using a different agent_run_id than the baseline capture allows # the verification to proceed past the independent-evidence check. def test_separate_run_allowed(self, db_session: Session, _visual_env: dict): """Separate-run actual evidence passes the independent-evidence invariant check.""" env = _visual_env result = create_verification_run(db_session, _visual_request( "visual", {"dashboard_id": 12345, "tab_identifier": "tab1"}, evidence_refs=[env["actual_draft"].id], agent_run_id=env["evidence_run"].id, release_id=env["release"].id, repository_id=env["repo"].id, environment_id=env["env"].id, )) outcome = result.category_outcomes[0] # The independent-evidence check passed. The verification may still be blocked # at later stages (Superset connectivity, fingerprint computation, etc.), but # it MUST NOT be blocked by the independent-evidence invariant. err_msg = outcome.summary.lower() assert "independent-evidence" not in err_msg, ( f"Separate-run should not trigger independent-evidence invariant, " f"got: {outcome.summary}" ) # Verify the run IS persisted regardless of blocking status record = db_session.get(VerificationRunRecord, str(result.id)) assert record is not None, "Verification run MUST be persisted even when blocked later" # #endregion Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence.SeparateRunAllowed # #endregion Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence # #endregion Test.Api.DashboardTesting.VerificationPersistence