busya
a32ca0631b
- Authoritative candidate capture with server-issued artifacts and raw-byte
immutability hashing (source_response_hash server-owned)
- Closed-period lifecycle: request-hash bound approvals, persisted closure
immutability violations, byte-for-byte catalog stability on reclosure
- Verification runs: persisted VerificationRun model + FK migration,
publish gate (block_publish), scheduled observability runs (02:00 UTC)
- FR-013 baseline inheritance: prior_release_id migration, plan_inheritance/
execute_inheritance classification and re-extraction, API endpoints
- Visual executor bound to release-deployment environment; caller mismatch
rejected; visual SSIM/reconciliation modules
- Query execution decomposed: envelope/model/executor split, no direct SQL
- AgentRun approvals extracted to submodule; evidence adapter; _utils
- Dashboard testing service decomposed into 30+ modules (all <400 LOC)
- Five Feature-037 agent tools with permission guards (tools_037.py)
- API readiness endpoint; Alembic env/migrations; test fixture repos
- Specs 036/037 contracts, openapi.yaml, schema.json, tasks/traceability
updated; semantic index rebuilt with 0 parse warnings
- Fix ADR-0003 parser ambiguity: remove [DEF🆔ADR] prose example
- Add axiom-mcp-agent-feedback.md: agent findings for MCP rework plan
- Tests: 298 service + 1464 API + 45 agent passing; ruff clean
571 lines
30 KiB
Python
571 lines
30 KiB
Python
# #region Test.Api.DashboardTesting.VerificationPersistence [C:4] [TYPE Module] [SEMANTICS testing,api,dashboard-testing,verification,persistence,repository-fk]
|
|
# @defgroup VerificationRun persistence, real executor, and transaction rollback tests.
|
|
# @LAYER Test
|
|
# @RELATION BINDS_TO -> [BaselineEngine.Verification.Service]
|
|
# @TEST_FIXTURE executor_params -> INLINE_JSON
|
|
# @TEST_EDGE metric_outcome -> Hardcoded matching values yield pass through the real comparison service.
|
|
# @TEST_EDGE visual_outcome -> Matching image hashes yield pass through the real visual service.
|
|
# @TEST_EDGE structure_outcome -> Matching persisted snapshots yield pass through the real diff service.
|
|
# @TEST_EDGE fk_set_null -> Deleting linked AgentRun or DashboardRelease preserves the verification record.
|
|
# @TEST_EDGE repository_fk_set_null -> Deleting GitRepository sets verification_runs.repository_id to NULL.
|
|
# @TEST_EDGE invalid_repository -> Referencing non-existent GitRepository raises ValueError.
|
|
# @TEST_EDGE commit_failure -> A commit exception rolls back the unpersisted verification record.
|
|
# @TEST_EDGE independent_evidence_same_run -> Actual evidence from same AgentRun as baseline is BLOCKED.
|
|
# @TEST_EDGE independent_evidence_separate_run -> Actual evidence from different AgentRun passes invariant.
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from pathlib import Path
|
|
import pytest
|
|
from uuid import uuid4
|
|
|
|
from sqlalchemy import create_engine, event
|
|
from sqlalchemy.orm import Session
|
|
import yaml
|
|
|
|
from src.models.agent_run import AgentRun, DraftArtifact
|
|
from src.models.dashboard_release import DashboardRelease
|
|
from src.models.deployment import DeploymentRecord
|
|
from src.models.git import DeploymentEnvironment, GitRepository, GitServerConfig
|
|
from src.models.mapping import Base
|
|
from src.models.verification_run import VerificationRunRecord
|
|
from src.schemas.dashboard_testing import VerificationRunRequest
|
|
from src.services.dashboard_testing.structure_diff_service import set_snapshot_base_path
|
|
from src.services.dashboard_testing.verification_service import create_verification_run
|
|
|
|
_ENGINE = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False})
|
|
event.listen(_ENGINE, "connect", lambda connection, _: connection.execute("PRAGMA foreign_keys=ON"))
|
|
Base.metadata.create_all(bind=_ENGINE)
|
|
_FIXTURE_BASE = Path(__file__).parents[1] / "fixtures" / "structure_diff"
|
|
_REPOSITORY_ID = "550e8400-e29b-41d4-a716-446655440000"
|
|
|
|
|
|
def _ensure_repository() -> GitRepository:
|
|
"""Create the default GitRepository in the shared engine if not present."""
|
|
with Session(_ENGINE) as session:
|
|
existing = session.query(GitRepository).filter(GitRepository.id == _REPOSITORY_ID).first()
|
|
if existing:
|
|
return existing
|
|
server = GitServerConfig(
|
|
id=str(uuid4()), name="auto-server", provider="GITHUB",
|
|
url="https://auto.test", pat="token",
|
|
)
|
|
session.add(server)
|
|
session.flush() # Ensure server is persisted before repo references it
|
|
repo = GitRepository(
|
|
id=_REPOSITORY_ID, dashboard_id=9999,
|
|
config_id=server.id, remote_url="https://auto.test/repo.git",
|
|
local_path="/tmp/auto",
|
|
)
|
|
session.add(repo)
|
|
session.commit()
|
|
return repo
|
|
|
|
|
|
# Ensure the default repository exists at module load time
|
|
_ensure_repository()
|
|
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.DbSession [C:2] [TYPE Function] [SEMANTICS test,verification,fixture]
|
|
# @BRIEF Supply an isolated SQLite session with foreign-key enforcement.
|
|
@pytest.fixture
|
|
def db_session() -> Session:
|
|
connection = _ENGINE.connect()
|
|
transaction = connection.begin()
|
|
session = Session(bind=connection)
|
|
try:
|
|
yield session
|
|
finally:
|
|
session.close()
|
|
if transaction.is_active:
|
|
transaction.rollback()
|
|
connection.close()
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.DbSession
|
|
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.Request [C:1] [TYPE Function] [SEMANTICS test,verification,fixture]
|
|
def _request(category: str, params: dict, **links: str) -> VerificationRunRequest:
|
|
return VerificationRunRequest(
|
|
repository_id=links.get("repository_id", _REPOSITORY_ID),
|
|
trigger="manual",
|
|
environment_id="ss-preprod",
|
|
categories=[category],
|
|
category_params={category: params},
|
|
agent_run_id=links.get("agent_run_id"),
|
|
release_id=links.get("release_id"),
|
|
)
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.Request
|
|
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.VisualRequest [C:1] [TYPE Function] [SEMANTICS test,verification,visual,evidence-refs]
|
|
def _visual_request(
|
|
category: str,
|
|
params: dict,
|
|
evidence_refs: list[str] | None = None,
|
|
**links: str,
|
|
) -> VerificationRunRequest:
|
|
"""Create a VerificationRunRequest with evidence_refs for visual/testing categories."""
|
|
return VerificationRunRequest(
|
|
repository_id=links.get("repository_id", _REPOSITORY_ID),
|
|
trigger="manual",
|
|
environment_id=links.get("environment_id", "ss-preprod"),
|
|
categories=[category],
|
|
category_params={category: params},
|
|
evidence_refs={category: evidence_refs} if evidence_refs else None,
|
|
agent_run_id=links.get("agent_run_id"),
|
|
release_id=links.get("release_id"),
|
|
)
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.VisualRequest
|
|
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.LinkedRelease [C:1] [TYPE Function] [SEMANTICS test,verification,release,fixture]
|
|
def _linked_release(session: Session) -> DashboardRelease:
|
|
# Use the pre-existing repository (created at module load)
|
|
repository = session.query(GitRepository).filter(GitRepository.id == _REPOSITORY_ID).first()
|
|
assert repository is not None, "Default repository must exist"
|
|
environment = DeploymentEnvironment(id=str(uuid4()), name="test", superset_url="https://superset.test", superset_token="token")
|
|
session.add(environment)
|
|
session.flush()
|
|
deployment = DeploymentRecord(repository_id=repository.id, environment_id=environment.id, commit_hash="a" * 40, content_hash="b" * 64, deployed_by="qa")
|
|
session.add(deployment)
|
|
session.flush()
|
|
release = DashboardRelease(id=str(uuid4()), repository_id=repository.id, deployment_id=deployment.id, name="v1.0.0", version="v1.0.0", notes="test", commit_hash="a" * 40, content_hash="b" * 64, created_by="qa")
|
|
session.add(release)
|
|
session.commit()
|
|
return release
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.LinkedRelease
|
|
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.RealExecutors [C:3] [TYPE Class] [SEMANTICS test,verification,executor,real]
|
|
class TestRealExecutorOutcomes:
|
|
"""Verification categories must derive statuses from their real domain services."""
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.RealExecutors.Metric [C:2] [TYPE Function] [SEMANTICS test,verification,metric]
|
|
# @BRIEF Matching hardcoded normalized values yield a real metric pass.
|
|
def test_metric_executor_reports_real_pass(self, db_session: Session):
|
|
result = create_verification_run(db_session, _request("metric", {
|
|
"comparisons": [{
|
|
"actual": {"kind": "integer", "canonical_value": "7"},
|
|
"expected": {"kind": "integer", "canonical_value": "7"},
|
|
"policy": {"type": "exact"},
|
|
}],
|
|
}))
|
|
assert result.overall_status == "pass"
|
|
assert result.category_outcomes[0].status == "pass"
|
|
assert result.category_outcomes[0].details["comparisons"][0]["status"] == "pass"
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.RealExecutors.Metric
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.RealExecutors.Visual [C:2] [TYPE Function] [SEMANTICS test,verification,visual,security]
|
|
# @BRIEF Caller-supplied expected_image_sha256 without catalog is now BLOCKED (security fix).
|
|
# The executor no longer accepts hashes/policy from params — it requires catalog + evidence.
|
|
def test_visual_executor_rejects_caller_hashes_without_catalog(self, db_session: Session):
|
|
image_hash = "c" * 64
|
|
# agent_run_id and release_id are required by VerificationRunRequest for visual category
|
|
visual_release = _linked_release(db_session)
|
|
visual_release_id = visual_release.id
|
|
# Create a valid AgentRun record
|
|
from src.schemas.agent_run import CreateAgentRunRequest, UIContextV2
|
|
from src.services.agent_runs.service import create_agent_run
|
|
agent_run = create_agent_run(db_session, CreateAgentRunRequest(
|
|
context=UIContextV2(objectType="dashboard", objectId="42",
|
|
envId="ss-preprod", route="/dashboards/42", contextVersion=2,
|
|
intent="build_dashboard_test_scenario"),
|
|
), user_id="qa")
|
|
result = create_verification_run(db_session, _request("visual", {
|
|
"actual_image_sha256": image_hash,
|
|
"expected_image_sha256": image_hash,
|
|
"policy": {"type": "visual_exact"},
|
|
}, agent_run_id=agent_run.id, release_id=visual_release_id))
|
|
assert result.overall_status == "blocked", (
|
|
f"Expected blocked (caller hashes rejected without catalog), "
|
|
f"got {result.overall_status}: {result.category_outcomes[0].summary}"
|
|
)
|
|
assert result.category_outcomes[0].status == "blocked"
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.RealExecutors.Visual
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.RealExecutors.Structure [C:2] [TYPE Function] [SEMANTICS test,verification,structure]
|
|
# @BRIEF Identical persisted snapshots yield a real structure pass without mocking the diff service.
|
|
def test_structure_executor_reports_real_pass(self, db_session: Session):
|
|
set_snapshot_base_path(_FIXTURE_BASE)
|
|
try:
|
|
result = create_verification_run(db_session, _request("structure", {
|
|
"dashboard_id": 42,
|
|
"release_version_from": "v1.0.0",
|
|
"release_version_to": "v1.0.0",
|
|
}))
|
|
finally:
|
|
set_snapshot_base_path(None)
|
|
assert result.overall_status == "pass"
|
|
assert result.category_outcomes[0].status == "pass"
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.RealExecutors.Structure
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.RealExecutors
|
|
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys [C:4] [TYPE Class] [SEMANTICS test,verification,fk,ondelete,repository-set-null]
|
|
class TestVerificationRunForeignKeys:
|
|
"""The verification record retains audit history when optional parent rows are deleted."""
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys.AgentRunSetNull [C:2] [TYPE Function] [SEMANTICS test,verification,agent-run,fk]
|
|
# @BRIEF Deleting a linked AgentRun sets verification_runs.agent_run_id to NULL.
|
|
def test_agent_run_delete_sets_link_to_null(self, db_session: Session):
|
|
agent_run = AgentRun(id=str(uuid4()), user_id="qa", intent="dashboard_scenario_build", trigger="manual", dashboard_id="42", environment_id="ss-preprod", context_snapshot={}, status="CREATED")
|
|
db_session.add(agent_run)
|
|
db_session.commit()
|
|
result = create_verification_run(db_session, _request("metric", {"comparisons": []}, agent_run_id=agent_run.id))
|
|
db_session.delete(agent_run)
|
|
db_session.commit()
|
|
record = db_session.get(VerificationRunRecord, str(result.id))
|
|
assert record is not None
|
|
assert record.agent_run_id is None
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys.AgentRunSetNull
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys.ReleaseSetNull [C:2] [TYPE Function] [SEMANTICS test,verification,release,fk]
|
|
# @BRIEF Deleting a linked release sets verification_runs.release_id to NULL.
|
|
def test_release_delete_sets_link_to_null(self, db_session: Session):
|
|
release = _linked_release(db_session)
|
|
result = create_verification_run(db_session, _request("metric", {"comparisons": []}, repository_id=release.repository_id, release_id=release.id))
|
|
db_session.delete(release)
|
|
db_session.commit()
|
|
record = db_session.get(VerificationRunRecord, str(result.id))
|
|
assert record is not None
|
|
assert record.release_id is None
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys.ReleaseSetNull
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys.RepositorySetNull [C:2] [TYPE Function] [SEMANTICS test,verification,repository,fk,set-null]
|
|
# @BRIEF Deleting a linked GitRepository sets verification_runs.repository_id to NULL.
|
|
def test_repository_delete_sets_link_to_null(self, db_session: Session):
|
|
# Create a dedicated repository for this test (not the shared _REPOSITORY_ID)
|
|
repo_id = str(uuid4())
|
|
server = GitServerConfig(id=str(uuid4()), name="fk-test-server", provider="GITHUB", url="https://fk.test", pat="token")
|
|
db_session.add(server)
|
|
db_session.flush()
|
|
repo = GitRepository(id=repo_id, dashboard_id=7002, config_id=server.id, remote_url="https://fk.test/repo.git", local_path="/tmp/fk")
|
|
db_session.add(repo)
|
|
db_session.commit()
|
|
# Create verification run referencing this repository
|
|
result = create_verification_run(db_session, _request("metric", {"comparisons": []}, repository_id=repo_id))
|
|
# Delete the repository
|
|
db_session.delete(repo)
|
|
db_session.commit()
|
|
record = db_session.get(VerificationRunRecord, str(result.id))
|
|
assert record is not None, "VerificationRunRecord should survive repository deletion"
|
|
assert record.repository_id is None, \
|
|
f"repository_id should be NULL after parent delete, got {record.repository_id!r}"
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys.RepositorySetNull
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.ForeignKeys
|
|
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.CommitRollback [C:2] [TYPE Function] [SEMANTICS test,verification,rollback]
|
|
# @BRIEF A database commit exception leaves no partially persisted VerificationRunRecord.
|
|
def test_commit_failure_rolls_back_verification_record(db_session: Session):
|
|
def reject_commit(_session: Session) -> None:
|
|
raise RuntimeError("forced commit failure")
|
|
|
|
event.listen(db_session, "before_commit", reject_commit, once=True)
|
|
with pytest.raises(RuntimeError, match="forced commit failure"):
|
|
create_verification_run(db_session, _request("metric", {"comparisons": []}))
|
|
assert db_session.query(VerificationRunRecord).count() == 0
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.CommitRollback
|
|
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.RepositoryValidation [C:2] [TYPE Class] [SEMANTICS test,verification,repository,validation,invalid-fk]
|
|
class TestVerificationRepositoryValidation:
|
|
"""Verification service validates repository existence before creating a run."""
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.RepositoryValidation.Missing [C:2] [TYPE Function]
|
|
# @BRIEF Referencing a non-existent GitRepository raises ValueError.
|
|
def test_missing_repository_raises_value_error(self, db_session: Session):
|
|
"""T047: create_verification_run raises ValueError when repository does not exist."""
|
|
bogus_repo_id = "00000000-0000-0000-0000-000000000000"
|
|
with pytest.raises(ValueError, match=r"repository_id.*not found"):
|
|
create_verification_run(
|
|
db_session,
|
|
_request("metric", {"comparisons": []}, repository_id=bogus_repo_id),
|
|
)
|
|
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.RepositoryValidation.Missing
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.RepositoryValidation.ValidSucceeds [C:2] [TYPE Function]
|
|
# @BRIEF Referencing an existing GitRepository succeeds.
|
|
def test_valid_repository_succeeds(self, db_session: Session):
|
|
"""T047: create_verification_run succeeds when repository exists."""
|
|
result = create_verification_run(
|
|
db_session,
|
|
_request("metric", {"comparisons": []}, repository_id=_REPOSITORY_ID),
|
|
)
|
|
assert result is not None
|
|
assert str(result.repository_id) == _REPOSITORY_ID
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.RepositoryValidation.ValidSucceeds
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.RepositoryValidation
|
|
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.ReleaseConsistency [C:2] [TYPE Class] [SEMANTICS test,verification,release,consistency,repository-mismatch]
|
|
class TestVerificationReleaseConsistency:
|
|
"""Verification service validates release belongs to the claimed repository."""
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.ReleaseConsistency.Mismatch [C:2] [TYPE Function]
|
|
# @BRIEF Referencing a release that belongs to a different repository raises ValueError.
|
|
def test_release_repository_mismatch_raises(self, db_session: Session):
|
|
"""T047: Creating a run with release_id from a different repository raises ValueError."""
|
|
release = _linked_release(db_session)
|
|
|
|
# Use a different (but existing) repository
|
|
other_repo_id = str(uuid4())
|
|
server = GitServerConfig(id=str(uuid4()), name="other-server", provider="GITHUB", url="https://other.test", pat="token")
|
|
db_session.add(server)
|
|
db_session.flush()
|
|
other_repo = GitRepository(id=other_repo_id, dashboard_id=8000, config_id=server.id, remote_url="https://other.test/repo.git", local_path="/tmp/other")
|
|
db_session.add(other_repo)
|
|
db_session.commit()
|
|
|
|
with pytest.raises(ValueError, match="belongs to repository"):
|
|
create_verification_run(
|
|
db_session,
|
|
_request("metric", {"comparisons": []},
|
|
repository_id=other_repo_id, release_id=release.id),
|
|
)
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.ReleaseConsistency.Mismatch
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.ReleaseConsistency.Valid [C:2] [TYPE Function]
|
|
# @BRIEF Referencing a release that belongs to the same repository succeeds.
|
|
def test_release_repository_consistent_succeeds(self, db_session: Session):
|
|
"""T047: Creating a run with consistent release+repository succeeds."""
|
|
release = _linked_release(db_session)
|
|
result = create_verification_run(
|
|
db_session,
|
|
_request("metric", {"comparisons": []},
|
|
repository_id=release.repository_id, release_id=release.id),
|
|
)
|
|
assert result is not None
|
|
assert str(result.release_id) == release.id
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.ReleaseConsistency.Valid
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.ReleaseConsistency
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence [C:3] [TYPE Class] [SEMANTICS test,verification,visual,independent-evidence,draft-artifact,fk,api-persisted]
|
|
class TestVisualIndependentEvidence:
|
|
"""Verify independent-evidence invariant: actual evidence AgentRun MUST differ
|
|
from VisualBaselineEntry.provenance.agent_run_id. Same-run actual/expected is blocked."""
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence.VisualEnv [C:3] [TYPE Function] [SEMANTICS test,verification,visual,fixture,catalog,draft,fk]
|
|
# @BRIEF Create catalog + repository + release + AgentRuns + FK-enforced DraftArtifacts for visual testing.
|
|
# @SIDE_EFFECT Creates temp catalog YAML on filesystem.
|
|
# @SIDE_EFFECT Resets DraftStorage singleton for test isolation.
|
|
@pytest.fixture
|
|
def _visual_env(self, db_session: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict:
|
|
"""Set up full visual verification environment with FK-enforced DraftArtifacts."""
|
|
from src.services.agent_runs.artifacts import get_draft_storage
|
|
|
|
# ── Temp paths ──────────────────────────────────────
|
|
drafts_root = tmp_path / "drafts"
|
|
drafts_root.mkdir()
|
|
|
|
# Reset DraftStorage singleton for test isolation
|
|
monkeypatch.setattr("src.services.agent_runs.artifacts._draft_storage", None)
|
|
monkeypatch.setenv("DRAFT_STORAGE_ROOT", str(drafts_root))
|
|
# Override safe_path default base to tmp_path so catalog uses test-relative path
|
|
monkeypatch.setattr(
|
|
"src.services.dashboard_testing.safe_path._DEFAULT_BASE",
|
|
tmp_path.resolve(),
|
|
)
|
|
|
|
# ── Baseline AgentRun (captured expected screenshot) ──
|
|
baseline_run = AgentRun(
|
|
id=str(uuid4()), user_id="qa", intent="dashboard_scenario_build",
|
|
trigger="manual", dashboard_id="12345", environment_id="ss-preprod",
|
|
context_snapshot={}, status="CREATED",
|
|
)
|
|
db_session.add(baseline_run)
|
|
|
|
# ── Evidence AgentRun (provides actual screenshot) ────
|
|
evidence_run = AgentRun(
|
|
id=str(uuid4()), user_id="qa", intent="dashboard_scenario_build",
|
|
trigger="manual", dashboard_id="12345", environment_id="ss-preprod",
|
|
context_snapshot={}, status="CREATED",
|
|
)
|
|
db_session.add(evidence_run)
|
|
|
|
# ── Repository ──────────────────────────────────────
|
|
repo_key = "test-vis-repo"
|
|
dash_key = "dash_12345"
|
|
server = GitServerConfig(
|
|
id=str(uuid4()), name="vis-test-server", provider="GITHUB",
|
|
url="https://vis-test.test", pat="token",
|
|
)
|
|
db_session.add(server)
|
|
db_session.flush()
|
|
repo = GitRepository(
|
|
id=str(uuid4()), dashboard_id=12345,
|
|
config_id=server.id, remote_url="https://vis-test.test/repo.git",
|
|
local_path=f"/tmp/{repo_key}",
|
|
)
|
|
db_session.add(repo)
|
|
db_session.flush()
|
|
|
|
# ── Release with approved status ────────────────────
|
|
env = DeploymentEnvironment(
|
|
id=str(uuid4()), name="test",
|
|
superset_url="https://superset.test", superset_token="token",
|
|
)
|
|
db_session.add(env)
|
|
db_session.flush()
|
|
deployment = DeploymentRecord(
|
|
repository_id=repo.id, environment_id=env.id,
|
|
commit_hash="a" * 40, content_hash="b" * 64, deployed_by="qa",
|
|
)
|
|
db_session.add(deployment)
|
|
db_session.flush()
|
|
release = DashboardRelease(
|
|
id=str(uuid4()), repository_id=repo.id, deployment_id=deployment.id,
|
|
name="v1.0.0", version="v1.0.0", notes="test",
|
|
commit_hash="a" * 40, content_hash="b" * 64, created_by="qa",
|
|
status="approved",
|
|
)
|
|
db_session.add(release)
|
|
db_session.flush()
|
|
|
|
# ── Expected screenshot DraftArtifact (baseline run) ─
|
|
storage = get_draft_storage()
|
|
expected_bytes = b"expected_screenshot_png_bytes"
|
|
expected_hash = hashlib.sha256(expected_bytes).hexdigest()
|
|
expected_content_ref = storage.store(baseline_run.id, expected_hash, expected_bytes)
|
|
|
|
expected_draft = DraftArtifact(
|
|
id=str(uuid4()), run_id=baseline_run.id, kind="visual",
|
|
name="expected_screenshot.png", intended_path="/tmp/expected.png",
|
|
content_ref=expected_content_ref, sha256=expected_hash,
|
|
)
|
|
db_session.add(expected_draft)
|
|
|
|
# ── Actual screenshot DraftArtifact (evidence run) ───
|
|
actual_bytes = b"actual_screenshot_png_bytes"
|
|
actual_hash = hashlib.sha256(actual_bytes).hexdigest()
|
|
actual_content_ref = storage.store(evidence_run.id, actual_hash, actual_bytes)
|
|
|
|
actual_draft = DraftArtifact(
|
|
id=str(uuid4()), run_id=evidence_run.id, kind="screenshot_evidence",
|
|
name="actual_screenshot.png", intended_path="/tmp/actual.png",
|
|
content_ref=actual_content_ref, sha256=actual_hash,
|
|
)
|
|
db_session.add(actual_draft)
|
|
|
|
# ── Catalog YAML at expected path ────────────────────
|
|
catalog_dir = tmp_path / "git_repos" / repo_key / "dashboard_tests" / dash_key
|
|
catalog_dir.mkdir(parents=True)
|
|
|
|
catalog_yaml = {
|
|
"schema_version": 1,
|
|
"dashboard": {"id": 12345},
|
|
"entries": [{
|
|
"schema_version": 1,
|
|
"baseline_id": str(uuid4()),
|
|
"kind": "visual",
|
|
"release_version": "v1.0.0",
|
|
"release_commit_hash": "a" * 40,
|
|
"dashboard_id": 12345,
|
|
"tab_identifier": "tab1",
|
|
"expected_image_sha256": expected_hash,
|
|
"expected_image_content_ref": expected_content_ref,
|
|
"source_response_hash": "d" * 64,
|
|
"captured_at": "2026-01-01T00:00:00Z",
|
|
"policy": {"type": "exact"},
|
|
"status": "approved",
|
|
"fingerprints": {
|
|
"layout": "f" * 64, "query": "f" * 64,
|
|
"dataset": "f" * 64, "filter": "f" * 64,
|
|
},
|
|
"provenance": {
|
|
"environment": "test", "actor": "test",
|
|
"agent_run_id": baseline_run.id,
|
|
},
|
|
"approval": {
|
|
"by": "test", "at": "2026-01-01T00:00:00Z",
|
|
},
|
|
"normalized_filters": {
|
|
"filters": [],
|
|
"filters_hash": "f" * 64,
|
|
},
|
|
"created_at": "2026-01-01T00:00:00Z",
|
|
}],
|
|
}
|
|
(catalog_dir / "baselines.yaml").write_text(yaml.safe_dump(catalog_yaml))
|
|
|
|
db_session.commit()
|
|
|
|
return {
|
|
"repo": repo,
|
|
"release": release,
|
|
"env": env,
|
|
"baseline_run": baseline_run,
|
|
"evidence_run": evidence_run,
|
|
"actual_draft": actual_draft,
|
|
"expected_draft": expected_draft,
|
|
}
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence.VisualEnv
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence.SameRunBlocked [C:2] [TYPE Function] [SEMANTICS test,verification,visual,independent-evidence,same-run-blocked]
|
|
# @BRIEF Actual evidence from same AgentRun as baseline provenance is BLOCKED by independent-evidence invariant.
|
|
# @TEST_EDGE same_run_evidence -> Using the same agent_run_id as the baseline capture yields blocked with
|
|
# "independent-evidence invariant" in the outcome summary.
|
|
# @INVARIANT The visual executor MUST reject same-run actual/expected before resolving any artifacts.
|
|
def test_same_run_blocked(self, db_session: Session, _visual_env: dict):
|
|
"""Same-run actual evidence is BLOCKED by independent-evidence invariant."""
|
|
env = _visual_env
|
|
result = create_verification_run(db_session, _visual_request(
|
|
"visual",
|
|
{"dashboard_id": 12345, "tab_identifier": "tab1"},
|
|
evidence_refs=[env["actual_draft"].id],
|
|
agent_run_id=env["baseline_run"].id,
|
|
release_id=env["release"].id,
|
|
repository_id=env["repo"].id,
|
|
environment_id=env["env"].id,
|
|
))
|
|
assert result.overall_status == "blocked", (
|
|
f"Expected blocked for same-run evidence, "
|
|
f"got {result.overall_status}: {result.category_outcomes[0].summary}"
|
|
)
|
|
outcome = result.category_outcomes[0]
|
|
assert outcome.status == "blocked"
|
|
assert "independent-evidence invariant" in outcome.summary.lower(), (
|
|
f"Expected independent-evidence invariant message, got: {outcome.summary}"
|
|
)
|
|
# Verify the run IS persisted (immutable audit record)
|
|
record = db_session.get(VerificationRunRecord, str(result.id))
|
|
assert record is not None, "Blocked verification run MUST be persisted"
|
|
assert record.overall_status == "blocked"
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence.SameRunBlocked
|
|
|
|
# #region Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence.SeparateRunAllowed [C:2] [TYPE Function] [SEMANTICS test,verification,visual,independent-evidence,separate-run-allowed]
|
|
# @BRIEF Actual evidence from a different AgentRun than baseline passes the independent-evidence invariant check.
|
|
# The verification proceeds (may be blocked at later stages like Superset connectivity).
|
|
# @TEST_EDGE separate_run_evidence -> Using a different agent_run_id than the baseline capture allows
|
|
# the verification to proceed past the independent-evidence check.
|
|
def test_separate_run_allowed(self, db_session: Session, _visual_env: dict):
|
|
"""Separate-run actual evidence passes the independent-evidence invariant check."""
|
|
env = _visual_env
|
|
result = create_verification_run(db_session, _visual_request(
|
|
"visual",
|
|
{"dashboard_id": 12345, "tab_identifier": "tab1"},
|
|
evidence_refs=[env["actual_draft"].id],
|
|
agent_run_id=env["evidence_run"].id,
|
|
release_id=env["release"].id,
|
|
repository_id=env["repo"].id,
|
|
environment_id=env["env"].id,
|
|
))
|
|
outcome = result.category_outcomes[0]
|
|
# The independent-evidence check passed. The verification may still be blocked
|
|
# at later stages (Superset connectivity, fingerprint computation, etc.), but
|
|
# it MUST NOT be blocked by the independent-evidence invariant.
|
|
err_msg = outcome.summary.lower()
|
|
assert "independent-evidence" not in err_msg, (
|
|
f"Separate-run should not trigger independent-evidence invariant, "
|
|
f"got: {outcome.summary}"
|
|
)
|
|
# Verify the run IS persisted regardless of blocking status
|
|
record = db_session.get(VerificationRunRecord, str(result.id))
|
|
assert record is not None, "Verification run MUST be persisted even when blocked later"
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence.SeparateRunAllowed
|
|
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence.IndependentEvidence
|
|
|
|
# #endregion Test.Api.DashboardTesting.VerificationPersistence
|