busya
a32ca0631b
- Authoritative candidate capture with server-issued artifacts and raw-byte
immutability hashing (source_response_hash server-owned)
- Closed-period lifecycle: request-hash bound approvals, persisted closure
immutability violations, byte-for-byte catalog stability on reclosure
- Verification runs: persisted VerificationRun model + FK migration,
publish gate (block_publish), scheduled observability runs (02:00 UTC)
- FR-013 baseline inheritance: prior_release_id migration, plan_inheritance/
execute_inheritance classification and re-extraction, API endpoints
- Visual executor bound to release-deployment environment; caller mismatch
rejected; visual SSIM/reconciliation modules
- Query execution decomposed: envelope/model/executor split, no direct SQL
- AgentRun approvals extracted to submodule; evidence adapter; _utils
- Dashboard testing service decomposed into 30+ modules (all <400 LOC)
- Five Feature-037 agent tools with permission guards (tools_037.py)
- API readiness endpoint; Alembic env/migrations; test fixture repos
- Specs 036/037 contracts, openapi.yaml, schema.json, tasks/traceability
updated; semantic index rebuilt with 0 parse warnings
- Fix ADR-0003 parser ambiguity: remove [DEF🆔ADR] prose example
- Add axiom-mcp-agent-feedback.md: agent findings for MCP rework plan
- Tests: 298 service + 1464 API + 45 agent passing; ruff clean
277 lines
12 KiB
Python
277 lines
12 KiB
Python
#region Test.BaselineEngine.StructureSnapshot.Mismatch [C:3] [TYPE Module] [SEMANTICS test,structure-snapshot,mismatch,provenance]
|
|
# @defgroup Test.BaselineEngine.StructureSnapshot.Mismatch Provenance dimension mismatch tests.
|
|
# @LAYER Test
|
|
# @RELATION VERIFIES -> [BaselineEngine.StructureSnapshot.Diff]
|
|
# @TEST_EDGE wrong_repo -> envelope repository_id mismatch raises ValueError
|
|
# @TEST_EDGE wrong_release -> envelope release_id mismatch raises ValueError
|
|
# @TEST_EDGE wrong_version -> envelope release_version mismatch raises ValueError
|
|
# @TEST_EDGE wrong_commit -> envelope release_commit_hash mismatch raises ValueError
|
|
# @TEST_EDGE wrong_env -> envelope environment_id mismatch raises ValueError
|
|
# @TEST_EDGE wrong_dashboard -> envelope dashboard_id mismatch raises ValueError
|
|
# @TEST_EDGE cross_repo -> envelope repository_id differs between releases raises ValueError
|
|
# @INVARIANT Every provenance dimension mismatch is detected by diff_release_snapshots.
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
import pytest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from src.models.dashboard_release import DashboardRelease
|
|
from src.models.deployment import DeploymentRecord
|
|
from src.models.git import GitRepository
|
|
from src.schemas.dashboard_testing import DashboardQueryModel
|
|
from src.schemas.dashboard_testing.structure_snapshot import (
|
|
ProvenanceEnvelope,
|
|
SnapshotDiffRequest,
|
|
)
|
|
from src.services.dashboard_testing.structure_snapshot_diff import diff_release_snapshots
|
|
|
|
|
|
# #region Test.StructureSnapshot.Mismatch [C:3] [TYPE Class]
|
|
class TestProvenanceMismatch:
|
|
"""Verify every provenance dimension mismatch is detected."""
|
|
|
|
def _setup_diff_mocks(self) -> tuple:
|
|
"""Set up DB mocks for a release-bound snapshot diff."""
|
|
release_from = MagicMock(spec=DashboardRelease)
|
|
release_from.id = "rel-1"
|
|
release_from.version = "v1.0.0"
|
|
release_from.commit_hash = "a" * 40
|
|
release_from.repository_id = "repo-1"
|
|
release_from.deployment_id = "dep-1"
|
|
|
|
release_to = MagicMock(spec=DashboardRelease)
|
|
release_to.id = "rel-2"
|
|
release_to.version = "v2.0.0"
|
|
release_to.commit_hash = "b" * 40
|
|
release_to.repository_id = "repo-1"
|
|
release_to.deployment_id = "dep-2"
|
|
|
|
repo_record = MagicMock(spec=GitRepository)
|
|
repo_record.id = "repo-1"
|
|
repo_record.dashboard_id = 42
|
|
repo_record.local_path = "git_repos/repo-1"
|
|
|
|
dep1 = MagicMock(spec=DeploymentRecord)
|
|
dep1.id = "dep-1"
|
|
dep1.environment_id = "env-1"
|
|
dep1.commit_hash = "a" * 40
|
|
|
|
dep2 = MagicMock(spec=DeploymentRecord)
|
|
dep2.id = "dep-2"
|
|
dep2.environment_id = "env-1"
|
|
dep2.commit_hash = "b" * 40
|
|
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.side_effect = [
|
|
release_from, release_to, repo_record, dep1, dep2,
|
|
]
|
|
return db, release_from, release_to
|
|
|
|
def _write_snapshot(self, tmp_path: Path, envelope: ProvenanceEnvelope,
|
|
env_id: str = "env-1", dash_id: int = 42) -> Path:
|
|
"""Write a provenance-wrapped snapshot file."""
|
|
p = tmp_path / f"{envelope.release_version}.json"
|
|
model = DashboardQueryModel(
|
|
environment_id=env_id, dashboard_id=dash_id, title="Test",
|
|
query_model_fingerprint="sha256:fp",
|
|
)
|
|
p.write_text(json.dumps({
|
|
"provenance": envelope.model_dump(mode="json"),
|
|
"query_model": model.model_dump(mode="json"),
|
|
}))
|
|
return p
|
|
|
|
def test_wrong_repo_id(self, tmp_path):
|
|
"""Envelope repository_id mismatch raises ValueError."""
|
|
env_from = ProvenanceEnvelope(
|
|
release_id="rel-1", release_version="v1.0.0",
|
|
release_commit_hash="a" * 40, repository_id="repo-1",
|
|
repository_key="repo-1", dashboard_id=42, environment_id="env-1",
|
|
)
|
|
env_to = ProvenanceEnvelope(
|
|
release_id="rel-2", release_version="v2.0.0",
|
|
release_commit_hash="b" * 40, repository_id="repo-999",
|
|
repository_key="repo-999", dashboard_id=42, environment_id="env-1",
|
|
)
|
|
snap_from = self._write_snapshot(tmp_path, env_from)
|
|
snap_to = self._write_snapshot(tmp_path, env_to)
|
|
db, _, _ = self._setup_diff_mocks()
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = str(tmp_path)
|
|
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_diff.build_snapshot_path",
|
|
side_effect=[snap_from, snap_to],
|
|
), pytest.raises(ValueError, match="Snapshot metadata verification failed"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="rel-1", release_id_to="rel-2"),
|
|
db=db, git_service=git_service,
|
|
)
|
|
|
|
def test_wrong_release_id(self, tmp_path):
|
|
"""Envelope release_id mismatch raises ValueError."""
|
|
env_from = ProvenanceEnvelope(
|
|
release_id="rel-wrong", release_version="v1.0.0",
|
|
release_commit_hash="a" * 40, repository_id="repo-1",
|
|
repository_key="repo-1", dashboard_id=42, environment_id="env-1",
|
|
)
|
|
env_to = ProvenanceEnvelope(
|
|
release_id="rel-2", release_version="v2.0.0",
|
|
release_commit_hash="b" * 40, repository_id="repo-1",
|
|
repository_key="repo-1", dashboard_id=42, environment_id="env-1",
|
|
)
|
|
snap_from = self._write_snapshot(tmp_path, env_from)
|
|
snap_to = self._write_snapshot(tmp_path, env_to)
|
|
db, _, _ = self._setup_diff_mocks()
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = str(tmp_path)
|
|
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_diff.build_snapshot_path",
|
|
side_effect=[snap_from, snap_to],
|
|
), pytest.raises(ValueError, match="Snapshot metadata verification failed"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="rel-1", release_id_to="rel-2"),
|
|
db=db, git_service=git_service,
|
|
)
|
|
|
|
def test_wrong_release_version(self, tmp_path):
|
|
"""Envelope release_version mismatch raises ValueError."""
|
|
env_from = ProvenanceEnvelope(
|
|
release_id="rel-1", release_version="v9.9.9",
|
|
release_commit_hash="a" * 40, repository_id="repo-1",
|
|
repository_key="repo-1", dashboard_id=42, environment_id="env-1",
|
|
)
|
|
env_to = ProvenanceEnvelope(
|
|
release_id="rel-2", release_version="v2.0.0",
|
|
release_commit_hash="b" * 40, repository_id="repo-1",
|
|
repository_key="repo-1", dashboard_id=42, environment_id="env-1",
|
|
)
|
|
snap_from = self._write_snapshot(tmp_path, env_from)
|
|
snap_to = self._write_snapshot(tmp_path, env_to)
|
|
db, _, _ = self._setup_diff_mocks()
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = str(tmp_path)
|
|
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_diff.build_snapshot_path",
|
|
side_effect=[snap_from, snap_to],
|
|
), pytest.raises(ValueError, match="Snapshot metadata verification failed"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="rel-1", release_id_to="rel-2"),
|
|
db=db, git_service=git_service,
|
|
)
|
|
|
|
def test_wrong_commit_hash(self, tmp_path):
|
|
"""Envelope release_commit_hash mismatch raises ValueError."""
|
|
env_from = ProvenanceEnvelope(
|
|
release_id="rel-1", release_version="v1.0.0",
|
|
release_commit_hash="c" * 40, repository_id="repo-1",
|
|
repository_key="repo-1", dashboard_id=42, environment_id="env-1",
|
|
)
|
|
env_to = ProvenanceEnvelope(
|
|
release_id="rel-2", release_version="v2.0.0",
|
|
release_commit_hash="b" * 40, repository_id="repo-1",
|
|
repository_key="repo-1", dashboard_id=42, environment_id="env-1",
|
|
)
|
|
snap_from = self._write_snapshot(tmp_path, env_from)
|
|
snap_to = self._write_snapshot(tmp_path, env_to)
|
|
db, _, _ = self._setup_diff_mocks()
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = str(tmp_path)
|
|
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_diff.build_snapshot_path",
|
|
side_effect=[snap_from, snap_to],
|
|
), pytest.raises(ValueError, match="Snapshot metadata verification failed"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="rel-1", release_id_to="rel-2"),
|
|
db=db, git_service=git_service,
|
|
)
|
|
|
|
def test_wrong_environment(self, tmp_path):
|
|
"""Envelope environment_id mismatch raises ValueError."""
|
|
env_from = ProvenanceEnvelope(
|
|
release_id="rel-1", release_version="v1.0.0",
|
|
release_commit_hash="a" * 40, repository_id="repo-1",
|
|
repository_key="repo-1", dashboard_id=42, environment_id="env-wrong",
|
|
)
|
|
env_to = ProvenanceEnvelope(
|
|
release_id="rel-2", release_version="v2.0.0",
|
|
release_commit_hash="b" * 40, repository_id="repo-1",
|
|
repository_key="repo-1", dashboard_id=42, environment_id="env-1",
|
|
)
|
|
snap_from = self._write_snapshot(tmp_path, env_from, env_id="env-wrong")
|
|
snap_to = self._write_snapshot(tmp_path, env_to)
|
|
db, _, _ = self._setup_diff_mocks()
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = str(tmp_path)
|
|
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_diff.build_snapshot_path",
|
|
side_effect=[snap_from, snap_to],
|
|
), pytest.raises(ValueError, match="Snapshot metadata verification failed"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="rel-1", release_id_to="rel-2"),
|
|
db=db, git_service=git_service,
|
|
)
|
|
|
|
def test_wrong_dashboard(self, tmp_path):
|
|
"""Envelope dashboard_id mismatch raises ValueError."""
|
|
env_from = ProvenanceEnvelope(
|
|
release_id="rel-1", release_version="v1.0.0",
|
|
release_commit_hash="a" * 40, repository_id="repo-1",
|
|
repository_key="repo-1", dashboard_id=99,
|
|
environment_id="env-1",
|
|
)
|
|
env_to = ProvenanceEnvelope(
|
|
release_id="rel-2", release_version="v2.0.0",
|
|
release_commit_hash="b" * 40, repository_id="repo-1",
|
|
repository_key="repo-1", dashboard_id=42, environment_id="env-1",
|
|
)
|
|
snap_from = self._write_snapshot(tmp_path, env_from, dash_id=99)
|
|
snap_to = self._write_snapshot(tmp_path, env_to)
|
|
db, _, _ = self._setup_diff_mocks()
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = str(tmp_path)
|
|
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_diff.build_snapshot_path",
|
|
side_effect=[snap_from, snap_to],
|
|
), pytest.raises(ValueError, match="Snapshot metadata verification failed"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="rel-1", release_id_to="rel-2"),
|
|
db=db, git_service=git_service,
|
|
)
|
|
|
|
def test_cross_repo_releases(self, tmp_path):
|
|
"""Different repo_id between two release envelopes raises ValueError."""
|
|
env_from = ProvenanceEnvelope(
|
|
release_id="rel-1", release_version="v1.0.0",
|
|
release_commit_hash="a" * 40, repository_id="repo-1",
|
|
repository_key="repo-1", dashboard_id=42, environment_id="env-1",
|
|
)
|
|
env_to = ProvenanceEnvelope(
|
|
release_id="rel-2", release_version="v2.0.0",
|
|
release_commit_hash="b" * 40, repository_id="repo-2",
|
|
repository_key="repo-2", dashboard_id=42, environment_id="env-1",
|
|
)
|
|
snap_from = self._write_snapshot(tmp_path, env_from)
|
|
snap_to = self._write_snapshot(tmp_path, env_to)
|
|
db, _, _ = self._setup_diff_mocks()
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = str(tmp_path)
|
|
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_diff.build_snapshot_path",
|
|
side_effect=[snap_from, snap_to],
|
|
), pytest.raises(ValueError, match="Snapshot metadata verification failed"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="rel-1", release_id_to="rel-2"),
|
|
db=db, git_service=git_service,
|
|
)
|
|
# #endregion
|
|
#endregion Test.BaselineEngine.StructureSnapshot.Mismatch
|