busya
a32ca0631b
- Authoritative candidate capture with server-issued artifacts and raw-byte
immutability hashing (source_response_hash server-owned)
- Closed-period lifecycle: request-hash bound approvals, persisted closure
immutability violations, byte-for-byte catalog stability on reclosure
- Verification runs: persisted VerificationRun model + FK migration,
publish gate (block_publish), scheduled observability runs (02:00 UTC)
- FR-013 baseline inheritance: prior_release_id migration, plan_inheritance/
execute_inheritance classification and re-extraction, API endpoints
- Visual executor bound to release-deployment environment; caller mismatch
rejected; visual SSIM/reconciliation modules
- Query execution decomposed: envelope/model/executor split, no direct SQL
- AgentRun approvals extracted to submodule; evidence adapter; _utils
- Dashboard testing service decomposed into 30+ modules (all <400 LOC)
- Five Feature-037 agent tools with permission guards (tools_037.py)
- API readiness endpoint; Alembic env/migrations; test fixture repos
- Specs 036/037 contracts, openapi.yaml, schema.json, tasks/traceability
updated; semantic index rebuilt with 0 parse warnings
- Fix ADR-0003 parser ambiguity: remove [DEF🆔ADR] prose example
- Add axiom-mcp-agent-feedback.md: agent findings for MCP rework plan
- Tests: 298 service + 1464 API + 45 agent passing; ruff clean
455 lines
18 KiB
Python
455 lines
18 KiB
Python
#region Test.BaselineEngine.StructureSnapshot.Diff [C:4] [TYPE Module] [SEMANTICS test,structure-snapshot,diff,metadata-verification]
|
|
# @defgroup Test.BaselineEngine.StructureSnapshot.Diff Diff-level tests for release-bound snapshot comparison.
|
|
# @LAYER Test
|
|
# @RELATION BINDS_TO -> [BaselineEngine.StructureSnapshot.Diff]
|
|
# @RELATION VERIFIES -> [BaselineEngine.StructureSnapshot.Diff]
|
|
# @TEST_EDGE wrong_dashboard_snapshot -> raises ValueError when snapshot metadata mismatches release request
|
|
# @TEST_EDGE omitted_semantic_dimensions -> diff covers dataset identity, chart dataset,
|
|
# filter dataset, type, column attributes, metric definition, access capability
|
|
# @TEST_EDGE missing_snapshot_file -> FileNotFoundError propagated
|
|
# @TEST_EDGE metadata_verification -> cross-verification of all 5 identity dimensions
|
|
# @INVARIANT Snapshot metadata is cross-verified before diff computation.
|
|
# @INVARIANT Mismatch raises ValueError, never produces a synthetic diff.
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
import pytest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from src.models.dashboard_release import DashboardRelease
|
|
from src.models.deployment import DeploymentRecord
|
|
from src.models.git import GitRepository
|
|
from src.schemas.dashboard_testing import DashboardQueryModel
|
|
from src.schemas.dashboard_testing.structure_snapshot import (
|
|
ProvenanceEnvelope,
|
|
SnapshotDiffRequest,
|
|
SnapshotMetadataVerification,
|
|
)
|
|
from src.services.dashboard_testing.structure_snapshot_diff import diff_release_snapshots
|
|
|
|
|
|
# #region Test.StructureSnapshot.Diff [C:3] [TYPE Class]
|
|
class TestDiffReleaseSnapshots:
|
|
"""Tests for diff_release_snapshots()."""
|
|
|
|
def _make_envelope(self, **overrides) -> ProvenanceEnvelope:
|
|
"""Create a standard ProvenanceEnvelope with overridable defaults."""
|
|
defaults = {
|
|
"release_id": "rel-1",
|
|
"release_version": "v1.0.0",
|
|
"release_commit_hash": "a" * 40,
|
|
"repository_id": "repo-1",
|
|
"repository_key": "test-project",
|
|
"dashboard_id": 42,
|
|
"environment_id": "env-1",
|
|
}
|
|
defaults.update(overrides)
|
|
return ProvenanceEnvelope(**defaults)
|
|
|
|
def _write_provenance_snapshot(
|
|
self, tmp_path: Path, release_version: str, envelope: ProvenanceEnvelope,
|
|
dashboard_id: int = 42, environment_id: str = "env-1",
|
|
) -> Path:
|
|
"""Write a provenance-wrapped snapshot file and return its path."""
|
|
snap_dir = tmp_path / "snapshots"
|
|
snap_dir.mkdir(parents=True, exist_ok=True)
|
|
snap_path = snap_dir / f"{release_version}.json"
|
|
model = DashboardQueryModel(
|
|
environment_id=environment_id, dashboard_id=dashboard_id,
|
|
title="Test", query_model_fingerprint="sha256:fp",
|
|
)
|
|
payload = {
|
|
"provenance": envelope.model_dump(mode="json"),
|
|
"query_model": model.model_dump(mode="json"),
|
|
}
|
|
snap_path.write_text(json.dumps(payload))
|
|
return snap_path
|
|
|
|
# #region Test.StructureSnapshot.Diff.TestReleaseNotFound
|
|
def test_release_not_found_from(self):
|
|
"""Missing base release raises ValueError."""
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.return_value = None
|
|
|
|
with pytest.raises(ValueError, match="not found"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="nonexistent", release_id_to="rel-2"),
|
|
db=db,
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Diff.TestReleaseNotFoundTo
|
|
def test_release_not_found_to(self):
|
|
"""Missing target release raises ValueError."""
|
|
release_from = MagicMock(spec=DashboardRelease)
|
|
release_from.id = "rel-1"
|
|
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.side_effect = [
|
|
release_from, None
|
|
]
|
|
|
|
with pytest.raises(ValueError, match="not found"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="rel-1", release_id_to="nonexistent"),
|
|
db=db,
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Diff.TestRepoNotFound
|
|
def test_repository_not_found_for_diff(self):
|
|
"""Missing GitRepository for release raises ValueError."""
|
|
release_from = MagicMock(spec=DashboardRelease)
|
|
release_from.id = "rel-1"
|
|
release_from.version = "v1.0.0"
|
|
release_from.commit_hash = "a" * 40
|
|
release_from.repository_id = "missing-repo"
|
|
release_from.deployment_id = "dep-1"
|
|
|
|
release_to = MagicMock(spec=DashboardRelease)
|
|
release_to.id = "rel-2"
|
|
release_to.version = "v2.0.0"
|
|
release_to.commit_hash = "b" * 40
|
|
release_to.repository_id = "missing-repo"
|
|
release_to.deployment_id = "dep-2"
|
|
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.side_effect = [
|
|
release_from, release_to, None # Third call for GitRepository returns None
|
|
]
|
|
|
|
with pytest.raises(ValueError, match="not found"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="rel-1", release_id_to="rel-2"),
|
|
db=db,
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Diff.TestMetadataMismatchEnv
|
|
def test_metadata_mismatch_environment(self, tmp_path):
|
|
"""Snapshot environment_id mismatch raises ValueError."""
|
|
release_from = MagicMock(spec=DashboardRelease)
|
|
release_from.id = "rel-1"
|
|
release_from.version = "v1.0.0"
|
|
release_from.commit_hash = "a" * 40
|
|
release_from.repository_id = "repo-1"
|
|
release_from.deployment_id = "dep-1"
|
|
|
|
release_to = MagicMock(spec=DashboardRelease)
|
|
release_to.id = "rel-2"
|
|
release_to.version = "v2.0.0"
|
|
release_to.commit_hash = "b" * 40
|
|
release_to.repository_id = "repo-1"
|
|
release_to.deployment_id = "dep-2"
|
|
|
|
repo_record = MagicMock(spec=GitRepository)
|
|
repo_record.id = "repo-1"
|
|
repo_record.dashboard_id = 42
|
|
repo_record.local_path = "git_repos/test-project"
|
|
|
|
dep_from = MagicMock(spec=DeploymentRecord)
|
|
dep_from.id = "dep-1"
|
|
dep_from.environment_id = "env-1"
|
|
dep_from.commit_hash = "a" * 40
|
|
|
|
dep_to = MagicMock(spec=DeploymentRecord)
|
|
dep_to.id = "dep-2"
|
|
dep_to.environment_id = "env-2" # Different from from!
|
|
dep_to.commit_hash = "b" * 40
|
|
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.side_effect = [
|
|
release_from, release_to, repo_record, dep_from, dep_to,
|
|
]
|
|
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = str(tmp_path / "git_repos")
|
|
|
|
# Write provenance snapshots where envelope env matches model env but not dep env
|
|
env_from = self._make_envelope(environment_id="env-1")
|
|
env_to = self._make_envelope(
|
|
release_id="rel-2", release_version="v2.0.0",
|
|
release_commit_hash="b" * 40, environment_id="env-1", # envelope says env-1
|
|
)
|
|
snap_from = self._write_provenance_snapshot(tmp_path, "v1.0.0", env_from)
|
|
snap_to = self._write_provenance_snapshot(tmp_path, "v2.0.0", env_to)
|
|
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_diff.build_snapshot_path",
|
|
side_effect=[snap_from, snap_to],
|
|
), pytest.raises(ValueError, match="Snapshot metadata verification failed"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="rel-1", release_id_to="rel-2"),
|
|
db=db, git_service=git_service,
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Diff.TestMetadataMismatchDashboard
|
|
def test_metadata_mismatch_dashboard_id(self, tmp_path):
|
|
"""Snapshot dashboard_id mismatch raises ValueError."""
|
|
release_from = MagicMock(spec=DashboardRelease)
|
|
release_from.id = "rel-1"
|
|
release_from.version = "v1.0.0"
|
|
release_from.commit_hash = "a" * 40
|
|
release_from.repository_id = "repo-1"
|
|
release_from.deployment_id = "dep-1"
|
|
|
|
release_to = MagicMock(spec=DashboardRelease)
|
|
release_to.id = "rel-2"
|
|
release_to.version = "v2.0.0"
|
|
release_to.commit_hash = "b" * 40
|
|
release_to.repository_id = "repo-1"
|
|
release_to.deployment_id = "dep-2"
|
|
|
|
repo_record = MagicMock(spec=GitRepository)
|
|
repo_record.id = "repo-1"
|
|
repo_record.dashboard_id = 42
|
|
repo_record.local_path = "git_repos/test-project"
|
|
|
|
dep_from = MagicMock(spec=DeploymentRecord)
|
|
dep_from.id = "dep-1"
|
|
dep_from.environment_id = "env-1"
|
|
dep_from.commit_hash = "a" * 40
|
|
|
|
dep_to = MagicMock(spec=DeploymentRecord)
|
|
dep_to.id = "dep-2"
|
|
dep_to.environment_id = "env-1"
|
|
dep_to.commit_hash = "b" * 40
|
|
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.side_effect = [
|
|
release_from, release_to, repo_record, dep_from, dep_to,
|
|
]
|
|
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = str(tmp_path / "git_repos")
|
|
|
|
# Envelope and model both say different dashboard_id
|
|
env_from = self._make_envelope(dashboard_id=42)
|
|
env_to = self._make_envelope(
|
|
release_id="rel-2", release_version="v2.0.0",
|
|
release_commit_hash="b" * 40, dashboard_id=99, # dash mismatch
|
|
)
|
|
snap_from = self._write_provenance_snapshot(tmp_path, "v1.0.0", env_from, dashboard_id=42)
|
|
snap_to = self._write_provenance_snapshot(tmp_path, "v2.0.0", env_to, dashboard_id=99)
|
|
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_diff.build_snapshot_path",
|
|
side_effect=[snap_from, snap_to],
|
|
), pytest.raises(ValueError, match="Snapshot metadata verification failed"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="rel-1", release_id_to="rel-2"),
|
|
db=db, git_service=git_service,
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Diff.TestCommitMismatch
|
|
def test_commit_hash_mismatch(self, tmp_path):
|
|
"""Deployment commit_hash mismatch with release raises ValueError."""
|
|
release_from = MagicMock(spec=DashboardRelease)
|
|
release_from.id = "rel-1"
|
|
release_from.version = "v1.0.0"
|
|
release_from.commit_hash = "a" * 40
|
|
release_from.repository_id = "repo-1"
|
|
release_from.deployment_id = "dep-1"
|
|
|
|
release_to = MagicMock(spec=DashboardRelease)
|
|
release_to.id = "rel-2"
|
|
release_to.version = "v2.0.0"
|
|
release_to.commit_hash = "b" * 40
|
|
release_to.repository_id = "repo-1"
|
|
release_to.deployment_id = "dep-2"
|
|
|
|
repo_record = MagicMock(spec=GitRepository)
|
|
repo_record.id = "repo-1"
|
|
repo_record.dashboard_id = 42
|
|
repo_record.local_path = "git_repos/test-project"
|
|
|
|
dep_from = MagicMock(spec=DeploymentRecord)
|
|
dep_from.id = "dep-1"
|
|
dep_from.environment_id = "env-1"
|
|
dep_from.commit_hash = "a" * 40
|
|
|
|
dep_to = MagicMock(spec=DeploymentRecord)
|
|
dep_to.id = "dep-2"
|
|
dep_to.environment_id = "env-1"
|
|
dep_to.commit_hash = "c" * 40 # doesn't match release commit b*40
|
|
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.side_effect = [
|
|
release_from, release_to, repo_record, dep_from, dep_to,
|
|
]
|
|
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = str(tmp_path / "git_repos")
|
|
|
|
env_from = self._make_envelope(release_commit_hash="a" * 40)
|
|
env_to = self._make_envelope(
|
|
release_id="rel-2", release_version="v2.0.0",
|
|
release_commit_hash="b" * 40,
|
|
)
|
|
snap_from = self._write_provenance_snapshot(tmp_path, "v1.0.0", env_from)
|
|
snap_to = self._write_provenance_snapshot(tmp_path, "v2.0.0", env_to)
|
|
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_diff.build_snapshot_path",
|
|
side_effect=[snap_from, snap_to],
|
|
), pytest.raises(ValueError, match="Snapshot metadata verification failed"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="rel-1", release_id_to="rel-2"),
|
|
db=db, git_service=git_service,
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Diff.TestLegacySnapshotRejected
|
|
def test_legacy_snapshot_without_provenance_rejected(self, tmp_path):
|
|
"""Legacy snapshot without provenance raises ValueError."""
|
|
release_from = MagicMock(spec=DashboardRelease)
|
|
release_from.id = "rel-1"
|
|
release_from.version = "v1.0.0"
|
|
release_from.commit_hash = "a" * 40
|
|
release_from.repository_id = "repo-1"
|
|
release_from.deployment_id = "dep-1"
|
|
|
|
release_to = MagicMock(spec=DashboardRelease)
|
|
release_to.id = "rel-2"
|
|
release_to.version = "v2.0.0"
|
|
release_to.commit_hash = "b" * 40
|
|
release_to.repository_id = "repo-1"
|
|
release_to.deployment_id = "dep-2"
|
|
|
|
repo_record = MagicMock(spec=GitRepository)
|
|
repo_record.id = "repo-1"
|
|
repo_record.dashboard_id = 42
|
|
repo_record.local_path = "git_repos/test-project"
|
|
|
|
dep_from = MagicMock(spec=DeploymentRecord)
|
|
dep_from.id = "dep-1"
|
|
dep_from.environment_id = "env-1"
|
|
dep_from.commit_hash = "a" * 40
|
|
|
|
dep_to = MagicMock(spec=DeploymentRecord)
|
|
dep_to.id = "dep-2"
|
|
dep_to.environment_id = "env-1"
|
|
dep_to.commit_hash = "b" * 40
|
|
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.side_effect = [
|
|
release_from, release_to, repo_record, dep_from, dep_to,
|
|
]
|
|
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = str(tmp_path / "git_repos")
|
|
|
|
# Write a LEGACY snapshot (bare model, no provenance)
|
|
snap_dir = tmp_path / "snapshots"
|
|
snap_dir.mkdir(parents=True, exist_ok=True)
|
|
snap_path = snap_dir / "v1.0.0.json"
|
|
model = DashboardQueryModel(
|
|
environment_id="env-1", dashboard_id=42, title="Legacy",
|
|
query_model_fingerprint="sha256:legacy",
|
|
)
|
|
snap_path.write_text(json.dumps(model.model_dump(mode="json")))
|
|
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_diff.build_snapshot_path",
|
|
return_value=snap_path,
|
|
), pytest.raises(ValueError, match="provenance"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="rel-1", release_id_to="rel-2"),
|
|
db=db, git_service=git_service,
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Diff.TestRepositoryMismatch
|
|
def test_repository_mismatch(self, tmp_path):
|
|
"""Cross-repo releases raise ValueError (same-repo contract)."""
|
|
release_from = MagicMock(spec=DashboardRelease)
|
|
release_from.id = "rel-1"
|
|
release_from.version = "v1.0.0"
|
|
release_from.commit_hash = "a" * 40
|
|
release_from.repository_id = "repo-1"
|
|
release_from.deployment_id = "dep-1"
|
|
|
|
release_to = MagicMock(spec=DashboardRelease)
|
|
release_to.id = "rel-2"
|
|
release_to.version = "v2.0.0"
|
|
release_to.commit_hash = "b" * 40
|
|
release_to.repository_id = "repo-2" # different repo!
|
|
release_to.deployment_id = "dep-2"
|
|
|
|
repo_record = MagicMock(spec=GitRepository)
|
|
repo_record.id = "repo-1"
|
|
repo_record.dashboard_id = 42
|
|
repo_record.local_path = "git_repos/test-project"
|
|
|
|
dep_from = MagicMock(spec=DeploymentRecord)
|
|
dep_from.id = "dep-1"
|
|
dep_from.environment_id = "env-1"
|
|
dep_from.commit_hash = "a" * 40
|
|
|
|
dep_to = MagicMock(spec=DeploymentRecord)
|
|
dep_to.id = "dep-2"
|
|
dep_to.environment_id = "env-1"
|
|
dep_to.commit_hash = "b" * 40
|
|
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.side_effect = [
|
|
release_from, release_to, repo_record, dep_from, dep_to,
|
|
]
|
|
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = str(tmp_path / "git_repos")
|
|
|
|
# from has repo-1, to has repo-2 in envelope (cross-repo)
|
|
env_from = self._make_envelope(repository_id="repo-1")
|
|
env_to = self._make_envelope(
|
|
release_id="rel-2", release_version="v2.0.0",
|
|
release_commit_hash="b" * 40, repository_id="repo-2",
|
|
)
|
|
snap_from = self._write_provenance_snapshot(tmp_path, "v1.0.0", env_from)
|
|
snap_to = self._write_provenance_snapshot(tmp_path, "v2.0.0", env_to)
|
|
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_diff.build_snapshot_path",
|
|
side_effect=[snap_from, snap_to],
|
|
), pytest.raises(ValueError, match="Snapshot metadata verification failed"):
|
|
diff_release_snapshots(
|
|
SnapshotDiffRequest(release_id_from="rel-1", release_id_to="rel-2"),
|
|
db=db, git_service=git_service,
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Diff.TestMetadataVerificationSummary
|
|
def test_metadata_verification_summary(self):
|
|
"""SnapshotMetadataVerification summary correctly reports individual match status."""
|
|
# Test all_match is True when all dimensions match
|
|
v = SnapshotMetadataVerification(
|
|
environment_match=True,
|
|
dashboard_match=True,
|
|
repository_match=True,
|
|
release_match=True,
|
|
commit_match=True,
|
|
all_match=True,
|
|
)
|
|
assert v.all_match is True
|
|
|
|
# Test all_match is False when any dimension mismatches
|
|
v2 = SnapshotMetadataVerification(
|
|
environment_match=True,
|
|
dashboard_match=False,
|
|
repository_match=True,
|
|
release_match=True,
|
|
commit_match=True,
|
|
all_match=False,
|
|
details="Dashboard mismatch",
|
|
)
|
|
assert v2.all_match is False
|
|
assert v2.dashboard_match is False
|
|
# #endregion
|
|
# #endregion
|
|
|
|
|
|
#endregion Test.BaselineEngine.StructureSnapshot.Diff
|