busya
a32ca0631b
- Authoritative candidate capture with server-issued artifacts and raw-byte
immutability hashing (source_response_hash server-owned)
- Closed-period lifecycle: request-hash bound approvals, persisted closure
immutability violations, byte-for-byte catalog stability on reclosure
- Verification runs: persisted VerificationRun model + FK migration,
publish gate (block_publish), scheduled observability runs (02:00 UTC)
- FR-013 baseline inheritance: prior_release_id migration, plan_inheritance/
execute_inheritance classification and re-extraction, API endpoints
- Visual executor bound to release-deployment environment; caller mismatch
rejected; visual SSIM/reconciliation modules
- Query execution decomposed: envelope/model/executor split, no direct SQL
- AgentRun approvals extracted to submodule; evidence adapter; _utils
- Dashboard testing service decomposed into 30+ modules (all <400 LOC)
- Five Feature-037 agent tools with permission guards (tools_037.py)
- API readiness endpoint; Alembic env/migrations; test fixture repos
- Specs 036/037 contracts, openapi.yaml, schema.json, tasks/traceability
updated; semantic index rebuilt with 0 parse warnings
- Fix ADR-0003 parser ambiguity: remove [DEF🆔ADR] prose example
- Add axiom-mcp-agent-feedback.md: agent findings for MCP rework plan
- Tests: 298 service + 1464 API + 45 agent passing; ruff clean
497 lines
20 KiB
Python
497 lines
20 KiB
Python
#region Test.BaselineEngine.StructureSnapshot.Service [C:5] [TYPE Module] [SEMANTICS test,structure-snapshot,service,capture,diff,release-bound]
|
|
# @defgroup Test.BaselineEngine.StructureSnapshot Service-level tests for release-bound snapshot capture + diff.
|
|
# @LAYER Test
|
|
# @RELATION BINDS_TO -> [BaselineEngine.StructureSnapshot.Service]
|
|
# @RELATION VERIFIES -> [BaselineEngine.StructureSnapshot.Capture]
|
|
# @RELATION VERIFIES -> [BaselineEngine.StructureSnapshot.Diff]
|
|
# @TEST_EDGE capture_persist_diff -> full capture → persist → diff flow
|
|
# @TEST_EDGE failed_inspection_no_write -> sentinel error / blocking warnings prevent persistence
|
|
# @TEST_EDGE unknown_repo -> unknown / unauthorized repository raises ValueError
|
|
# @TEST_EDGE wrong_dashboard -> snapshot metadata mismatch on diff raises ValueError
|
|
# @TEST_EDGE semver_validation -> invalid semver raises ValueError
|
|
# @TEST_EDGE commit_validation -> invalid commit hash raises ValueError
|
|
# @TEST_EDGE omitted_semantic_dimensions -> diff covers dataset identity, chart dataset,
|
|
# filter dataset, type, column attributes, metric definition, access capability
|
|
# @INVARIANT Every capture is bound to a real DashboardRelease record.
|
|
# @INVARIANT Never persist if inspection returns sentinel errors / blocking warnings.
|
|
# @INVARIANT Path resolution uses GitService base_path, never CWD.
|
|
# @INVARIANT Diff cross-verifies loaded snapshot metadata against release records.
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
from src.models.dashboard_release import DashboardRelease
|
|
from src.models.git import GitRepository
|
|
from src.schemas.dashboard_testing import DashboardQueryModel, SnapshotCaptureResponse, Warning as WarningSchema
|
|
from src.schemas.dashboard_testing.structure_snapshot import (
|
|
SENTINEL_ERROR_FINGERPRINT,
|
|
SnapshotCaptureRequest,
|
|
SnapshotDiffRequest,
|
|
)
|
|
from src.services.dashboard_testing.structure_snapshot_capture import (
|
|
capture_release_snapshot,
|
|
derive_dash_key,
|
|
derive_repo_key,
|
|
has_blocking_warnings,
|
|
validate_commit_hash,
|
|
validate_semver,
|
|
)
|
|
|
|
|
|
# #region Test.StructureSnapshot.Validation [C:2] [TYPE Class]
|
|
class TestValidation:
|
|
"""Unit tests for helper validators."""
|
|
|
|
# #region Test.StructureSnapshot.Validation.TestSemverValid
|
|
def test_semver_valid(self):
|
|
assert validate_semver("v1.0.0") == "v1.0.0"
|
|
assert validate_semver("v2.3.4-rc1") == "v2.3.4-rc1"
|
|
assert validate_semver("v10.20.30") == "v10.20.30"
|
|
assert validate_semver("v0.0.1+build123") == "v0.0.1+build123"
|
|
assert validate_semver("v1.2.3-alpha.1") == "v1.2.3-alpha.1"
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Validation.TestSemverInvalid
|
|
def test_semver_invalid(self):
|
|
with pytest.raises(ValueError, match="v-prefixed SemVer"):
|
|
validate_semver("1.0.0") # missing v prefix
|
|
with pytest.raises(ValueError, match="v-prefixed SemVer"):
|
|
validate_semver("v1.0") # incomplete
|
|
with pytest.raises(ValueError, match="v-prefixed SemVer"):
|
|
validate_semver("") # empty
|
|
with pytest.raises(ValueError, match="v-prefixed SemVer"):
|
|
validate_semver("v1.0.0.0") # too many parts
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Validation.TestCommitValid
|
|
def test_commit_hash_valid(self):
|
|
assert validate_commit_hash("a" * 40) == "a" * 40
|
|
assert validate_commit_hash("abcdef0123456789abcdef0123456789abcdef01") == "abcdef0123456789abcdef0123456789abcdef01"
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Validation.TestCommitInvalid
|
|
def test_commit_hash_invalid(self):
|
|
with pytest.raises(ValueError, match="40-char git SHA"):
|
|
validate_commit_hash("") # empty
|
|
with pytest.raises(ValueError, match="40-char git SHA"):
|
|
validate_commit_hash("xyz") # invalid char
|
|
with pytest.raises(ValueError, match="40-char git SHA"):
|
|
validate_commit_hash("abc123") # too short (6)
|
|
with pytest.raises(ValueError, match="40-char git SHA"):
|
|
validate_commit_hash("z" * 40) # invalid hex char
|
|
with pytest.raises(ValueError, match="40-char git SHA"):
|
|
validate_commit_hash("abc1234") # 7-char, rejected
|
|
with pytest.raises(ValueError, match="40-char git SHA"):
|
|
validate_commit_hash("a" * 39) # 39 chars, rejected
|
|
with pytest.raises(ValueError, match="uppercase"):
|
|
validate_commit_hash("ABCDEF0123456789abcdef0123456789abcdef01") # uppercase rejected
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Validation.TestBlockingWarnings
|
|
def testhas_blocking_warnings_sentinel_fingerprint(self):
|
|
model = MagicMock(spec=DashboardQueryModel)
|
|
model.query_model_fingerprint = SENTINEL_ERROR_FINGERPRINT
|
|
model.warnings = []
|
|
assert has_blocking_warnings(model) is True
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Validation.TestBlockingWarningsFetchFailed
|
|
def testhas_blocking_warnings_fetch_failed(self):
|
|
model = MagicMock(spec=DashboardQueryModel)
|
|
model.query_model_fingerprint = "sha256:normal_hash"
|
|
model.warnings = [WarningSchema(source="inspection", resource="42",
|
|
code="DASHBOARD_FETCH_FAILED", detail="timeout")]
|
|
assert has_blocking_warnings(model) is True
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Validation.TestBlockingWarningsInaccessibleChart
|
|
def testhas_blocking_warnings_inaccessible_chart_not_blocking(self):
|
|
"""INACCESSIBLE_CHART is not a blocking code — charts can be individually broken."""
|
|
model = MagicMock(spec=DashboardQueryModel)
|
|
model.query_model_fingerprint = "sha256:normal"
|
|
model.warnings = [WarningSchema(source="inspection", resource="129",
|
|
code="INACCESSIBLE_CHART", detail="timeout")]
|
|
assert has_blocking_warnings(model) is False
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Validation.TestBlockingWarningsNone
|
|
def testhas_blocking_warnings_no_warnings(self):
|
|
model = MagicMock(spec=DashboardQueryModel)
|
|
model.query_model_fingerprint = "sha256:ok"
|
|
model.warnings = []
|
|
assert has_blocking_warnings(model) is False
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Validation.TestDeriveRepoKey
|
|
def testderive_repo_key(self):
|
|
repo = MagicMock(spec=GitRepository)
|
|
repo.local_path = "git_repos/my-project"
|
|
repo.id = "uuid-123"
|
|
repo.dashboard_id = 42
|
|
key = derive_repo_key(repo)
|
|
assert key == "my-project" # basename of local_path
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Validation.TestDeriveDashKey
|
|
def testderive_dash_key(self):
|
|
repo = MagicMock(spec=GitRepository)
|
|
repo.dashboard_id = 99
|
|
key = derive_dash_key(repo)
|
|
assert key == "dash_99"
|
|
# #endregion
|
|
# #endregion
|
|
|
|
|
|
# #region Test.StructureSnapshot.SemverCommitValidation [C:2] [TYPE Class]
|
|
class TestSchemaValidators:
|
|
"""Test that request schemas enforce their validation rules."""
|
|
|
|
# #region Test.StructureSnapshot.SemverCommitValidation.TestCaptureRequest
|
|
def test_capture_request_empty_release_id(self):
|
|
with pytest.raises(ValueError, match="non-empty"):
|
|
SnapshotCaptureRequest(release_id="")
|
|
|
|
def test_capture_request_valid(self):
|
|
req = SnapshotCaptureRequest(release_id="abc-123")
|
|
assert req.release_id == "abc-123"
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.SemverCommitValidation.TestDiffRequest
|
|
def test_diff_request_same_ids(self):
|
|
with pytest.raises(ValueError, match="must differ"):
|
|
SnapshotDiffRequest(release_id_from="same", release_id_to="same")
|
|
|
|
def test_diff_request_empty(self):
|
|
with pytest.raises(ValueError, match="non-empty"):
|
|
SnapshotDiffRequest(release_id_from="", release_id_to="other")
|
|
|
|
def test_diff_request_valid(self):
|
|
req = SnapshotDiffRequest(release_id_from="id-1", release_id_to="id-2")
|
|
assert req.release_id_from == "id-1"
|
|
assert req.release_id_to == "id-2"
|
|
# #endregion
|
|
# #endregion
|
|
|
|
|
|
# #region Test.StructureSnapshot.Capture [C:3] [TYPE Class]
|
|
class TestCaptureReleaseSnapshot:
|
|
"""Tests for capture_release_snapshot()."""
|
|
|
|
# #region Test.StructureSnapshot.Capture.TestEnvironmentIdEmpty
|
|
@pytest.mark.asyncio
|
|
async def test_environment_id_empty_rejected(self):
|
|
"""Empty environment_id raises ValueError."""
|
|
db = MagicMock()
|
|
|
|
with pytest.raises(ValueError, match="environment_id is required"):
|
|
await capture_release_snapshot(
|
|
SnapshotCaptureRequest(release_id="rel-1"),
|
|
db=db, client=MagicMock(),
|
|
environment_id="",
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_environment_id_unknown_rejected(self):
|
|
"""'unknown' environment_id raises ValueError."""
|
|
db = MagicMock()
|
|
|
|
with pytest.raises(ValueError, match="environment_id is required"):
|
|
await capture_release_snapshot(
|
|
SnapshotCaptureRequest(release_id="rel-1"),
|
|
db=db, client=MagicMock(),
|
|
environment_id="unknown",
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_environment_id_none_rejected(self):
|
|
"""None environment_id raises ValueError."""
|
|
db = MagicMock()
|
|
|
|
with pytest.raises(ValueError, match="environment_id is required"):
|
|
await capture_release_snapshot(
|
|
SnapshotCaptureRequest(release_id="rel-1"),
|
|
db=db, client=MagicMock(),
|
|
environment_id=None, # type: ignore[arg-type]
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Capture.TestReleaseNotFound
|
|
@pytest.mark.asyncio
|
|
async def test_release_not_found(self):
|
|
"""Unknown release_id raises ValueError."""
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.return_value = None
|
|
|
|
with pytest.raises(ValueError, match="not found"):
|
|
await capture_release_snapshot(
|
|
SnapshotCaptureRequest(release_id="nonexistent"),
|
|
db=db, client=MagicMock(),
|
|
environment_id="env-1",
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Capture.TestSemverInvalid
|
|
@pytest.mark.asyncio
|
|
async def test_release_invalid_semver(self):
|
|
"""Release with non-v-prefixed version raises ValueError."""
|
|
release = MagicMock(spec=DashboardRelease)
|
|
release.id = "rel-1"
|
|
release.version = "1.0.0" # missing v prefix
|
|
release.commit_hash = "a" * 40
|
|
release.repository_id = "repo-1"
|
|
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.return_value = release
|
|
|
|
with pytest.raises(ValueError, match="v-prefixed SemVer"):
|
|
await capture_release_snapshot(
|
|
SnapshotCaptureRequest(release_id="rel-1"),
|
|
db=db, client=MagicMock(),
|
|
environment_id="env-1",
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Capture.TestCommitInvalid
|
|
@pytest.mark.asyncio
|
|
async def test_release_invalid_commit(self):
|
|
"""Release with non-canonical commit hash raises ValueError."""
|
|
release = MagicMock(spec=DashboardRelease)
|
|
release.id = "rel-1"
|
|
release.version = "v1.0.0"
|
|
release.commit_hash = "xyz123" # invalid hex
|
|
release.repository_id = "repo-1"
|
|
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.return_value = release
|
|
|
|
with pytest.raises(ValueError, match="40-char git SHA"):
|
|
await capture_release_snapshot(
|
|
SnapshotCaptureRequest(release_id="rel-1"),
|
|
db=db, client=MagicMock(),
|
|
environment_id="env-1",
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Capture.TestRepoNotFound
|
|
@pytest.mark.asyncio
|
|
async def test_repository_not_found(self):
|
|
"""Release's GitRepository not found raises ValueError."""
|
|
release = MagicMock(spec=DashboardRelease)
|
|
release.id = "rel-1"
|
|
release.version = "v1.0.0"
|
|
release.commit_hash = "a" * 40
|
|
release.repository_id = "repo-1"
|
|
|
|
db = MagicMock()
|
|
# First call returns release, second call (for repository) returns None
|
|
db.query.return_value.filter.side_effect = [
|
|
MagicMock(first=MagicMock(return_value=release)),
|
|
MagicMock(first=MagicMock(return_value=None)),
|
|
]
|
|
|
|
# Override the mock properly
|
|
db.query.side_effect = None
|
|
db.query.return_value.filter.return_value.first.side_effect = [
|
|
release, None
|
|
]
|
|
|
|
with pytest.raises(ValueError, match="not found"):
|
|
await capture_release_snapshot(
|
|
SnapshotCaptureRequest(release_id="rel-1"),
|
|
db=db, client=MagicMock(),
|
|
environment_id="env-1",
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Capture.TestGitServiceUnavailable
|
|
@pytest.mark.asyncio
|
|
async def test_git_service_cannot_access_repo(self):
|
|
"""GitService cannot access the repository raises ValueError."""
|
|
release = MagicMock(spec=DashboardRelease)
|
|
release.id = "rel-1"
|
|
release.version = "v1.0.0"
|
|
release.commit_hash = "a" * 40
|
|
release.repository_id = "repo-1"
|
|
release.deployment = MagicMock()
|
|
release.deployment.environment_id = "env-1"
|
|
|
|
repo_record = MagicMock(spec=GitRepository)
|
|
repo_record.id = "repo-1"
|
|
repo_record.dashboard_id = 42
|
|
repo_record.local_path = "git_repos/my-project"
|
|
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.side_effect = [
|
|
release, repo_record
|
|
]
|
|
|
|
git_service = MagicMock()
|
|
git_service.get_repo.side_effect = Exception("Not cloned")
|
|
|
|
with pytest.raises(ValueError, match="not accessible"):
|
|
await capture_release_snapshot(
|
|
SnapshotCaptureRequest(release_id="rel-1"),
|
|
db=db, client=MagicMock(),
|
|
environment_id="env-1",
|
|
git_service=git_service,
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Capture.TestBlockingWarningsPreventPersist
|
|
@pytest.mark.asyncio
|
|
async def test_blocking_warnings_prevent_persistence(self):
|
|
"""Inspection with DASHBOARD_FETCH_FAILED warning does NOT persist."""
|
|
release = MagicMock(spec=DashboardRelease)
|
|
release.id = "rel-1"
|
|
release.version = "v1.0.0"
|
|
release.commit_hash = "a" * 40
|
|
release.repository_id = "repo-1"
|
|
release.deployment = MagicMock()
|
|
release.deployment.environment_id = "env-1"
|
|
|
|
repo_record = MagicMock(spec=GitRepository)
|
|
repo_record.id = "repo-1"
|
|
repo_record.dashboard_id = 42
|
|
repo_record.local_path = "git_repos/my-project"
|
|
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.side_effect = [
|
|
release, repo_record
|
|
]
|
|
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = "/tmp/git_repos"
|
|
git_service.get_repo.return_value = MagicMock()
|
|
|
|
client = AsyncMock()
|
|
|
|
# Mock inspect_dashboard_query_model to return a model with sentinel error
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_capture.inspect_dashboard_query_model",
|
|
new_callable=AsyncMock,
|
|
) as mock_inspect:
|
|
model = MagicMock(spec=DashboardQueryModel)
|
|
model.query_model_fingerprint = "sha256:normal"
|
|
model.warnings = [
|
|
WarningSchema(source="inspection", resource="42",
|
|
code="DASHBOARD_FETCH_FAILED", detail="API error")
|
|
]
|
|
mock_inspect.return_value = model
|
|
|
|
with pytest.raises(ValueError, match="blocking warnings"):
|
|
await capture_release_snapshot(
|
|
SnapshotCaptureRequest(release_id="rel-1"),
|
|
db=db, client=client,
|
|
environment_id="env-1",
|
|
git_service=git_service,
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Capture.TestSentinelFingerprintPreventsPersist
|
|
@pytest.mark.asyncio
|
|
async def test_sentinel_fingerprint_prevents_persistence(self):
|
|
"""Sentinel error fingerprint does NOT persist."""
|
|
release = MagicMock(spec=DashboardRelease)
|
|
release.id = "rel-2"
|
|
release.version = "v2.0.0"
|
|
release.commit_hash = "b" * 40
|
|
release.repository_id = "repo-2"
|
|
release.deployment = MagicMock()
|
|
release.deployment.environment_id = "env-1"
|
|
|
|
repo_record = MagicMock(spec=GitRepository)
|
|
repo_record.id = "repo-2"
|
|
repo_record.dashboard_id = 43
|
|
repo_record.local_path = "git_repos/other-project"
|
|
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.side_effect = [
|
|
release, repo_record
|
|
]
|
|
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = "/tmp/git_repos"
|
|
git_service.get_repo.return_value = MagicMock()
|
|
|
|
client = AsyncMock()
|
|
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_capture.inspect_dashboard_query_model",
|
|
new_callable=AsyncMock,
|
|
) as mock_inspect:
|
|
model = MagicMock(spec=DashboardQueryModel)
|
|
model.query_model_fingerprint = SENTINEL_ERROR_FINGERPRINT
|
|
model.warnings = []
|
|
mock_inspect.return_value = model
|
|
|
|
with pytest.raises(ValueError, match="blocking warnings"):
|
|
await capture_release_snapshot(
|
|
SnapshotCaptureRequest(release_id="rel-2"),
|
|
db=db, client=client,
|
|
environment_id="env-1",
|
|
git_service=git_service,
|
|
)
|
|
# #endregion
|
|
|
|
# #region Test.StructureSnapshot.Capture.TestSuccessfulCapture
|
|
@pytest.mark.asyncio
|
|
async def test_successful_capture(self, tmp_path):
|
|
"""Full capture → persist flow succeeds with valid release."""
|
|
release = MagicMock(spec=DashboardRelease)
|
|
release.id = "rel-3"
|
|
release.version = "v3.0.0"
|
|
release.commit_hash = "c" * 40
|
|
release.repository_id = "repo-3"
|
|
release.deployment = MagicMock()
|
|
release.deployment.environment_id = "env-1"
|
|
|
|
repo_record = MagicMock(spec=GitRepository)
|
|
repo_record.id = "repo-3"
|
|
repo_record.dashboard_id = 44
|
|
repo_record.local_path = "git_repos/success-project"
|
|
|
|
db = MagicMock()
|
|
db.query.return_value.filter.return_value.first.side_effect = [
|
|
release, repo_record
|
|
]
|
|
|
|
git_service = MagicMock()
|
|
git_service.legacy_base_path = str(tmp_path / "git_repos")
|
|
git_service.get_repo.return_value = MagicMock()
|
|
|
|
client = AsyncMock()
|
|
|
|
with patch(
|
|
"src.services.dashboard_testing.structure_snapshot_capture.inspect_dashboard_query_model",
|
|
new_callable=AsyncMock,
|
|
) as mock_inspect:
|
|
model = DashboardQueryModel(
|
|
environment_id="env-1",
|
|
dashboard_id=44,
|
|
title="Test Dashboard",
|
|
charts=[],
|
|
datasets=[],
|
|
native_filters=[],
|
|
query_model_fingerprint="sha256:valid_fp",
|
|
)
|
|
mock_inspect.return_value = model
|
|
|
|
response = await capture_release_snapshot(
|
|
SnapshotCaptureRequest(release_id="rel-3"),
|
|
db=db, client=client,
|
|
environment_id="env-1",
|
|
git_service=git_service,
|
|
dashboard_id_override=44,
|
|
)
|
|
|
|
assert isinstance(response, SnapshotCaptureResponse)
|
|
assert response.release_version == "v3.0.0"
|
|
assert response.environment_id == "env-1"
|
|
assert response.query_model_fingerprint == "sha256:valid_fp"
|
|
assert response.snapshot_path is not None
|
|
assert "v3.0.0.json" in response.snapshot_path or "v3.0.0" in response.snapshot_path
|
|
# #endregion
|
|
# #endregion
|
|
#endregion Test.BaselineEngine.StructureSnapshot.Service
|