busya
a32ca0631b
- Authoritative candidate capture with server-issued artifacts and raw-byte
immutability hashing (source_response_hash server-owned)
- Closed-period lifecycle: request-hash bound approvals, persisted closure
immutability violations, byte-for-byte catalog stability on reclosure
- Verification runs: persisted VerificationRun model + FK migration,
publish gate (block_publish), scheduled observability runs (02:00 UTC)
- FR-013 baseline inheritance: prior_release_id migration, plan_inheritance/
execute_inheritance classification and re-extraction, API endpoints
- Visual executor bound to release-deployment environment; caller mismatch
rejected; visual SSIM/reconciliation modules
- Query execution decomposed: envelope/model/executor split, no direct SQL
- AgentRun approvals extracted to submodule; evidence adapter; _utils
- Dashboard testing service decomposed into 30+ modules (all <400 LOC)
- Five Feature-037 agent tools with permission guards (tools_037.py)
- API readiness endpoint; Alembic env/migrations; test fixture repos
- Specs 036/037 contracts, openapi.yaml, schema.json, tasks/traceability
updated; semantic index rebuilt with 0 parse warnings
- Fix ADR-0003 parser ambiguity: remove [DEF🆔ADR] prose example
- Add axiom-mcp-agent-feedback.md: agent findings for MCP rework plan
- Tests: 298 service + 1464 API + 45 agent passing; ruff clean
310 lines
17 KiB
Python
310 lines
17 KiB
Python
# #region Test.Api.DashboardTesting.ClosedPeriod [C:4] [TYPE Module] [SEMANTICS testing,api,dashboard-testing,closed-period,immutability,authoritative]
|
|
# @defgroup Closed-period transition E2E tests — server-issued timestamp/hash, reclosure rejection, hash mutation.
|
|
# @LAYER Test
|
|
# @RELATION VERIFIES -> [BaselineEngine.Candidates.ApprovalLifecycle.ConsumeApproval]
|
|
# @RELATION BINDS_TO -> [BaselineEngine.Candidates.Guards.CheckClosedPeriodInCatalog]
|
|
# @TEST_EDGE: authoritative_close -> Server sets period_closed_at and source_response_hash at consume.
|
|
# @TEST_EDGE: close_period_bound_in_gate -> close_period is stored in capture_meta and request-hash bound.
|
|
# @TEST_EDGE: reject_reclosure -> Already-closed period can't be overwritten (409 conflict, catalog unchanged).
|
|
# @TEST_EDGE: changed_bytes_same_scalar -> compare_values with different hashes returns immutability_violation.
|
|
# @TEST_EDGE: same_body_no_violation -> Same hash passes immutability check.
|
|
# @TEST_EDGE: close_period_mutation -> _verify_request_hash rejects when close_period differs.
|
|
from __future__ import annotations
|
|
|
|
from .conftest import (
|
|
create_dashboard_testing_agent_run as _create_agent_run,
|
|
create_dashboard_testing_capture_artifact as _create_capture_artifact,
|
|
make_dashboard_testing_candidate_payload as _make_candidate_payload,
|
|
)
|
|
|
|
|
|
# #region Test.Api.DashboardTesting.ClosedPeriod.Transition [C:4] [TYPE Class] [SEMANTICS test,api,approval,close-period,immutability,authoritative]
|
|
# @BRIEF Governed closed-period transition E2E — server-issued timestamp/hash, no client closure fields,
|
|
# same-body no violation, changed bytes persisted critical violation.
|
|
# @RELATION VERIFIES -> [BaselineEngine.Candidates.ApprovalLifecycle.ConsumeApproval]
|
|
# @TEST_EDGE: authoritative_close -> Server sets period_closed_at and source_response_hash at consume.
|
|
# @TEST_EDGE: close_period_bound_in_gate -> close_period is stored in capture_meta and request-hash bound.
|
|
# @TEST_EDGE: reject_reclosure -> Already-closed period cannot be overwritten.
|
|
class TestClosedPeriodTransition:
|
|
"""Governed closed-period transition — server-issued closure, no client hash/time, reclosure rejection."""
|
|
|
|
# #region Test.Api.ClosedPeriod.AuthoritativeCloseViaGate [C:3] [TYPE Function] [SEMANTICS test,api,approval,close-period,authoritative]
|
|
# @BRIEF Close_period in approval-gate request produces server-issued period_closed_at and source_response_hash.
|
|
# @TEST_EDGE authoritative_close -> After consume with close_period, the catalog entry's immutability block
|
|
# has period_closed_at = server timestamp and source_response_hash = capture artifact hash.
|
|
# @INVARIANT Clients cannot supply period_closed_at or source_response_hash — server computes both.
|
|
def test_authoritative_close_via_gate(self, dashboard_testing_client, tmp_path, monkeypatch):
|
|
"""Server-issued closure: close_period in approval-gate → consume writes period_closed_at + hash."""
|
|
from src.core.database import SessionLocal
|
|
from src.models.agent_run import DraftArtifact
|
|
|
|
monkeypatch.setattr(
|
|
"src.services.dashboard_testing.safe_path._DEFAULT_BASE",
|
|
tmp_path.resolve(),
|
|
)
|
|
monkeypatch.setattr("pathlib.Path.cwd", lambda: tmp_path.resolve())
|
|
draft_root = tmp_path / "drafts"
|
|
draft_root.mkdir()
|
|
monkeypatch.setenv("DRAFT_STORAGE_ROOT", str(draft_root))
|
|
monkeypatch.setattr("src.services.agent_runs.artifacts._draft_storage", None)
|
|
|
|
setup_session = SessionLocal()
|
|
try:
|
|
run = _create_agent_run(setup_session)
|
|
run_id = run.id
|
|
artifact_id, sha256 = _create_capture_artifact(setup_session, run_id)
|
|
setup_session.commit()
|
|
finally:
|
|
setup_session.close()
|
|
|
|
payload = _make_candidate_payload(run_id, capture_artifact_ref=artifact_id, source_response_hash=sha256)
|
|
resp = dashboard_testing_client.post("/api/dashboard-testing/baseline-candidates", json=payload)
|
|
assert resp.status_code == 201
|
|
candidate_id = resp.json()["candidate_id"]
|
|
|
|
resp = dashboard_testing_client.post(
|
|
f"/api/dashboard-testing/baseline-candidates/{candidate_id}/approval-gate",
|
|
json={
|
|
"agent_run_id": run_id, "release_version": "v1.0.0",
|
|
"release_commit_hash": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b",
|
|
"close_period": "2026-07", "reason": "Q3 close",
|
|
},
|
|
)
|
|
assert resp.status_code == 201
|
|
gate_id = resp.json()["gate_id"]
|
|
|
|
verify_session = SessionLocal()
|
|
try:
|
|
draft = verify_session.query(DraftArtifact).filter(DraftArtifact.id == candidate_id).first()
|
|
assert draft is not None
|
|
meta = draft.capture_meta or {}
|
|
assert meta.get("close_period") == "2026-07"
|
|
finally:
|
|
verify_session.close()
|
|
|
|
resp = dashboard_testing_client.post(
|
|
f"/api/dashboard-testing/baseline-candidates/{candidate_id}/approval-gate/{gate_id}/decide",
|
|
json={"decision": "confirm", "reason": "QA approved"},
|
|
)
|
|
assert resp.status_code == 200
|
|
|
|
resp = dashboard_testing_client.post(
|
|
f"/api/dashboard-testing/baseline-candidates/{candidate_id}/approval-gate/{gate_id}/consume"
|
|
f"?release_version=v1.0.0&release_commit_hash=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b",
|
|
)
|
|
assert resp.status_code == 200
|
|
assert resp.json()["consumed"] is True
|
|
|
|
catalog_path = tmp_path / "git_repos" / "test-repo" / "dashboard_tests" / "test-dash" / "baselines.yaml"
|
|
if not catalog_path.exists():
|
|
catalog_path = list(tmp_path.rglob("baselines.yaml"))
|
|
assert catalog_path
|
|
cat_file = catalog_path[0] if isinstance(catalog_path, list) else catalog_path
|
|
import yaml as _yaml
|
|
raw = _yaml.safe_load(cat_file.read_text())
|
|
entry = raw["entries"][0]
|
|
imm = entry.get("immutability")
|
|
assert imm is not None
|
|
assert imm["enabled"] is True
|
|
assert imm["period"] == "2026-07"
|
|
assert imm.get("period_closed_at") is not None
|
|
assert imm["source_response_hash"] == sha256
|
|
assert imm["policy"] == "block_publish"
|
|
# #endregion Test.Api.ClosedPeriod.AuthoritativeCloseViaGate
|
|
|
|
# #region Test.Api.ClosedPeriod.RejectReclosure [C:3] [TYPE Function] [SEMANTICS test,api,approval,close-period,reject-reclosure]
|
|
# @BRIEF Second candidate for same metric coordinate with same close_period → 409 conflict + catalog unchanged.
|
|
def test_authoritative_close_idempotent_reclosure_raises_error(self, dashboard_testing_client, tmp_path, monkeypatch):
|
|
"""Second candidate reclosure attempt returns 409, catalog unchanged."""
|
|
from src.core.database import SessionLocal
|
|
from src.models.agent_run import DraftArtifact as _DraftArtifact
|
|
|
|
monkeypatch.setattr("src.services.dashboard_testing.safe_path._DEFAULT_BASE", tmp_path.resolve())
|
|
monkeypatch.setattr("pathlib.Path.cwd", lambda: tmp_path.resolve())
|
|
draft_root = tmp_path / "drafts"
|
|
draft_root.mkdir()
|
|
monkeypatch.setenv("DRAFT_STORAGE_ROOT", str(draft_root))
|
|
monkeypatch.setattr("src.services.agent_runs.artifacts._draft_storage", None)
|
|
|
|
setup_session = SessionLocal()
|
|
try:
|
|
run = _create_agent_run(setup_session)
|
|
run_id = run.id
|
|
artifact_id, sha256 = _create_capture_artifact(setup_session, run_id)
|
|
setup_session.commit()
|
|
finally:
|
|
setup_session.close()
|
|
|
|
# First candidate: consume with close_period
|
|
payload = _make_candidate_payload(run_id, capture_artifact_ref=artifact_id, source_response_hash=sha256)
|
|
resp = dashboard_testing_client.post("/api/dashboard-testing/baseline-candidates", json=payload)
|
|
assert resp.status_code == 201
|
|
candidate_id = resp.json()["candidate_id"]
|
|
|
|
resp = dashboard_testing_client.post(
|
|
f"/api/dashboard-testing/baseline-candidates/{candidate_id}/approval-gate",
|
|
json={"agent_run_id": run_id, "release_version": "v1.0.0",
|
|
"release_commit_hash": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b",
|
|
"close_period": "2026-07"},
|
|
)
|
|
assert resp.status_code == 201
|
|
gate_id = resp.json()["gate_id"]
|
|
|
|
resp = dashboard_testing_client.post(
|
|
f"/api/dashboard-testing/baseline-candidates/{candidate_id}/approval-gate/{gate_id}/decide",
|
|
json={"decision": "confirm"},
|
|
)
|
|
assert resp.status_code == 200
|
|
|
|
resp = dashboard_testing_client.post(
|
|
f"/api/dashboard-testing/baseline-candidates/{candidate_id}/approval-gate/{gate_id}/consume"
|
|
f"?release_version=v1.0.0&release_commit_hash=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b",
|
|
)
|
|
assert resp.status_code == 200
|
|
|
|
# Snapshot catalog bytes and baseline ID BEFORE second consume attempt
|
|
cat_file = list(tmp_path.rglob("baselines.yaml"))
|
|
assert cat_file
|
|
with open(cat_file[0], "rb") as _f:
|
|
catalog_bytes_before = _f.read()
|
|
import yaml as _yaml
|
|
catalog_before = _yaml.safe_load(catalog_bytes_before)
|
|
original_baseline_id = catalog_before["entries"][0]["baseline_id"]
|
|
original_source_hash = catalog_before["entries"][0]["source_response_hash"]
|
|
|
|
# Second candidate: same coordinates, same close_period → 409
|
|
reclose_session = SessionLocal()
|
|
try:
|
|
import hashlib as _hl2
|
|
raw2 = b'{"result": "count", "value": 200}'
|
|
sha2 = _hl2.sha256(raw2).hexdigest()
|
|
art2_id, _ = _create_capture_artifact(reclose_session, run_id, result_key="count_reclose")
|
|
art2 = reclose_session.query(_DraftArtifact).filter(_DraftArtifact.id == art2_id).first()
|
|
meta2 = dict(art2.capture_meta or {})
|
|
meta2["result_key"] = "count"
|
|
meta2["dashboard_id"] = 42
|
|
meta2["chart_id"] = 1
|
|
meta2["repo_key"] = "test-repo"
|
|
meta2["dash_key"] = "test-dash"
|
|
art2.capture_meta = meta2
|
|
art2.sha256 = sha2
|
|
reclose_session.commit()
|
|
finally:
|
|
reclose_session.close()
|
|
|
|
payload2 = _make_candidate_payload(run_id, capture_artifact_ref=art2_id, source_response_hash=sha2)
|
|
payload2["result_key"] = "count"
|
|
resp = dashboard_testing_client.post("/api/dashboard-testing/baseline-candidates", json=payload2)
|
|
assert resp.status_code == 201
|
|
cand2_id = resp.json()["candidate_id"]
|
|
|
|
resp = dashboard_testing_client.post(
|
|
f"/api/dashboard-testing/baseline-candidates/{cand2_id}/approval-gate",
|
|
json={"agent_run_id": run_id, "release_version": "v1.0.0",
|
|
"release_commit_hash": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b",
|
|
"close_period": "2026-07"},
|
|
)
|
|
assert resp.status_code == 201
|
|
gate2_id = resp.json()["gate_id"]
|
|
|
|
resp = dashboard_testing_client.post(
|
|
f"/api/dashboard-testing/baseline-candidates/{cand2_id}/approval-gate/{gate2_id}/decide",
|
|
json={"decision": "confirm"},
|
|
)
|
|
assert resp.status_code == 200
|
|
|
|
resp = dashboard_testing_client.post(
|
|
f"/api/dashboard-testing/baseline-candidates/{cand2_id}/approval-gate/{gate2_id}/consume"
|
|
f"?release_version=v1.0.0&release_commit_hash=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b",
|
|
)
|
|
assert resp.status_code == 409
|
|
assert "Cannot close period" in resp.json()["detail"]
|
|
|
|
import yaml as _yaml
|
|
cat_file = list(tmp_path.rglob("baselines.yaml"))
|
|
assert cat_file
|
|
catalog_bytes_after = cat_file[0].read_bytes()
|
|
assert catalog_bytes_after == catalog_bytes_before, (
|
|
"Catalog bytes changed after 409 reclosure rejection. "
|
|
"The catalog must remain byte-identical to the pre-attempt state."
|
|
)
|
|
raw = _yaml.safe_load(catalog_bytes_after)
|
|
assert len(raw["entries"]) == 1
|
|
assert raw["entries"][0]["baseline_id"] == original_baseline_id
|
|
assert raw["entries"][0]["source_response_hash"] == original_source_hash
|
|
assert raw["entries"][0]["immutability"]["period_closed_at"] is not None
|
|
# #endregion Test.Api.ClosedPeriod.RejectReclosure
|
|
|
|
# #region Test.Api.ClosedPeriod.ImmutabilityViolationViaComparison [C:2] [TYPE Function] [SEMANTICS test,api,approval,immutability,violation,critical]
|
|
# @BRIEF Different hashes with same canonical value → immutability_violation via real compare_values SUT.
|
|
@staticmethod
|
|
def test_changed_bytes_same_scalar_produces_critical_violation():
|
|
from datetime import UTC, datetime
|
|
|
|
from src.schemas.dashboard_testing import ComparisonPolicy, ComparisonStatus, NormalizedValue, ValueKind
|
|
from src.schemas.dashboard_testing.catalog import ImmutabilityBlock
|
|
from src.schemas.dashboard_testing.enums import ImmutabilityPolicy
|
|
from src.services.dashboard_testing.comparison import compare_values
|
|
|
|
now = datetime.now(UTC)
|
|
closure_hash = "dbeb45b9d9081838c8f4b0b8e7a8d8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a"
|
|
current_hash = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2"
|
|
closed = ImmutabilityBlock(enabled=True, period="2026-07", period_closed_at=now,
|
|
frozen_at=now, source_response_hash=closure_hash,
|
|
policy=ImmutabilityPolicy.BLOCK_PUBLISH)
|
|
nv = NormalizedValue(kind=ValueKind.INTEGER, canonical_value="100")
|
|
result = compare_values(actual=nv, expected=nv, policy=ComparisonPolicy(type="exact"),
|
|
immutability=closed, current_source_response_hash=current_hash)
|
|
assert result.status == ComparisonStatus.IMMUTABILITY_VIOLATION
|
|
# #endregion Test.Api.ClosedPeriod.ImmutabilityViolationViaComparison
|
|
|
|
# #region Test.Api.ClosedPeriod.SameBodyNoViolation [C:2] [TYPE Function] [SEMANTICS test,api,approval,immutability,no-violation]
|
|
# @BRIEF Same hardcoded hash → no violation via real compare_values SUT.
|
|
@staticmethod
|
|
def test_same_body_no_immutability_violation():
|
|
from datetime import UTC, datetime
|
|
|
|
from src.schemas.dashboard_testing import ComparisonPolicy, ComparisonStatus, NormalizedValue, ValueKind
|
|
from src.schemas.dashboard_testing.catalog import ImmutabilityBlock
|
|
from src.schemas.dashboard_testing.enums import ImmutabilityPolicy
|
|
from src.services.dashboard_testing.comparison import compare_values
|
|
|
|
now = datetime.now(UTC)
|
|
fixture_hash = "dbeb45b9d9081838c8f4b0b8e7a8d8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a8a"
|
|
closed = ImmutabilityBlock(enabled=True, period="2026-07", period_closed_at=now,
|
|
frozen_at=now, source_response_hash=fixture_hash,
|
|
policy=ImmutabilityPolicy.BLOCK_PUBLISH)
|
|
nv = NormalizedValue(kind=ValueKind.INTEGER, canonical_value="100")
|
|
result = compare_values(actual=nv, expected=nv, policy=ComparisonPolicy(type="exact"),
|
|
immutability=closed, current_source_response_hash=fixture_hash)
|
|
assert result.status != ComparisonStatus.IMMUTABILITY_VIOLATION
|
|
# #endregion Test.Api.ClosedPeriod.SameBodyNoViolation
|
|
|
|
# #region Test.Api.ClosedPeriod.ClosePeriodMutationHashRejection [C:3] [TYPE Function] [SEMANTICS test,api,approval,close-period,hash,mutation]
|
|
# @BRIEF close_period bound in request hash — mutation detected by _verify_request_hash.
|
|
@staticmethod
|
|
def test_close_period_mutation_causes_hash_rejection():
|
|
import pytest as _pt
|
|
|
|
from src.services.dashboard_testing.candidate_guards import _compute_request_hash, _verify_request_hash
|
|
|
|
base = {"candidate_id": "cand-mutation-001", "content_hash": "d" * 64,
|
|
"intended_path": "git_repos/test-repo/dashboard_tests/test-dash/baselines.yaml",
|
|
"operation": "write_baseline", "release_version": "v1.0.0",
|
|
"release_commit_hash": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b"}
|
|
h_with = _compute_request_hash(**base, close_period="2026-07")
|
|
h_without = _compute_request_hash(**base)
|
|
assert h_with != h_without
|
|
h_diff = _compute_request_hash(**base, close_period="2026-08")
|
|
assert h_with != h_diff
|
|
_verify_request_hash(stored_hash=h_with, **base, close_period="2026-07")
|
|
with _pt.raises(ValueError, match="request_hash mismatch"):
|
|
_verify_request_hash(stored_hash=h_with, **base, close_period="2026-08")
|
|
with _pt.raises(ValueError, match="request_hash mismatch"):
|
|
_verify_request_hash(stored_hash=h_with, **base, close_period=None)
|
|
# #endregion Test.Api.ClosedPeriod.ClosePeriodMutationHashRejection
|
|
# #endregion Test.Api.DashboardTesting.ClosedPeriod.Transition
|
|
|
|
|
|
# #endregion Test.Api.DashboardTesting.ClosedPeriod
|