busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity
Files
a1cb18fad9b0e6e4a5f05a18b963cf43c9109d2d
ss-tools/backend/tests/services/dashboard_testing/test_visual_baseline_lifecycle.py
busya a32ca0631b feat(037): capture, verification lifecycle, inheritance + close 036 stabilization 
- Authoritative candidate capture with server-issued artifacts and raw-byte
  immutability hashing (source_response_hash server-owned)
- Closed-period lifecycle: request-hash bound approvals, persisted closure
  immutability violations, byte-for-byte catalog stability on reclosure
- Verification runs: persisted VerificationRun model + FK migration,
  publish gate (block_publish), scheduled observability runs (02:00 UTC)
- FR-013 baseline inheritance: prior_release_id migration, plan_inheritance/
  execute_inheritance classification and re-extraction, API endpoints
- Visual executor bound to release-deployment environment; caller mismatch
  rejected; visual SSIM/reconciliation modules
- Query execution decomposed: envelope/model/executor split, no direct SQL
- AgentRun approvals extracted to submodule; evidence adapter; _utils
- Dashboard testing service decomposed into 30+ modules (all <400 LOC)
- Five Feature-037 agent tools with permission guards (tools_037.py)
- API readiness endpoint; Alembic env/migrations; test fixture repos
- Specs 036/037 contracts, openapi.yaml, schema.json, tasks/traceability
  updated; semantic index rebuilt with 0 parse warnings
- Fix ADR-0003 parser ambiguity: remove [DEF🆔ADR] prose example
- Add axiom-mcp-agent-feedback.md: agent findings for MCP rework plan
- Tests: 298 service + 1464 API + 45 agent passing; ruff clean
2026-07-31 11:28:50 +03:00

514 lines
21 KiB
Python
Raw Blame History

# #region Test.DashboardTesting.VisualBaselineLifecycle [C:3] [TYPE Module] [SEMANTICS testing,baseline,visual,cross-kind,staleness,orchestrator]
# @defgroup Tests for visual baseline lifecycle — cross-kind guard, staleness detection, orchestrator integration.
# @LAYER Test
# @RELATION VERIFIES -> [BaselineEngine.Visual.Compare]
# @RELATION VERIFIES -> [BaselineEngine.Visual.DetectStaleness]
from __future__ import annotations
from datetime import UTC, datetime
import hashlib
import io
from uuid import uuid4
import numpy as np
from PIL import Image
from src.schemas.dashboard_testing import (
ApprovalInfo,
ComparisonPolicy,
ComparisonPolicyType,
ComparisonStatus,
NormalizedFilterContext,
Provenance,
VisualBaselineEntry,
VisualFingerprints,
)
from src.services.dashboard_testing.visual_baseline import (
compare_visual_baseline,
compute_layout_fingerprint,
)
from src.services.dashboard_testing.visual_ssim import (
compare_visual_perceptual,
compute_ssim,
)
def _make_image_fixture(width: int, height: int, fill: int) -> bytes:
"""Generate a PNG image from a fixed fill value."""
arr = np.full((height, width), fill, dtype=np.uint8)
buf = io.BytesIO()
Image.fromarray(arr, mode="L").save(buf, format="PNG")
return buf.getvalue()
def _make_checkerboard(width: int, height: int, tile_size: int = 8) -> bytes:
"""Generate a checkerboard PNG from a fixed pixel pattern."""
arr = np.zeros((height, width), dtype=np.uint8)
for y in range(height):
for x in range(width):
arr[y, x] = 0 if ((x // tile_size) + (y // tile_size)) % 2 == 0 else 255
buf = io.BytesIO()
Image.fromarray(arr, mode="L").save(buf, format="PNG")
return buf.getvalue()
def _make_modified(original_bytes: bytes, x: int, y: int, new_val: int) -> bytes:
"""Modify a single pixel and re-encode as PNG (for near-identical images)."""
buf = io.BytesIO(original_bytes)
img = Image.open(buf).convert("L")
arr = np.array(img, dtype=np.uint8)
if y < arr.shape[0] and x < arr.shape[1]:
arr[y, x] = new_val
buf_out = io.BytesIO()
Image.fromarray(arr, mode="L").save(buf_out, format="PNG")
return buf_out.getvalue()
def _make_gradient(width: int, height: int) -> bytes:
"""Generate a horizontal gradient PNG from a fixed pixel pattern."""
arr = np.zeros((height, width), dtype=np.uint8)
for x in range(width):
val = round((x / max(width - 1, 1)) * 255)
arr[:, x] = val
buf = io.BytesIO()
Image.fromarray(arr, mode="L").save(buf, format="PNG")
return buf.getvalue()
def _make_visual_entry(**overrides) -> VisualBaselineEntry:
"""Build a VisualBaselineEntry for tests with sensible defaults (feature-037: release pinning)."""
now = datetime.now(UTC)
params = {
"baseline_id": uuid4(),
"release_version": "v1.0.0",
"release_commit_hash": "a" * 40,
"dashboard_id": 42,
"normalized_filters": NormalizedFilterContext(
filters=[],
filters_hash="sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
),
"tab_identifier": "TAB-main",
"expected_image_sha256": "e" * 64,
"source_response_hash": "s" * 64,
"captured_at": now,
"policy": ComparisonPolicy(type=ComparisonPolicyType.VISUAL_EXACT),
"status": "approved",
"fingerprints": VisualFingerprints(
query="a" * 64,
dataset="b" * 64,
filter="c" * 64,
layout="d" * 64,
),
"provenance": Provenance(environment="ss-preprod", actor="qa"),
"approval": ApprovalInfo(by="qa_analyst", at=now),
"created_at": now,
"updated_at": now,
}
params.update(overrides)
return VisualBaselineEntry(**params)
# ═══════════════════════════════════════════════════════════════════════════════
# Cross-kind guard + stale detection (compare_visual_baseline orchestrator)
# ═══════════════════════════════════════════════════════════════════════════════
# #region Test.DashboardTesting.VisualBaseline.CrossKind [C:3] [TYPE Function] [SEMANTICS testing,cross-kind,staleness]
def test_metric_policy_on_visual_is_inconclusive():
"""T047: Metric policy (exact) applied to visual baseline → inconclusive."""
policy = ComparisonPolicy(type=ComparisonPolicyType.EXACT)
result = compare_visual_baseline(
actual_image_sha256="sha1",
expected_image_sha256="sha2",
policy=policy,
)
assert result.status == ComparisonStatus.INCONCLUSIVE
assert any("CROSS_KIND" in w.code for w in result.warnings)
def test_visual_policy_works_exact():
"""T047: Visual exact policy routes to hash comparison."""
policy = ComparisonPolicy(type=ComparisonPolicyType.VISUAL_EXACT)
result = compare_visual_baseline(
actual_image_sha256="abc",
expected_image_sha256="abc",
policy=policy,
)
assert result.status == ComparisonStatus.PASS
def test_visual_policy_works_perceptual():
"""Visual perceptual policy routes to SSIM comparison with image bytes."""
img = _make_image_fixture(64, 64, 128)
policy = ComparisonPolicy(type=ComparisonPolicyType.VISUAL_PERCEPTUAL, amount="0.95")
result = compare_visual_baseline(
actual_image_data=img,
expected_image_data=img,
policy=policy,
)
assert result.status == ComparisonStatus.PASS
def test_stale_visual_baseline():
"""T047: Stale layout dimensions → stale_visual_baseline status (derived from fingerprints)."""
now = datetime.now(UTC)
baseline = VisualBaselineEntry(
baseline_id=uuid4(),
release_version="v1.0.0",
release_commit_hash="a" * 40,
dashboard_id=42,
normalized_filters=NormalizedFilterContext(filters=[], filters_hash="sha256:" + "a" * 64),
tab_identifier="TAB-main",
expected_image_sha256="abc",
source_response_hash="s" * 64,
captured_at=now,
policy=ComparisonPolicy(type=ComparisonPolicyType.VISUAL_EXACT),
fingerprints=VisualFingerprints(query="q_fp", dataset="d_fp", filter="f_fp", layout="baseline_layout"),
provenance=Provenance(environment="ss-preprod", actor="qa"),
approval=ApprovalInfo(by="qa_analyst", at=now),
created_at=now,
)
policy = ComparisonPolicy(type=ComparisonPolicyType.VISUAL_EXACT)
# Current layout fingerprint differs from baseline => staleness derived from fingerprints
result = compare_visual_baseline(
actual_image_sha256="abc",
expected_image_sha256="abc",
policy=policy,
visual_baseline=baseline,
current_layout_fingerprint="current_layout",
)
assert result.status == ComparisonStatus.STALE_VISUAL_BASELINE
assert "layout" in result.stale_dimensions
def test_stale_dimensions_propagate():
"""Multiple stale dimensions all appear in result (derived from fingerprints, not caller-supplied)."""
now = datetime.now(UTC)
baseline = VisualBaselineEntry(
baseline_id=uuid4(),
release_version="v1.0.0",
release_commit_hash="a" * 40,
dashboard_id=42,
normalized_filters=NormalizedFilterContext(filters=[], filters_hash="sha256:" + "a" * 64),
tab_identifier="TAB-main",
expected_image_sha256="abc",
source_response_hash="s" * 64,
captured_at=now,
policy=ComparisonPolicy(type=ComparisonPolicyType.VISUAL_EXACT),
fingerprints=VisualFingerprints(query="b_q", dataset="d_fp", filter="f_fp", layout="b_l"),
provenance=Provenance(environment="ss-preprod", actor="qa"),
approval=ApprovalInfo(by="qa_analyst", at=now),
created_at=now,
)
policy = ComparisonPolicy(type=ComparisonPolicyType.VISUAL_EXACT)
result = compare_visual_baseline(
actual_image_sha256="abc",
expected_image_sha256="abc",
policy=policy,
visual_baseline=baseline,
current_layout_fingerprint="c_l",
current_query_fingerprint="c_q",
)
assert result.status == ComparisonStatus.STALE_VISUAL_BASELINE
assert "layout" in result.stale_dimensions
assert "query" in result.stale_dimensions
# #endregion Test.DashboardTesting.VisualBaseline.CrossKind
# ═══════════════════════════════════════════════════════════════════════════════
# Orchestrator integration (compare_visual_baseline with real image data)
# ═══════════════════════════════════════════════════════════════════════════════
# #region Test.DashboardTesting.VisualBaseline.Orchestrator [C:3] [TYPE Function] [SEMANTICS testing,orchestrator,integration]
def test_compare_baseline_exact_with_image_bytes():
"""compare_visual_baseline with VISUAL_EXACT uses hash from image bytes."""
img = _make_image_fixture(32, 32, 128)
h = hashlib.sha256(img).hexdigest()
policy = ComparisonPolicy(type=ComparisonPolicyType.VISUAL_EXACT)
result = compare_visual_baseline(
actual_image_sha256=h,
expected_image_sha256=h,
policy=policy,
)
assert result.status == ComparisonStatus.PASS
def test_compare_baseline_exact_fail_different_images():
"""compare_visual_baseline with VISUAL_EXACT fails for different images."""
img1 = _make_image_fixture(32, 32, 128)
img2 = _make_image_fixture(32, 32, 200)
h1 = hashlib.sha256(img1).hexdigest()
h2 = hashlib.sha256(img2).hexdigest()
policy = ComparisonPolicy(type=ComparisonPolicyType.VISUAL_EXACT)
result = compare_visual_baseline(
actual_image_sha256=h1,
expected_image_sha256=h2,
policy=policy,
)
assert result.status == ComparisonStatus.FAIL
assert len(result.diff) > 0
def test_compare_baseline_perceptual_pass():
"""compare_visual_baseline with VISUAL_PERCEPTUAL passes for identical images."""
img = _make_image_fixture(64, 64, 128)
policy = ComparisonPolicy(type=ComparisonPolicyType.VISUAL_PERCEPTUAL, amount="0.95")
result = compare_visual_baseline(
actual_image_data=img,
expected_image_data=img,
policy=policy,
)
assert result.status == ComparisonStatus.PASS
def test_compare_baseline_perceptual_fail():
"""compare_visual_baseline with VISUAL_PERCEPTUAL fails for different images."""
check_img = _make_checkerboard(64, 64)
grad_img = _make_gradient(64, 64)
policy = ComparisonPolicy(type=ComparisonPolicyType.VISUAL_PERCEPTUAL, amount="0.95")
result = compare_visual_baseline(
actual_image_data=check_img,
expected_image_data=grad_img,
policy=policy,
)
assert result.status == ComparisonStatus.FAIL
def test_compare_baseline_perceptual_fallback_hash_pass():
"""VISUAL_PERCEPTUAL with hash data but no image data falls back to hash comparison."""
img = _make_image_fixture(64, 64, 128)
h = hashlib.sha256(img).hexdigest()
policy = ComparisonPolicy(type=ComparisonPolicyType.VISUAL_PERCEPTUAL, amount="0.95")
result = compare_visual_baseline(
actual_image_sha256=h,
expected_image_sha256=h,
policy=policy,
)
assert result.status == ComparisonStatus.PASS
def test_compare_baseline_perceptual_fallback_hash_fail():
"""VISUAL_PERCEPTUAL fallback passes same hashes, is inconclusive for different."""
policy = ComparisonPolicy(type=ComparisonPolicyType.VISUAL_PERCEPTUAL, amount="0.95")
result = compare_visual_baseline(
actual_image_sha256="hash_a",
expected_image_sha256="hash_b",
policy=policy,
)
assert result.status == ComparisonStatus.INCONCLUSIVE
def test_compare_baseline_no_data_inconclusive():
"""compare_visual_baseline returns INCONCLUSIVE when no data provided."""
policy = ComparisonPolicy(type=ComparisonPolicyType.VISUAL_PERCEPTUAL)
result = compare_visual_baseline(policy=policy)
assert result.status == ComparisonStatus.INCONCLUSIVE
def test_compare_baseline_stale_takes_priority():
"""Stale dimensions take priority over comparison logic (derived from fingerprints)."""
now = datetime.now(UTC)
baseline = VisualBaselineEntry(
baseline_id=uuid4(),
release_version="v1.0.0",
release_commit_hash="a" * 40,
dashboard_id=42,
normalized_filters=NormalizedFilterContext(filters=[], filters_hash="sha256:" + "a" * 64),
tab_identifier="TAB-main",
expected_image_sha256="abc",
source_response_hash="s" * 64,
captured_at=now,
policy=ComparisonPolicy(type=ComparisonPolicyType.VISUAL_EXACT),
fingerprints=VisualFingerprints(query="q", dataset="d", filter="f", layout="baseline_fp"),
provenance=Provenance(environment="ss-preprod", actor="qa"),
approval=ApprovalInfo(by="qa_analyst", at=now),
created_at=now,
)
img1 = _make_image_fixture(32, 32, 128)
img2 = _make_image_fixture(32, 32, 200)
h1 = hashlib.sha256(img1).hexdigest()
h2 = hashlib.sha256(img2).hexdigest()
policy = ComparisonPolicy(type=ComparisonPolicyType.VISUAL_EXACT)
result = compare_visual_baseline(
actual_image_sha256=h1,
expected_image_sha256=h2,
policy=policy,
visual_baseline=baseline,
current_layout_fingerprint="current_fp",
)
assert result.status == ComparisonStatus.STALE_VISUAL_BASELINE
# #endregion Test.DashboardTesting.VisualBaseline.Orchestrator
# ═══════════════════════════════════════════════════════════════════════════════
# Feature-037: Anti-correlated fixture + ssim_min/pixel_diff_threshold validation
# ═══════════════════════════════════════════════════════════════════════════════
# #region Test.DashboardTesting.VisualBaseline.AntiCorrelated [C:3] [TYPE Function] [SEMANTICS testing,ssim,anti-correlated,threshold]
def test_ssim_anti_correlated_patterns():
"""Feature-037: Two complementary checkerboard patterns (inverse of each other)
produce SSIM ≈ 0 (anti-correlated)."""
w, h = 64, 64
# Pattern A: standard checkerboard
a = np.zeros((h, w), dtype=np.uint8)
for y in range(h):
for x in range(w):
a[y, x] = 0 if ((x // 8) + (y // 8)) % 2 == 0 else 255
# Pattern B: inverse of A
b = np.zeros((h, w), dtype=np.uint8)
for y in range(h):
for x in range(w):
b[y, x] = 255 if ((x // 8) + (y // 8)) % 2 == 0 else 0
ssim = compute_ssim(a, b)
assert ssim == 0.0, f"Anti-correlated patterns should yield SSIM=0, got {ssim}"
def test_ssim_clamped_to_zero_one():
"""Feature-037: SSIM is always clamped to [0, 1] even with extreme inputs."""
# Two identical images: SSIM = 1.0
a = np.full((10, 10), 128, dtype=np.uint8)
assert compute_ssim(a, a) == 1.0
# Two opposite images: SSIM = 0.0 (clamped)
black = np.zeros((10, 10), dtype=np.uint8)
white = np.full((10, 10), 255, dtype=np.uint8)
ssim = compute_ssim(black, white)
assert 0.0 <= ssim <= 1.0
assert ssim < 0.01
# #endregion Test.DashboardTesting.VisualBaseline.AntiCorrelated
# #region Test.DashboardTesting.VisualBaseline.ThresholdValidation [C:3] [TYPE Function] [SEMANTICS testing,ssim_min,threshold,validation]
def test_ssim_min_out_of_range_high():
"""Feature-037: ssim_min > 1.0 returns INCONCLUSIVE."""
img = _make_image_fixture(32, 32, 128)
status, _diff = compare_visual_perceptual(img, img, ssim_min=1.5)
assert status == ComparisonStatus.INCONCLUSIVE
def test_ssim_min_out_of_range_low():
"""Feature-037: ssim_min < 0 returns INCONCLUSIVE."""
img = _make_image_fixture(32, 32, 128)
status, _diff = compare_visual_perceptual(img, img, ssim_min=-0.1)
assert status == ComparisonStatus.INCONCLUSIVE
def test_pixel_diff_threshold_negative():
"""Feature-037: Negative pixel_diff_threshold returns INCONCLUSIVE."""
base = _make_image_fixture(32, 32, 128)
modified = _make_modified(base, 0, 0, 200)
status, _diff = compare_visual_perceptual(base, modified, ssim_min=0.95, pixel_diff_threshold=-0.01)
assert status == ComparisonStatus.INCONCLUSIVE
def test_pixel_diff_threshold_gt_one():
"""Feature-037: pixel_diff_threshold > 1.0 returns INCONCLUSIVE (must be in [0,1])."""
base = _make_image_fixture(32, 32, 128)
modified = _make_modified(base, 0, 0, 200)
status, _diff = compare_visual_perceptual(base, modified, ssim_min=0.95, pixel_diff_threshold=1.5)
assert status == ComparisonStatus.INCONCLUSIVE
def test_pixel_diff_threshold_ok():
"""Feature-037: pixel_diff_threshold >= actual diff ratio => PASS."""
base = _make_image_fixture(32, 32, 200)
# Modify one pixel
modified = _make_modified(base, 0, 0, 201)
status, _diff = compare_visual_perceptual(base, modified, ssim_min=0.95, pixel_diff_threshold=0.01)
# SSIM is near 1, pixel diff is 1/(32*32) ≈ 0.001 < 0.01 => PASS
assert status == ComparisonStatus.PASS
def test_pixel_diff_threshold_fail():
"""Feature-037: pixel_diff_threshold below actual diff ratio => FAIL."""
base = _make_image_fixture(32, 32, 128)
# Make half the image different
buf = io.BytesIO(base)
img = Image.open(buf).convert("L")
arr = np.array(img, dtype=np.uint8)
arr[:, 16:] = 0 # Right half black
buf_out = io.BytesIO()
Image.fromarray(arr, mode="L").save(buf_out, format="PNG")
modified = buf_out.getvalue()
# pixel_diff ~0.5, threshold 0.01 => FAIL
status, _diff = compare_visual_perceptual(base, modified, ssim_min=0.0, pixel_diff_threshold=0.01)
assert status == ComparisonStatus.FAIL
# #endregion Test.DashboardTesting.VisualBaseline.ThresholdValidation
# ═══════════════════════════════════════════════════════════════════════════════
# Feature-037: Layout fingerprint — tab/region hierarchy
# ═══════════════════════════════════════════════════════════════════════════════
# #region Test.DashboardTesting.VisualBaseline.LayoutTabHierarchy [C:3] [TYPE Function] [SEMANTICS testing,layout,fingerprint,tab-hierarchy]
def test_layout_fingerprint_includes_tab_hierarchy():
"""Feature-037: Tab hierarchy is included in layout fingerprint."""
position = {
"TAB-1": {"meta": {"children": ["CHART-128", "CHART-129"]}},
"CHART-128": {"meta": {"chartId": 128, "width": 6, "height": 12}, "parent_id": "TAB-1"},
"CHART-129": {"meta": {"chartId": 129, "width": 12, "height": 4}, "parent_id": "TAB-1"},
}
fp = compute_layout_fingerprint(position, [128, 129])
assert len(fp) == 64 # SHA-256 hex
def test_layout_fingerprint_tab_reorder_changes():
"""Feature-037: Reordering tabs changes fingerprint even if chart positions unchanged.
Tab children are inserted in order; reordering produces a different fingerprint."""
# Two tabs with identical chart structure but different order
pos_a = {
"TAB-A": {"meta": {"children": ["CHART-1"]}},
"TAB-B": {"meta": {"children": ["CHART-2"]}},
"CHART-1": {"meta": {"chartId": 1, "width": 6, "height": 6}, "parent_id": "TAB-A"},
"CHART-2": {"meta": {"chartId": 2, "width": 6, "height": 6}, "parent_id": "TAB-B"},
}
pos_b = {
"TAB-B": {"meta": {"children": ["CHART-2"]}},
"TAB-A": {"meta": {"children": ["CHART-1"]}},
"CHART-1": {"meta": {"chartId": 1, "width": 6, "height": 6}, "parent_id": "TAB-A"},
"CHART-2": {"meta": {"chartId": 2, "width": 6, "height": 6}, "parent_id": "TAB-B"},
}
fp_a = compute_layout_fingerprint(pos_a, [1, 2])
fp_b = compute_layout_fingerprint(pos_b, [1, 2])
# Insertion order is preserved (not sorted), so reordered tabs produce different fingerprints
assert fp_a != fp_b, (
"Tab reordering must produce different fingerprint "
"(insertion order preserved, not sorted alphabetically)"
)
def test_layout_fingerprint_chart_geometry_oriented():
"""Feature-037: Chart geometry (row/col/width/height) changes fingerprint."""
pos1 = {
"CHART-1": {"meta": {"chartId": 1, "width": 6, "height": 6, "row": 0, "col": 0}},
}
pos2 = {
"CHART-1": {"meta": {"chartId": 1, "width": 12, "height": 6, "row": 0, "col": 0}},
}
fp1 = compute_layout_fingerprint(pos1, [1])
fp2 = compute_layout_fingerprint(pos2, [1])
assert fp1 != fp2
# #endregion Test.DashboardTesting.VisualBaseline.LayoutTabHierarchy
# #endregion Test.DashboardTesting.VisualBaselineLifecycle
Reference in New Issue View Git Blame Copy Permalink