ss-tools/backend/tests/services/dashboard_testing/test_visual_baseline.py

# #region Test.DashboardTesting.VisualBaseline [C:3] [TYPE Module] [SEMANTICS testing,baseline,visual,layout-fingerprint,ssim]
# @defgroup Core visual baseline tests — layout fingerprint, SSIM, exact/perceptual comparison, image fixtures.
# @LAYER Test
# @RELATION VERIFIES -> [BaselineEngine.Visual.Compare]
# @NOTE Lifecycle tests (cross-kind, staleness, orchestrator) moved to test_visual_baseline_lifecycle.py

from __future__ import annotations

from datetime import UTC, datetime
import hashlib
import io
import pytest
from uuid import uuid4

import numpy as np
from PIL import Image

from src.schemas.dashboard_testing import (
    ApprovalInfo,
    BaselineStatus,
    ComparisonPolicy,
    ComparisonPolicyType,
    ComparisonStatus,
    NormalizedFilterContext,
    Provenance,
    VisualBaselineEntry,
    VisualFingerprints,
)
from src.services.dashboard_testing.visual_baseline import (
    compute_layout_fingerprint,
)
from src.services.dashboard_testing.visual_ssim import (
    compare_visual_exact,
    compare_visual_perceptual,
    compute_ssim,
)

# ── Image fixture helpers ────────────────────────────────────────────────────
# Hardcoded pixel arrays: these are NOT mirrors of the implementation; they are
# fixed deterministically generated images for reproducible test fixtures.


def _make_image_fixture(width: int, height: int, fill: int) -> bytes:
    """Generate a PNG image from a fixed fill value."""
    arr = np.full((height, width), fill, dtype=np.uint8)
    buf = io.BytesIO()
    Image.fromarray(arr, mode="L").save(buf, format="PNG")
    return buf.getvalue()


def _make_checkerboard(width: int, height: int, tile_size: int = 8) -> bytes:
    """Generate a checkerboard PNG from a fixed pixel pattern."""
    arr = np.zeros((height, width), dtype=np.uint8)
    for y in range(height):
        for x in range(width):
            arr[y, x] = 0 if ((x // tile_size) + (y // tile_size)) % 2 == 0 else 255
    buf = io.BytesIO()
    Image.fromarray(arr, mode="L").save(buf, format="PNG")
    return buf.getvalue()


def _make_gradient(width: int, height: int) -> bytes:
    """Generate a horizontal gradient PNG from a fixed pixel pattern."""
    arr = np.zeros((height, width), dtype=np.uint8)
    for x in range(width):
        val = round((x / max(width - 1, 1)) * 255)
        arr[:, x] = val
    buf = io.BytesIO()
    Image.fromarray(arr, mode="L").save(buf, format="PNG")
    return buf.getvalue()


def _make_modified(original_bytes: bytes, x: int, y: int, new_val: int) -> bytes:
    """Modify a single pixel and re-encode as PNG (for near-identical images)."""
    buf = io.BytesIO(original_bytes)
    img = Image.open(buf).convert("L")
    arr = np.array(img, dtype=np.uint8)
    if y < arr.shape[0] and x < arr.shape[1]:
        arr[y, x] = new_val
    buf_out = io.BytesIO()
    Image.fromarray(arr, mode="L").save(buf_out, format="PNG")
    return buf_out.getvalue()


def _make_striped(width: int, height: int, stripe_width: int = 10) -> bytes:
    """Generate vertical stripes (for clearly different image)."""
    arr = np.zeros((height, width), dtype=np.uint8)
    for x in range(width):
        arr[:, x] = 255 if (x // stripe_width) % 2 == 0 else 0
    buf = io.BytesIO()
    Image.fromarray(arr, mode="L").save(buf, format="PNG")
    return buf.getvalue()


# ── VisualBaselineEntry fixture builder ──────────────────────────────────────


def _make_visual_entry(**overrides) -> VisualBaselineEntry:
    """Build a VisualBaselineEntry for tests with sensible defaults (feature-037: release pinning)."""
    now = datetime.now(UTC)
    params = {
        "baseline_id": uuid4(),
        "release_version": "v1.0.0",
        "release_commit_hash": "a" * 40,
        "dashboard_id": 42,
        "normalized_filters": NormalizedFilterContext(
            filters=[],
            filters_hash="sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
        ),
        "tab_identifier": "TAB-main",
        "expected_image_sha256": "e" * 64,
        "source_response_hash": "s" * 64,
        "captured_at": now,
        "policy": ComparisonPolicy(type=ComparisonPolicyType.VISUAL_EXACT),
        "status": BaselineStatus.APPROVED,
        "fingerprints": VisualFingerprints(
            query="a" * 64,
            dataset="b" * 64,
            filter="c" * 64,
            layout="d" * 64,
        ),
        "provenance": Provenance(environment="ss-preprod", actor="qa"),
        "approval": ApprovalInfo(by="qa_analyst", at=now),
        "created_at": now,
        "updated_at": now,
    }
    params.update(overrides)
    return VisualBaselineEntry(**params)


# ═══════════════════════════════════════════════════════════════════════════════
# Layout Fingerprint
# ═══════════════════════════════════════════════════════════════════════════════


# #region Test.DashboardTesting.VisualBaseline.LayoutFingerprint [C:3] [TYPE Function]
def test_layout_fingerprint_deterministic():
    """T041: Same position data produces same fingerprint."""
    position = {
        "CHART-128": {"id": "CHART-128", "meta": {"chartId": 128, "width": 6, "height": 12}},
        "CHART-129": {"id": "CHART-129", "meta": {"chartId": 129, "width": 12, "height": 4}},
    }

    fp1 = compute_layout_fingerprint(position, [128, 129])
    fp2 = compute_layout_fingerprint(position, [128, 129])

    assert fp1 == fp2
    assert len(fp1) == 64  # SHA-256 hex


def test_layout_fingerprint_changes_with_position():
    """T041: Different positions produce different fingerprints."""
    pos1 = {"CHART-128": {"meta": {"chartId": 128, "width": 6}}}
    pos2 = {"CHART-128": {"meta": {"chartId": 128, "width": 12}}}

    fp1 = compute_layout_fingerprint(pos1, [128])
    fp2 = compute_layout_fingerprint(pos2, [128])

    assert fp1 != fp2, "Layout change must produce different fingerprint"


def test_layout_fingerprint_ignores_non_chart():
    """T041: Only chart entries contribute to fingerprint."""
    pos = {"TAB-1": {"meta": {}}, "CHART-128": {"meta": {"chartId": 128, "width": 6}}}

    fp1 = compute_layout_fingerprint(pos, [128])
    fp2 = compute_layout_fingerprint(pos, [128])

    assert fp1 == fp2  # deterministic, non-chart entries ignored
# #endregion Test.DashboardTesting.VisualBaseline.LayoutFingerprint


# ═══════════════════════════════════════════════════════════════════════════════
# SSIM (compute_ssim)
# ═══════════════════════════════════════════════════════════════════════════════


# #region Test.DashboardTesting.VisualBaseline.SSIM [C:3] [TYPE Function] [SEMANTICS testing,ssim,numpy,comparison]
def test_ssim_identical_images():
    """SSIM=1.0 for identical arrays."""
    a = np.full((100, 100), 128, dtype=np.uint8)
    ssim = compute_ssim(a, a)
    assert ssim == 1.0


def test_ssim_different_images():
    """SSIM near 0 for very different images."""
    black = np.zeros((100, 100), dtype=np.uint8)
    white = np.full((100, 100), 255, dtype=np.uint8)
    ssim = compute_ssim(black, white)
    assert ssim < 0.01  # Effectively no structural similarity


def test_ssim_near_identical():
    """SSIM close to 1.0 for single-pixel difference."""
    base = np.full((100, 100), 128, dtype=np.uint8)
    modified = base.copy()
    modified[0, 0] = 129
    ssim = compute_ssim(base, modified)
    assert ssim > 0.99  # Single pixel change has minimal SSIM impact


def test_ssim_shape_mismatch_raises():
    """SSIM raises ValueError for different shapes."""
    a = np.zeros((10, 10), dtype=np.uint8)
    b = np.zeros((20, 20), dtype=np.uint8)
    with pytest.raises(ValueError, match="shape mismatch"):
        compute_ssim(a, b)


def test_ssim_non_uint8_raises():
    """SSIM raises ValueError for non-uint8 dtype."""
    a = np.zeros((10, 10), dtype=np.float64)
    b = np.zeros((10, 10), dtype=np.float64)
    with pytest.raises(ValueError, match="uint8"):
        compute_ssim(a, b)


def test_ssim_3d_raises():
    """SSIM raises ValueError for 3D (color) arrays."""
    a = np.zeros((10, 10, 3), dtype=np.uint8)
    b = np.zeros((10, 10, 3), dtype=np.uint8)
    with pytest.raises(ValueError, match="2D"):
        compute_ssim(a, b)


def test_ssim_all_zero_produces_1():
    """SSIM returns 1.0 when both images are uniform and identical."""
    black = np.zeros((50, 50), dtype=np.uint8)
    ssim = compute_ssim(black, black)
    assert ssim == 1.0


def test_ssim_checkerboard_vs_gradient():
    """SSIM between clearly different patterns is well below threshold."""
    w, h = 64, 64
    check = np.zeros((h, w), dtype=np.uint8)
    for y in range(h):
        for x in range(w):
            check[y, x] = 0 if ((x // 8) + (y // 8)) % 2 == 0 else 255

    grad = np.zeros((h, w), dtype=np.uint8)
    for x in range(w):
        grad[:, x] = round((x / (w - 1)) * 255)

    ssim = compute_ssim(check, grad)
    assert ssim < 0.5  # Very different patterns
# #endregion Test.DashboardTesting.VisualBaseline.SSIM


# ═══════════════════════════════════════════════════════════════════════════════
# Exact visual comparison
# ═══════════════════════════════════════════════════════════════════════════════


# #region Test.DashboardTesting.VisualBaseline.ExactComparison [C:3] [TYPE Function] [SEMANTICS testing,exact,hash]
def test_visual_exact_pass():
    """T043: Exact visual comparison — matching hashes pass."""
    status, diff = compare_visual_exact("abc123abc123", "abc123abc123")
    assert status == ComparisonStatus.PASS
    assert len(diff) == 0


def test_visual_exact_fail():
    """T043: Exact visual comparison — mismatched hashes fail."""
    status, diff = compare_visual_exact("abc123", "def456")
    assert status == ComparisonStatus.FAIL
    assert len(diff) > 0


def test_visual_exact_with_image_fixtures():
    """Exact comparison with real PNG images — matching images pass."""
    img1 = _make_image_fixture(32, 32, 128)
    h1 = hashlib.sha256(img1).hexdigest()
    img2 = _make_image_fixture(32, 32, 128)
    h2 = hashlib.sha256(img2).hexdigest()

    # Same pixel data => same bytes => same hash => pass
    status, _diff = compare_visual_exact(h1, h2)
    assert status == ComparisonStatus.PASS


def test_visual_exact_different_image_fixtures():
    """Exact comparison with different images — different images fail."""
    img1 = _make_image_fixture(32, 32, 128)
    img2 = _make_image_fixture(32, 32, 200)
    h1 = hashlib.sha256(img1).hexdigest()
    h2 = hashlib.sha256(img2).hexdigest()

    status, diff = compare_visual_exact(h1, h2)
    assert status == ComparisonStatus.FAIL
    assert len(diff) > 0
# #endregion Test.DashboardTesting.VisualBaseline.ExactComparison


# ═══════════════════════════════════════════════════════════════════════════════
# Perceptual comparison (SSIM with image bytes)
# ═══════════════════════════════════════════════════════════════════════════════


# #region Test.DashboardTesting.VisualBaseline.PerceptualComparison [C:3] [TYPE Function] [SEMANTICS testing,ssim,perceptual,image-bytes]
def test_perceptual_pass_identical_images():
    """SSIM perceptual: identical image bytes => PASS."""
    img = _make_image_fixture(64, 64, 128)
    status, diff = compare_visual_perceptual(img, img, ssim_min=0.95)
    assert status == ComparisonStatus.PASS
    assert len(diff) == 0


def test_perceptual_pass_near_identical():
    """SSIM perceptual: single-pixel change above threshold => PASS."""
    base = _make_image_fixture(64, 64, 128)
    modified = _make_modified(base, 0, 0, 129)
    status, _diff = compare_visual_perceptual(base, modified, ssim_min=0.95)
    assert status == ComparisonStatus.PASS


def test_perceptual_fail_clearly_different():
    """SSIM perceptual: checkerboard vs gradient => FAIL (< default 0.95)."""
    check_img = _make_checkerboard(64, 64)
    grad_img = _make_gradient(64, 64)
    status, diff = compare_visual_perceptual(check_img, grad_img, ssim_min=0.95)
    assert status == ComparisonStatus.FAIL
    assert len(diff) > 0
    assert diff[0].field == "visual_perceptual_ssim"


def test_perceptual_pass_with_custom_threshold():
    """SSIM perceptual: sufficiently low threshold makes different images pass."""
    check_img = _make_checkerboard(64, 64)
    grad_img = _make_gradient(64, 64)
    # These are very different; SSIM ≈ 0.003.
    # threshold=0.001 (below actual SSIM) should pass.
    status, _diff = compare_visual_perceptual(check_img, grad_img, ssim_min=0.001)
    assert status == ComparisonStatus.PASS


def test_perceptual_fail_with_high_threshold():
    """SSIM perceptual: even near-identical fails if threshold is 1.0."""
    base = _make_image_fixture(64, 64, 128)
    modified = _make_modified(base, 0, 0, 129)
    status, _diff = compare_visual_perceptual(base, modified, ssim_min=1.0)
    assert status == ComparisonStatus.FAIL


def test_perceptual_ssim_value_correct():
    """SSIM perceptual returns correct SSIM value in DiffDetail.actual."""
    base = _make_image_fixture(64, 64, 128)
    # Stripe pattern is very different from uniform fill
    striped = _make_striped(64, 64)
    status, diff = compare_visual_perceptual(base, striped, ssim_min=0.95)
    assert status == ComparisonStatus.FAIL
    # Actual SSIM value should be a float string < 0.95
    ssim_val = float(diff[0].actual)
    assert ssim_val < 0.95
    assert ssim_val >= 0.0


def test_perceptual_size_mismatch_fails():
    """SSIM perceptual: different size images produce inconclusive."""
    small = _make_image_fixture(32, 32, 128)
    large = _make_image_fixture(64, 64, 128)
    status, _diff = compare_visual_perceptual(small, large, ssim_min=0.95)
    assert status == ComparisonStatus.INCONCLUSIVE


def test_perceptual_with_hash_fallback():
    """SSIM perceptual: when image data is invalid, uses hash fallback."""
    real_img = _make_image_fixture(32, 32, 128)
    # Corrupt PNG data (not a valid image)
    corrupt_bytes = b"not_a_valid_png_file_data"
    actual_sha = hashlib.sha256(real_img).hexdigest()
    status, _diff = compare_visual_perceptual(
        corrupt_bytes, real_img, ssim_min=0.95,
        actual_image_sha256=actual_sha,
        expected_image_sha256="different_hash",
    )
    assert status == ComparisonStatus.INCONCLUSIVE
# #endregion Test.DashboardTesting.VisualBaseline.PerceptualComparison

# #endregion Test.DashboardTesting.VisualBaseline