ss-tools/backend/tests/services/test_payload_reduction.py

#region TestPayloadReduction [C:3] [TYPE Module] [SEMANTICS test, llm, payload, reduction, token-limit, fallback]
# @BRIEF Tests for FR-056: multi-chunk screenshot payload reduction and Path B fallback signaling.
# @RELATION BINDS_TO -> [LLMClient._estimate_payload_size]
# @RELATION BINDS_TO -> [LLMClient.analyze_dashboard_multimodal]
# @TEST_SCENARIO estimate_below_threshold -> payload estimated at <80% → no reduction
# @TEST_SCENARIO estimate_exceeds_threshold -> payload estimated at >80% → quality reduced to 30
# @TEST_SCENARIO large_text_triggers_reduction -> text-heavy payload exceeds even after reduction
# @TEST_EDGE: missing_images -> empty screenshot_paths → ValueError
# @TEST_EDGE: corrupt_image -> unreadable image → fallback to raw base64
# @TEST_EDGE: zero_sized_images -> empty image file handled gracefully
# @INVARIANT Payload exceeding 80% of model context window triggers image quality reduction (FR-056)

import pytest
from unittest.mock import AsyncMock, patch

from src.plugins.llm_analysis.models import LLMProviderType
from src.plugins.llm_analysis.service import LLMClient


#region test_estimate_payload_size_below_threshold [C:2] [TYPE Function]
# @BRIEF T050: payload estimated at <80% → no reduction needed.
def test_estimate_payload_size_below_threshold():
    """Small payload (1 image, short text) — <80% of context window."""
    estimate = LLMClient._estimate_payload_size(
        image_paths=["shot.png"],
        text_length=500,
        model_context=128000,
    )
    assert estimate["exceeds_limit"] is False
    assert estimate["pct_of_limit"] < 80
    assert estimate["estimated_tokens"] < 128000 * 0.8
#endregion test_estimate_payload_size_below_threshold


#region test_estimate_payload_size_exceeds_threshold [C:2] [TYPE Function]
# @BRIEF T050: many large images exceed 80% → reduction triggered.
# @NOTE 258 tokens/image * 5 multiplier per image * N images + text/4.
#       100 images = 129000 image tokens + 1250 text = ~130k tokens > 128k*0.8
def test_estimate_payload_size_exceeds_threshold():
    """Many images (100) estimate >80% of context window."""
    estimate = LLMClient._estimate_payload_size(
        image_paths=[f"shot_{i}.png" for i in range(100)],
        text_length=5000,
        model_context=128000,
    )
    assert estimate["exceeds_limit"] is True
    assert estimate["pct_of_limit"] > 80
#endregion test_estimate_payload_size_exceeds_threshold


#region test_estimate_payload_size_small_context [C:2] [TYPE Function]
# @BRIEF T050: small context window (older model) still detected.
def test_estimate_payload_size_small_context():
    """Many images exceed 80% on smaller context (32k)."""
    estimate = LLMClient._estimate_payload_size(
        image_paths=[f"shot_{i}.png" for i in range(100)],
        text_length=2000,
        model_context=32000,
    )
    assert estimate["exceeds_limit"] is True
#endregion test_estimate_payload_size_small_context


#region test_reduce_image_quality_reduces_size [C:2] [TYPE Function]
# @BRIEF T050: _reduce_image_quality reduces image bytes at lower quality setting.
def test_reduce_image_quality_reduces_size(tmp_path):
    """JPEG quality=30 produces smaller payload than quality=85."""
    from PIL import Image

    # Create a test image
    img_path = tmp_path / "test_screenshot.png"
    img = Image.new("RGB", (1920, 1080), color="blue")
    img.save(img_path)

    # Encode at high quality
    b64_high, bytes_high = LLMClient._reduce_image_quality(
        str(img_path), max_width=1920, image_quality=85,
    )
    # Encode at low quality
    b64_low, bytes_low = LLMClient._reduce_image_quality(
        str(img_path), max_width=1920, image_quality=30,
    )

    assert bytes_low <= bytes_high
    assert len(b64_low) <= len(b64_high)
#endregion test_reduce_image_quality_reduces_size


#region test_payload_reduction_triggers_fallback [C:2] [TYPE Function]
# @BRIEF T050: Screenshots exceed 80% context → quality reduction triggered.
#       If still exceeded after reduction, Path B fallback should be signaled.
@pytest.mark.anyio
async def test_payload_reduction_triggers_fallback(tmp_path):
    """Multi-chunk screenshots exceeding 80% → quality reduction to 30, then send.

    The multimodal method reduces quality when >80% of context window.
    After reduction, if still exceeded, it sends anyway (no auto-fallback).
    Fallback to Path B is orchestration-level, not within this method.
    """
    from PIL import Image

    # Create several large test screenshots
    screenshot_paths = []
    for i in range(8):
        img_path = tmp_path / f"tab_{i}.png"
        img = Image.new("RGB", (1920, 1080), color=f"hsl({i * 45}, 100%, 50%)")
        img.save(img_path, "PNG")
        screenshot_paths.append(str(img_path))

    client = LLMClient(
        provider_type=LLMProviderType.LITELLM,
        api_key="sk-test-reduction-key",
        base_url="http://localhost:4000/v1",
        default_model="gpt-4o",
    )
    # Mock the LLM call to return a canned response
    client.get_json_completion = AsyncMock(return_value={
        "status": "PASS",
        "summary": "Reduction test",
        "issues": [],
    })

    # Mock _reduce_image_quality to track quality parameter
    original_reduce = LLMClient._reduce_image_quality
    quality_calls = []

    def _tracking_reduce(path, max_width=1024, image_quality=60):
        quality_calls.append(image_quality)
        return original_reduce(path, max_width, image_quality)

    with (
        patch.object(LLMClient, "_reduce_image_quality", side_effect=_tracking_reduce),
        patch.object(LLMClient, "_estimate_payload_size", return_value={
            "estimated_tokens": 200000,
            "exceeds_limit": True,
            "pct_of_limit": 156.0,
        }),
    ):
        result = await client.analyze_dashboard_multimodal(
            screenshot_paths=screenshot_paths,
            logs=["Session started", "Data loaded"],
            prompt_template="Analyze this dashboard:\n{{ logs }}",
        )

    # Verify reduction was triggered (quality went from 60 to 30)
    # First call for each image is with default quality=60 (for estimate)
    # Then all images are re-encoded at quality=30
    assert any(q == 60 for q in quality_calls), "Initial encode happened"
    assert 30 in quality_calls, "Quality reduction to 30 was applied"

    # Verify the analysis still returned a valid result
    assert result["status"] == "PASS"
    client.get_json_completion.assert_called_once()
#endregion test_payload_reduction_triggers_fallback


#region test_payload_reduction_skip_when_small [C:2] [TYPE Function]
# @BRIEF T050: small payload does not trigger reduction.
@pytest.mark.anyio
async def test_payload_reduction_skip_when_small(tmp_path):
    """Single screenshot, short text — no quality reduction."""
    from PIL import Image

    img_path = tmp_path / "single.png"
    img = Image.new("RGB", (800, 600), color="white")
    img.save(img_path)

    client = LLMClient(
        provider_type=LLMProviderType.LITELLM,
        api_key="sk-test-small-key",
        base_url="http://localhost:4000/v1",
        default_model="gpt-4o-mini",
    )
    client.get_json_completion = AsyncMock(return_value={
        "status": "PASS",
        "summary": "Small payload OK",
        "issues": [],
    })

    quality_calls = []
    original_reduce = LLMClient._reduce_image_quality

    def _tracking_reduce(path, max_width=1024, image_quality=60):
        quality_calls.append(image_quality)
        return original_reduce(path, max_width, image_quality)

    with patch.object(LLMClient, "_reduce_image_quality", side_effect=_tracking_reduce):
        result = await client.analyze_dashboard_multimodal(
            screenshot_paths=[str(img_path)],
            logs=["Short log"],
            prompt_template="Analyze:\n{{ logs }}",
        )

    # Only quality=60 should be used (no reduction)
    assert set(quality_calls) == {60}, f"Expected only quality=60, got {set(quality_calls)}"
    assert result["status"] == "PASS"
#endregion test_payload_reduction_skip_when_small

#endregion TestPayloadReduction