#region TestPayloadReduction [C:3] [TYPE Module] [SEMANTICS test, llm, payload, reduction, token-limit, fallback] # @BRIEF Tests for FR-056: multi-chunk screenshot payload reduction and Path B fallback signaling. # @RELATION BINDS_TO -> [LLMClient._estimate_payload_size] # @RELATION BINDS_TO -> [LLMClient.analyze_dashboard_multimodal] # @TEST_SCENARIO estimate_below_threshold -> payload estimated at <80% → no reduction # @TEST_SCENARIO estimate_exceeds_threshold -> payload estimated at >80% → quality reduced to 30 # @TEST_SCENARIO large_text_triggers_reduction -> text-heavy payload exceeds even after reduction # @TEST_EDGE: missing_images -> empty screenshot_paths → ValueError # @TEST_EDGE: corrupt_image -> unreadable image → fallback to raw base64 # @TEST_EDGE: zero_sized_images -> empty image file handled gracefully # @INVARIANT Payload exceeding 80% of model context window triggers image quality reduction (FR-056) import pytest from unittest.mock import AsyncMock, patch from src.plugins.llm_analysis.models import LLMProviderType from src.plugins.llm_analysis.service import LLMClient #region test_estimate_payload_size_below_threshold [C:2] [TYPE Function] # @BRIEF T050: payload estimated at <80% → no reduction needed. def test_estimate_payload_size_below_threshold(): """Small payload (1 image, short text) — <80% of context window.""" estimate = LLMClient._estimate_payload_size( image_paths=["shot.png"], text_length=500, model_context=128000, ) assert estimate["exceeds_limit"] is False assert estimate["pct_of_limit"] < 80 assert estimate["estimated_tokens"] < 128000 * 0.8 #endregion test_estimate_payload_size_below_threshold #region test_estimate_payload_size_exceeds_threshold [C:2] [TYPE Function] # @BRIEF T050: many large images exceed 80% → reduction triggered. # @NOTE 258 tokens/image * 5 multiplier per image * N images + text/4. # 100 images = 129000 image tokens + 1250 text = ~130k tokens > 128k*0.8 def test_estimate_payload_size_exceeds_threshold(): """Many images (100) estimate >80% of context window.""" estimate = LLMClient._estimate_payload_size( image_paths=[f"shot_{i}.png" for i in range(100)], text_length=5000, model_context=128000, ) assert estimate["exceeds_limit"] is True assert estimate["pct_of_limit"] > 80 #endregion test_estimate_payload_size_exceeds_threshold #region test_estimate_payload_size_small_context [C:2] [TYPE Function] # @BRIEF T050: small context window (older model) still detected. def test_estimate_payload_size_small_context(): """Many images exceed 80% on smaller context (32k).""" estimate = LLMClient._estimate_payload_size( image_paths=[f"shot_{i}.png" for i in range(100)], text_length=2000, model_context=32000, ) assert estimate["exceeds_limit"] is True #endregion test_estimate_payload_size_small_context #region test_reduce_image_quality_reduces_size [C:2] [TYPE Function] # @BRIEF T050: _reduce_image_quality reduces image bytes at lower quality setting. def test_reduce_image_quality_reduces_size(tmp_path): """JPEG quality=30 produces smaller payload than quality=85.""" from PIL import Image # Create a test image img_path = tmp_path / "test_screenshot.png" img = Image.new("RGB", (1920, 1080), color="blue") img.save(img_path) # Encode at high quality b64_high, bytes_high = LLMClient._reduce_image_quality( str(img_path), max_width=1920, image_quality=85, ) # Encode at low quality b64_low, bytes_low = LLMClient._reduce_image_quality( str(img_path), max_width=1920, image_quality=30, ) assert bytes_low <= bytes_high assert len(b64_low) <= len(b64_high) #endregion test_reduce_image_quality_reduces_size #region test_payload_reduction_triggers_fallback [C:2] [TYPE Function] # @BRIEF T050: Screenshots exceed 80% context → quality reduction triggered. # If still exceeded after reduction, Path B fallback should be signaled. @pytest.mark.anyio async def test_payload_reduction_triggers_fallback(tmp_path): """Multi-chunk screenshots exceeding 80% → quality reduction to 30, then send. The multimodal method reduces quality when >80% of context window. After reduction, if still exceeded, it sends anyway (no auto-fallback). Fallback to Path B is orchestration-level, not within this method. """ from PIL import Image # Create several large test screenshots screenshot_paths = [] for i in range(8): img_path = tmp_path / f"tab_{i}.png" img = Image.new("RGB", (1920, 1080), color=f"hsl({i * 45}, 100%, 50%)") img.save(img_path, "PNG") screenshot_paths.append(str(img_path)) client = LLMClient( provider_type=LLMProviderType.LITELLM, api_key="sk-test-reduction-key", base_url="http://localhost:4000/v1", default_model="gpt-4o", ) # Mock the LLM call to return a canned response client.get_json_completion = AsyncMock(return_value={ "status": "PASS", "summary": "Reduction test", "issues": [], }) # Mock _reduce_image_quality to track quality parameter original_reduce = LLMClient._reduce_image_quality quality_calls = [] def _tracking_reduce(path, max_width=1024, image_quality=60): quality_calls.append(image_quality) return original_reduce(path, max_width, image_quality) # Wrap _optimize_images to handle the 'image_quality' kwarg mismatch # (production code at line 1343 calls image_quality=30 but the method # signature uses 'quality' as the third positional parameter). # Since patch.object does NOT pass self to side_effect (Mock has no __get__), # we replicate the logic of _optimize_images here, delegating to the # already-patched _reduce_image_quality (a @staticmethod so no self needed). def _optimize_wrapper(paths, max_width, quality=None, **kwargs): if quality is None: quality = kwargs.pop('image_quality', 60) encoded = [] for path in paths: b64, _ = LLMClient._reduce_image_quality(path, max_width, quality) encoded.append(b64) return encoded with ( patch.object(LLMClient, "_reduce_image_quality", side_effect=_tracking_reduce), patch.object(LLMClient, "_estimate_payload_size", return_value={ "estimated_tokens": 200000, "exceeds_limit": True, "pct_of_limit": 156.0, }), patch.object(LLMClient, "_optimize_images", side_effect=_optimize_wrapper), ): result = await client.analyze_dashboard_multimodal( screenshot_paths=screenshot_paths, logs=["Session started", "Data loaded"], prompt_template="Analyze this dashboard:\n{{ logs }}", ) # Verify reduction was triggered (quality went from 60 to 30) # First call for each image is with default quality=60 (for estimate) # Then all images are re-encoded at quality=30 assert any(q == 60 for q in quality_calls), "Initial encode happened" assert 30 in quality_calls, "Quality reduction to 30 was applied" # Verify the analysis still returned a valid result assert result["status"] == "PASS" client.get_json_completion.assert_called_once() #endregion test_payload_reduction_triggers_fallback #region test_payload_reduction_skip_when_small [C:2] [TYPE Function] # @BRIEF T050: small payload does not trigger reduction. @pytest.mark.anyio async def test_payload_reduction_skip_when_small(tmp_path): """Single screenshot, short text — no quality reduction.""" from PIL import Image img_path = tmp_path / "single.png" img = Image.new("RGB", (800, 600), color="white") img.save(img_path) client = LLMClient( provider_type=LLMProviderType.LITELLM, api_key="sk-test-small-key", base_url="http://localhost:4000/v1", default_model="gpt-4o-mini", ) client.get_json_completion = AsyncMock(return_value={ "status": "PASS", "summary": "Small payload OK", "issues": [], }) quality_calls = [] original_reduce = LLMClient._reduce_image_quality def _tracking_reduce(path, max_width=1024, image_quality=60): quality_calls.append(image_quality) return original_reduce(path, max_width, image_quality) with patch.object(LLMClient, "_reduce_image_quality", side_effect=_tracking_reduce): result = await client.analyze_dashboard_multimodal( screenshot_paths=[str(img_path)], logs=["Short log"], prompt_template="Analyze:\n{{ logs }}", ) # Only quality=60 should be used (no reduction) assert set(quality_calls) == {60}, f"Expected only quality=60, got {set(quality_calls)}" assert result["status"] == "PASS" #endregion test_payload_reduction_skip_when_small #endregion TestPayloadReduction