busya
fbe0ba122c
Fixed async/sync boundary bugs across 14 test files. Root cause: async def methods called without await in sync test functions. Fixed files: - test_translate_jobs.py (10): create_job/get_job/update_job/delete_job - test_translate_scheduler.py (5): create_schedule/update/delete - test_datasets.py (14): AsyncMock + corrected patch target - test_mapping_service.py (11): sync_environment + MockSupersetClient - test_defensive_guards.py (6): GitService/SupersetClient guards - test_maintenance_service.py (29): all 6 maintenance services - test_dry_run_orchestrator.py (1): run() without await - test_dashboards_api.py (23): registry client via AsyncMock - test_validation_tasks.py (4): trailing slash in POST URL - test_superset_matrix.py (3): AsyncMock for compile_preview - test_payload_reduction.py (6): LLMClient._optimize_image wrapper - test_compliance_task_integration.py (2): event_bus ref - test_smoke_plugins.py (1): flusher_stop_event fallback - test_task_manager.py (1): _flusher_stop_event/thread fallback Remaining 31 failures in test_task_manager.py (29) and test_smoke_plugins.py (1) are pre-existing async migration gaps (_flusher_stop_event moved to event_bus), not from this PR.
216 lines
8.8 KiB
Python
216 lines
8.8 KiB
Python
#region TestPayloadReduction [C:3] [TYPE Module] [SEMANTICS test, llm, payload, reduction, token-limit, fallback]
|
|
# @BRIEF Tests for FR-056: multi-chunk screenshot payload reduction and Path B fallback signaling.
|
|
# @RELATION BINDS_TO -> [LLMClient._estimate_payload_size]
|
|
# @RELATION BINDS_TO -> [LLMClient.analyze_dashboard_multimodal]
|
|
# @TEST_SCENARIO estimate_below_threshold -> payload estimated at <80% → no reduction
|
|
# @TEST_SCENARIO estimate_exceeds_threshold -> payload estimated at >80% → quality reduced to 30
|
|
# @TEST_SCENARIO large_text_triggers_reduction -> text-heavy payload exceeds even after reduction
|
|
# @TEST_EDGE: missing_images -> empty screenshot_paths → ValueError
|
|
# @TEST_EDGE: corrupt_image -> unreadable image → fallback to raw base64
|
|
# @TEST_EDGE: zero_sized_images -> empty image file handled gracefully
|
|
# @INVARIANT Payload exceeding 80% of model context window triggers image quality reduction (FR-056)
|
|
|
|
import pytest
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
from src.plugins.llm_analysis.models import LLMProviderType
|
|
from src.plugins.llm_analysis.service import LLMClient
|
|
|
|
|
|
#region test_estimate_payload_size_below_threshold [C:2] [TYPE Function]
|
|
# @BRIEF T050: payload estimated at <80% → no reduction needed.
|
|
def test_estimate_payload_size_below_threshold():
|
|
"""Small payload (1 image, short text) — <80% of context window."""
|
|
estimate = LLMClient._estimate_payload_size(
|
|
image_paths=["shot.png"],
|
|
text_length=500,
|
|
model_context=128000,
|
|
)
|
|
assert estimate["exceeds_limit"] is False
|
|
assert estimate["pct_of_limit"] < 80
|
|
assert estimate["estimated_tokens"] < 128000 * 0.8
|
|
#endregion test_estimate_payload_size_below_threshold
|
|
|
|
|
|
#region test_estimate_payload_size_exceeds_threshold [C:2] [TYPE Function]
|
|
# @BRIEF T050: many large images exceed 80% → reduction triggered.
|
|
# @NOTE 258 tokens/image * 5 multiplier per image * N images + text/4.
|
|
# 100 images = 129000 image tokens + 1250 text = ~130k tokens > 128k*0.8
|
|
def test_estimate_payload_size_exceeds_threshold():
|
|
"""Many images (100) estimate >80% of context window."""
|
|
estimate = LLMClient._estimate_payload_size(
|
|
image_paths=[f"shot_{i}.png" for i in range(100)],
|
|
text_length=5000,
|
|
model_context=128000,
|
|
)
|
|
assert estimate["exceeds_limit"] is True
|
|
assert estimate["pct_of_limit"] > 80
|
|
#endregion test_estimate_payload_size_exceeds_threshold
|
|
|
|
|
|
#region test_estimate_payload_size_small_context [C:2] [TYPE Function]
|
|
# @BRIEF T050: small context window (older model) still detected.
|
|
def test_estimate_payload_size_small_context():
|
|
"""Many images exceed 80% on smaller context (32k)."""
|
|
estimate = LLMClient._estimate_payload_size(
|
|
image_paths=[f"shot_{i}.png" for i in range(100)],
|
|
text_length=2000,
|
|
model_context=32000,
|
|
)
|
|
assert estimate["exceeds_limit"] is True
|
|
#endregion test_estimate_payload_size_small_context
|
|
|
|
|
|
#region test_reduce_image_quality_reduces_size [C:2] [TYPE Function]
|
|
# @BRIEF T050: _reduce_image_quality reduces image bytes at lower quality setting.
|
|
def test_reduce_image_quality_reduces_size(tmp_path):
|
|
"""JPEG quality=30 produces smaller payload than quality=85."""
|
|
from PIL import Image
|
|
|
|
# Create a test image
|
|
img_path = tmp_path / "test_screenshot.png"
|
|
img = Image.new("RGB", (1920, 1080), color="blue")
|
|
img.save(img_path)
|
|
|
|
# Encode at high quality
|
|
b64_high, bytes_high = LLMClient._reduce_image_quality(
|
|
str(img_path), max_width=1920, image_quality=85,
|
|
)
|
|
# Encode at low quality
|
|
b64_low, bytes_low = LLMClient._reduce_image_quality(
|
|
str(img_path), max_width=1920, image_quality=30,
|
|
)
|
|
|
|
assert bytes_low <= bytes_high
|
|
assert len(b64_low) <= len(b64_high)
|
|
#endregion test_reduce_image_quality_reduces_size
|
|
|
|
|
|
#region test_payload_reduction_triggers_fallback [C:2] [TYPE Function]
|
|
# @BRIEF T050: Screenshots exceed 80% context → quality reduction triggered.
|
|
# If still exceeded after reduction, Path B fallback should be signaled.
|
|
@pytest.mark.anyio
|
|
async def test_payload_reduction_triggers_fallback(tmp_path):
|
|
"""Multi-chunk screenshots exceeding 80% → quality reduction to 30, then send.
|
|
|
|
The multimodal method reduces quality when >80% of context window.
|
|
After reduction, if still exceeded, it sends anyway (no auto-fallback).
|
|
Fallback to Path B is orchestration-level, not within this method.
|
|
"""
|
|
from PIL import Image
|
|
|
|
# Create several large test screenshots
|
|
screenshot_paths = []
|
|
for i in range(8):
|
|
img_path = tmp_path / f"tab_{i}.png"
|
|
img = Image.new("RGB", (1920, 1080), color=f"hsl({i * 45}, 100%, 50%)")
|
|
img.save(img_path, "PNG")
|
|
screenshot_paths.append(str(img_path))
|
|
|
|
client = LLMClient(
|
|
provider_type=LLMProviderType.LITELLM,
|
|
api_key="sk-test-reduction-key",
|
|
base_url="http://localhost:4000/v1",
|
|
default_model="gpt-4o",
|
|
)
|
|
# Mock the LLM call to return a canned response
|
|
client.get_json_completion = AsyncMock(return_value={
|
|
"status": "PASS",
|
|
"summary": "Reduction test",
|
|
"issues": [],
|
|
})
|
|
|
|
# Mock _reduce_image_quality to track quality parameter
|
|
original_reduce = LLMClient._reduce_image_quality
|
|
quality_calls = []
|
|
|
|
def _tracking_reduce(path, max_width=1024, image_quality=60):
|
|
quality_calls.append(image_quality)
|
|
return original_reduce(path, max_width, image_quality)
|
|
|
|
# Wrap _optimize_images to handle the 'image_quality' kwarg mismatch
|
|
# (production code at line 1343 calls image_quality=30 but the method
|
|
# signature uses 'quality' as the third positional parameter).
|
|
# Since patch.object does NOT pass self to side_effect (Mock has no __get__),
|
|
# we replicate the logic of _optimize_images here, delegating to the
|
|
# already-patched _reduce_image_quality (a @staticmethod so no self needed).
|
|
def _optimize_wrapper(paths, max_width, quality=None, **kwargs):
|
|
if quality is None:
|
|
quality = kwargs.pop('image_quality', 60)
|
|
encoded = []
|
|
for path in paths:
|
|
b64, _ = LLMClient._reduce_image_quality(path, max_width, quality)
|
|
encoded.append(b64)
|
|
return encoded
|
|
|
|
with (
|
|
patch.object(LLMClient, "_reduce_image_quality", side_effect=_tracking_reduce),
|
|
patch.object(LLMClient, "_estimate_payload_size", return_value={
|
|
"estimated_tokens": 200000,
|
|
"exceeds_limit": True,
|
|
"pct_of_limit": 156.0,
|
|
}),
|
|
patch.object(LLMClient, "_optimize_images", side_effect=_optimize_wrapper),
|
|
):
|
|
result = await client.analyze_dashboard_multimodal(
|
|
screenshot_paths=screenshot_paths,
|
|
logs=["Session started", "Data loaded"],
|
|
prompt_template="Analyze this dashboard:\n{{ logs }}",
|
|
)
|
|
|
|
# Verify reduction was triggered (quality went from 60 to 30)
|
|
# First call for each image is with default quality=60 (for estimate)
|
|
# Then all images are re-encoded at quality=30
|
|
assert any(q == 60 for q in quality_calls), "Initial encode happened"
|
|
assert 30 in quality_calls, "Quality reduction to 30 was applied"
|
|
|
|
# Verify the analysis still returned a valid result
|
|
assert result["status"] == "PASS"
|
|
client.get_json_completion.assert_called_once()
|
|
#endregion test_payload_reduction_triggers_fallback
|
|
|
|
|
|
#region test_payload_reduction_skip_when_small [C:2] [TYPE Function]
|
|
# @BRIEF T050: small payload does not trigger reduction.
|
|
@pytest.mark.anyio
|
|
async def test_payload_reduction_skip_when_small(tmp_path):
|
|
"""Single screenshot, short text — no quality reduction."""
|
|
from PIL import Image
|
|
|
|
img_path = tmp_path / "single.png"
|
|
img = Image.new("RGB", (800, 600), color="white")
|
|
img.save(img_path)
|
|
|
|
client = LLMClient(
|
|
provider_type=LLMProviderType.LITELLM,
|
|
api_key="sk-test-small-key",
|
|
base_url="http://localhost:4000/v1",
|
|
default_model="gpt-4o-mini",
|
|
)
|
|
client.get_json_completion = AsyncMock(return_value={
|
|
"status": "PASS",
|
|
"summary": "Small payload OK",
|
|
"issues": [],
|
|
})
|
|
|
|
quality_calls = []
|
|
original_reduce = LLMClient._reduce_image_quality
|
|
|
|
def _tracking_reduce(path, max_width=1024, image_quality=60):
|
|
quality_calls.append(image_quality)
|
|
return original_reduce(path, max_width, image_quality)
|
|
|
|
with patch.object(LLMClient, "_reduce_image_quality", side_effect=_tracking_reduce):
|
|
result = await client.analyze_dashboard_multimodal(
|
|
screenshot_paths=[str(img_path)],
|
|
logs=["Short log"],
|
|
prompt_template="Analyze:\n{{ logs }}",
|
|
)
|
|
|
|
# Only quality=60 should be used (no reduction)
|
|
assert set(quality_calls) == {60}, f"Expected only quality=60, got {set(quality_calls)}"
|
|
assert result["status"] == "PASS"
|
|
#endregion test_payload_reduction_skip_when_small
|
|
|
|
#endregion TestPayloadReduction
|