feat(validation): chunk screenshots by max_images limit + fix websocket crash

- analyze_dashboard_multimodal now splits screenshots into chunks
  of max_images (from provider config) and sends them in parallel
- Results merged: worst status, deduped issues by (severity, msg, loc)
- New helper methods: _deduplicate_issues, _merge_chunk_results, _call_llm_for_images
- Plugin passes db_provider.max_images to the LLM client
- Report UI shows 'Chunked ×N' badge when analysis used multiple chunks
- i18n: added 'chunked' / 'По частям' key to validation.json
- Fix: isinstance(StopIteration) -> isinstance(_ws_exc, StopIteration)
  which crashed the websocket and broke task execution mid-flight
- Fix: update test mocks (_FakeLLMClient, _FakeScreenshotService)
This commit is contained in:
2026-05-31 22:43:06 +03:00
parent 9b938fcb81
commit 2760fa09ea
7 changed files with 134 additions and 18 deletions

View File

@@ -634,8 +634,8 @@ async def websocket_endpoint(
extra={"task_id": task_id, "message": result.message},
)
await asyncio.sleep(2)
except (WebSocketDisconnect, StopIteration):
if isinstance(StopIteration):
except (WebSocketDisconnect, StopIteration) as _ws_exc:
if isinstance(_ws_exc, StopIteration):
# Task reached terminal state — close cleanly with code 1000
try:
await websocket.close(code=1000, reason="Task completed")

View File

@@ -371,6 +371,7 @@ class DashboardValidationPlugin(PluginBase):
screenshot_paths=jpeg_paths,
logs=logs,
prompt_template=dashboard_prompt,
max_images=db_provider.max_images,
)
else:
# Fallback: text-only analysis if no screenshots
@@ -418,6 +419,7 @@ class DashboardValidationPlugin(PluginBase):
result_payload = _json_safe_value(validation_result.model_dump())
result_payload["execution_path"] = "multimodal"
result_payload["chunk_count"] = analysis.get("chunk_count") or analysis.get("chunks") or 1
result_payload["screenshot_paths"] = webp_paths or jpeg_paths
result_payload["logs_sent_to_llm"] = logs
result_payload["logs_sent_count"] = len(logs)

View File

@@ -1193,13 +1193,71 @@ class LLMClient:
}
# endregion LLMClient._estimate_payload_size
# region LLMClient._deduplicate_issues [TYPE Function] [C:2]
# @PURPOSE Deduplicate issues by (severity, message, location) while preserving order.
def _deduplicate_issues(self, issues: list[dict]) -> list[dict]:
seen: set[tuple[str, str, str]] = set()
result: list[dict] = []
for issue in issues:
key = (issue.get("severity", ""), issue.get("message", ""), issue.get("location", "") or "")
if key not in seen:
seen.add(key)
result.append(issue)
return result
# endregion LLMClient._deduplicate_issues
# region LLMClient._merge_chunk_results [TYPE Function] [C:2]
# @PURPOSE Merge multiple chunk analyses into one. Takes the worst status,
# concatenates summaries, and deduplicates issues.
# @PRE chunks is a non-empty list of {status, summary, issues} dicts.
# @POST Returns a single merged dict.
def _merge_chunk_results(self, chunks: list[dict[str, Any]]) -> dict[str, Any]:
STATUS_ORDER = {"FAIL": 0, "WARN": 1, "PASS": 2, "UNKNOWN": 3}
worst_status = "UNKNOWN"
all_summaries: list[str] = []
all_issues: list[dict] = []
for i, chunk in enumerate(chunks):
s = chunk.get("status", "UNKNOWN")
if STATUS_ORDER.get(s, 3) < STATUS_ORDER.get(worst_status, 3):
worst_status = s
all_summaries.append(f"[Chunk {i + 1}/{len(chunks)}] {chunk.get('summary', 'No summary')}")
all_issues.extend(chunk.get("issues", []))
merged = {
"status": worst_status,
"summary": " | ".join(all_summaries),
"issues": self._deduplicate_issues(all_issues),
"chunks": len(chunks),
}
return merged
# endregion LLMClient._merge_chunk_results
# region LLMClient._call_llm_for_images [TYPE Function] [C:2]
# @PURPOSE Send a single chunk of images to the LLM and return parsed result.
async def _call_llm_for_images(
self, encoded_images: list[str], prompt: str
) -> dict[str, Any]:
content: list[dict] = [{"type": "text", "text": prompt}]
for b64_img in encoded_images:
content.append({
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{b64_img}"},
})
messages = [{"role": "user", "content": content}]
return await self.get_json_completion(messages)
# endregion LLMClient._call_llm_for_images
# region LLMClient.analyze_dashboard_multimodal [TYPE Function] [C:3]
# @PURPOSE Path A: send multiple tab screenshots + logs to multimodal LLM.
# @PURPOSE Path A: send screenshots + logs to multimodal LLM, with chunking support.
# @PRE screenshot_paths is a non-empty list of paths.
# @POST Returns dict {status, summary, issues}.
# @SIDE_EFFECT Compresses images, calls external LLM API.
# @RATIONALE Multi-chunk: one screenshot per tab. All images sent in single content[] array.
# Token budget estimated before send; quality reduced if >80% of model context window.
# @SIDE_EFFECT Compresses images, calls external LLM API (possibly multiple times for chunks).
# @RATIONALE Screenshots are split into chunks of max_images to respect provider image limits.
# Each chunk is sent in parallel; results are merged via _merge_chunk_results.
async def analyze_dashboard_multimodal(
self,
screenshot_paths: list[str],
@@ -1207,6 +1265,7 @@ class LLMClient:
prompt_template: str = DEFAULT_LLM_PROMPTS["dashboard_validation_prompt"],
max_width: int = 1024,
image_quality: int = 60,
max_images: int | None = None,
) -> dict[str, Any]:
with belief_scope("analyze_dashboard_multimodal"):
if not screenshot_paths:
@@ -1248,19 +1307,45 @@ class LLMClient:
b64 = base64.b64encode(f.read()).decode("utf-8")
encoded_images.append(b64)
# 3. Build multimodal content array with all images
content: list[dict] = [{"type": "text", "text": prompt}]
for b64_img in encoded_images:
content.append({
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{b64_img}"},
})
# 3. Chunk images if max_images is set
n_total = len(encoded_images)
chunk_size = max_images if (max_images and max_images > 0 and max_images < n_total) else n_total
messages = [{"role": "user", "content": content}]
if chunk_size < n_total:
logger.reason(
f"[analyze_dashboard_multimodal] Chunking {n_total} images into "
f"{ (n_total + chunk_size - 1) // chunk_size } chunks of {chunk_size}",
extra={"src": "analyze_dashboard_multimodal", "total": n_total, "chunk_size": chunk_size},
)
# 4. Call LLM
# Split into chunks
chunks: list[list[str]] = []
for i in range(0, n_total, chunk_size):
chunks.append(encoded_images[i:i + chunk_size])
# 4. Call LLM — parallel for multiple chunks, single for one
try:
return await self.get_json_completion(messages)
if len(chunks) == 1:
result = await self._call_llm_for_images(chunks[0], prompt)
else:
tasks = [self._call_llm_for_images(chunk, prompt) for chunk in chunks]
chunk_results = await asyncio.gather(*tasks, return_exceptions=True)
# Filter out exceptions
valid_results: list[dict] = []
for i, cr in enumerate(chunk_results):
if isinstance(cr, Exception):
logger.error(f"[analyze_dashboard_multimodal] Chunk {i + 1}/{len(chunks)} failed: {cr!s}")
valid_results.append({
"status": "UNKNOWN",
"summary": f"Chunk {i + 1} failed: {cr!s}",
"issues": [],
})
else:
valid_results.append(cr)
result = self._merge_chunk_results(valid_results)
result["chunk_count"] = len(chunks)
except Exception as e:
logger.error(f"[analyze_dashboard_multimodal] Failed to get analysis: {e!s}")
return {
@@ -1268,6 +1353,8 @@ class LLMClient:
"summary": f"Failed to get response from LLM: {e!s}",
"issues": [{"severity": "UNKNOWN", "message": "LLM provider returned empty or invalid response"}],
}
return result
# endregion LLMClient.analyze_dashboard_multimodal
# region LLMClient.analyze_dashboard_text_batch [TYPE Function] [C:3]

View File

@@ -104,6 +104,10 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
async def capture_dashboard(self, _dashboard_id, _screenshot_path):
return [], []
@staticmethod
def _cleanup_temp_files(_paths):
pass
# endregion _FakeScreenshotService
# region _FakeLLMClient [TYPE Class]
@@ -127,6 +131,22 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
"issues": [],
}
async def analyze_dashboard_multimodal(self, *_args, **_kwargs):
return {
"status": "PASS",
"summary": "Dashboard healthy",
"issues": [],
}
async def analyze_dashboard_text_batch(self, *_args, **_kwargs):
return {
"dashboards": [{
"status": "PASS",
"summary": "Dashboard healthy",
"issues": [],
}],
}
# endregion _FakeLLMClient
# region _FakeNotificationService [TYPE Class]

View File

@@ -212,5 +212,6 @@
"path_b": "Path B — Text-only",
"no_issues": "No issues detected",
"no_logs": "No logs available",
"no_screenshots": "No screenshots saved"
"no_screenshots": "No screenshots saved",
"chunked": "Chunked"
}

View File

@@ -212,5 +212,6 @@
"path_b": "Путь B — Только текст",
"no_issues": "Проблем не обнаружено",
"no_logs": "Нет доступных логов",
"no_screenshots": "Нет сохраненных скриншотов"
"no_screenshots": "Нет сохраненных скриншотов",
"chunked": "По частям"
}

View File

@@ -414,6 +414,11 @@
{#if dbPath === 'A'}
<span class="text-purple-600 font-medium">📸 {$t.validation?.path_a || 'Path A — Screenshot'}</span>
<span class="text-gray-400">| {$t.validation?.screenshots || 'Screenshots'} captured via browser automation</span>
{#if dashboard.chunk_count && dashboard.chunk_count > 1}
<span class="inline-flex items-center gap-1 px-2 py-0.5 rounded-full bg-amber-100 text-amber-700 text-xs font-medium">
{$t.validation?.chunked || 'Chunked'} ×{dashboard.chunk_count}
</span>
{/if}
{:else}
<span class="text-teal-600 font-medium">📝 {$t.validation?.path_b || 'Path B — Text-only'}</span>
<span class="text-gray-400">| {$t.validation?.issues || 'Issue'} detection via DOM extraction</span>