feat(validation): chunk screenshots by max_images limit + fix websocket crash
- analyze_dashboard_multimodal now splits screenshots into chunks of max_images (from provider config) and sends them in parallel - Results merged: worst status, deduped issues by (severity, msg, loc) - New helper methods: _deduplicate_issues, _merge_chunk_results, _call_llm_for_images - Plugin passes db_provider.max_images to the LLM client - Report UI shows 'Chunked ×N' badge when analysis used multiple chunks - i18n: added 'chunked' / 'По частям' key to validation.json - Fix: isinstance(StopIteration) -> isinstance(_ws_exc, StopIteration) which crashed the websocket and broke task execution mid-flight - Fix: update test mocks (_FakeLLMClient, _FakeScreenshotService)
This commit is contained in:
@@ -634,8 +634,8 @@ async def websocket_endpoint(
|
||||
extra={"task_id": task_id, "message": result.message},
|
||||
)
|
||||
await asyncio.sleep(2)
|
||||
except (WebSocketDisconnect, StopIteration):
|
||||
if isinstance(StopIteration):
|
||||
except (WebSocketDisconnect, StopIteration) as _ws_exc:
|
||||
if isinstance(_ws_exc, StopIteration):
|
||||
# Task reached terminal state — close cleanly with code 1000
|
||||
try:
|
||||
await websocket.close(code=1000, reason="Task completed")
|
||||
|
||||
@@ -371,6 +371,7 @@ class DashboardValidationPlugin(PluginBase):
|
||||
screenshot_paths=jpeg_paths,
|
||||
logs=logs,
|
||||
prompt_template=dashboard_prompt,
|
||||
max_images=db_provider.max_images,
|
||||
)
|
||||
else:
|
||||
# Fallback: text-only analysis if no screenshots
|
||||
@@ -418,6 +419,7 @@ class DashboardValidationPlugin(PluginBase):
|
||||
|
||||
result_payload = _json_safe_value(validation_result.model_dump())
|
||||
result_payload["execution_path"] = "multimodal"
|
||||
result_payload["chunk_count"] = analysis.get("chunk_count") or analysis.get("chunks") or 1
|
||||
result_payload["screenshot_paths"] = webp_paths or jpeg_paths
|
||||
result_payload["logs_sent_to_llm"] = logs
|
||||
result_payload["logs_sent_count"] = len(logs)
|
||||
|
||||
@@ -1193,13 +1193,71 @@ class LLMClient:
|
||||
}
|
||||
# endregion LLMClient._estimate_payload_size
|
||||
|
||||
# region LLMClient._deduplicate_issues [TYPE Function] [C:2]
|
||||
# @PURPOSE Deduplicate issues by (severity, message, location) while preserving order.
|
||||
def _deduplicate_issues(self, issues: list[dict]) -> list[dict]:
|
||||
seen: set[tuple[str, str, str]] = set()
|
||||
result: list[dict] = []
|
||||
for issue in issues:
|
||||
key = (issue.get("severity", ""), issue.get("message", ""), issue.get("location", "") or "")
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
result.append(issue)
|
||||
return result
|
||||
|
||||
# endregion LLMClient._deduplicate_issues
|
||||
|
||||
# region LLMClient._merge_chunk_results [TYPE Function] [C:2]
|
||||
# @PURPOSE Merge multiple chunk analyses into one. Takes the worst status,
|
||||
# concatenates summaries, and deduplicates issues.
|
||||
# @PRE chunks is a non-empty list of {status, summary, issues} dicts.
|
||||
# @POST Returns a single merged dict.
|
||||
def _merge_chunk_results(self, chunks: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
STATUS_ORDER = {"FAIL": 0, "WARN": 1, "PASS": 2, "UNKNOWN": 3}
|
||||
worst_status = "UNKNOWN"
|
||||
all_summaries: list[str] = []
|
||||
all_issues: list[dict] = []
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
s = chunk.get("status", "UNKNOWN")
|
||||
if STATUS_ORDER.get(s, 3) < STATUS_ORDER.get(worst_status, 3):
|
||||
worst_status = s
|
||||
all_summaries.append(f"[Chunk {i + 1}/{len(chunks)}] {chunk.get('summary', 'No summary')}")
|
||||
all_issues.extend(chunk.get("issues", []))
|
||||
|
||||
merged = {
|
||||
"status": worst_status,
|
||||
"summary": " | ".join(all_summaries),
|
||||
"issues": self._deduplicate_issues(all_issues),
|
||||
"chunks": len(chunks),
|
||||
}
|
||||
return merged
|
||||
|
||||
# endregion LLMClient._merge_chunk_results
|
||||
|
||||
# region LLMClient._call_llm_for_images [TYPE Function] [C:2]
|
||||
# @PURPOSE Send a single chunk of images to the LLM and return parsed result.
|
||||
async def _call_llm_for_images(
|
||||
self, encoded_images: list[str], prompt: str
|
||||
) -> dict[str, Any]:
|
||||
content: list[dict] = [{"type": "text", "text": prompt}]
|
||||
for b64_img in encoded_images:
|
||||
content.append({
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/jpeg;base64,{b64_img}"},
|
||||
})
|
||||
messages = [{"role": "user", "content": content}]
|
||||
return await self.get_json_completion(messages)
|
||||
|
||||
# endregion LLMClient._call_llm_for_images
|
||||
|
||||
# region LLMClient.analyze_dashboard_multimodal [TYPE Function] [C:3]
|
||||
# @PURPOSE Path A: send multiple tab screenshots + logs to multimodal LLM.
|
||||
# @PURPOSE Path A: send screenshots + logs to multimodal LLM, with chunking support.
|
||||
# @PRE screenshot_paths is a non-empty list of paths.
|
||||
# @POST Returns dict {status, summary, issues}.
|
||||
# @SIDE_EFFECT Compresses images, calls external LLM API.
|
||||
# @RATIONALE Multi-chunk: one screenshot per tab. All images sent in single content[] array.
|
||||
# Token budget estimated before send; quality reduced if >80% of model context window.
|
||||
# @SIDE_EFFECT Compresses images, calls external LLM API (possibly multiple times for chunks).
|
||||
# @RATIONALE Screenshots are split into chunks of max_images to respect provider image limits.
|
||||
# Each chunk is sent in parallel; results are merged via _merge_chunk_results.
|
||||
async def analyze_dashboard_multimodal(
|
||||
self,
|
||||
screenshot_paths: list[str],
|
||||
@@ -1207,6 +1265,7 @@ class LLMClient:
|
||||
prompt_template: str = DEFAULT_LLM_PROMPTS["dashboard_validation_prompt"],
|
||||
max_width: int = 1024,
|
||||
image_quality: int = 60,
|
||||
max_images: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
with belief_scope("analyze_dashboard_multimodal"):
|
||||
if not screenshot_paths:
|
||||
@@ -1248,19 +1307,45 @@ class LLMClient:
|
||||
b64 = base64.b64encode(f.read()).decode("utf-8")
|
||||
encoded_images.append(b64)
|
||||
|
||||
# 3. Build multimodal content array with all images
|
||||
content: list[dict] = [{"type": "text", "text": prompt}]
|
||||
for b64_img in encoded_images:
|
||||
content.append({
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/jpeg;base64,{b64_img}"},
|
||||
})
|
||||
# 3. Chunk images if max_images is set
|
||||
n_total = len(encoded_images)
|
||||
chunk_size = max_images if (max_images and max_images > 0 and max_images < n_total) else n_total
|
||||
|
||||
messages = [{"role": "user", "content": content}]
|
||||
if chunk_size < n_total:
|
||||
logger.reason(
|
||||
f"[analyze_dashboard_multimodal] Chunking {n_total} images into "
|
||||
f"{ (n_total + chunk_size - 1) // chunk_size } chunks of {chunk_size}",
|
||||
extra={"src": "analyze_dashboard_multimodal", "total": n_total, "chunk_size": chunk_size},
|
||||
)
|
||||
|
||||
# 4. Call LLM
|
||||
# Split into chunks
|
||||
chunks: list[list[str]] = []
|
||||
for i in range(0, n_total, chunk_size):
|
||||
chunks.append(encoded_images[i:i + chunk_size])
|
||||
|
||||
# 4. Call LLM — parallel for multiple chunks, single for one
|
||||
try:
|
||||
return await self.get_json_completion(messages)
|
||||
if len(chunks) == 1:
|
||||
result = await self._call_llm_for_images(chunks[0], prompt)
|
||||
else:
|
||||
tasks = [self._call_llm_for_images(chunk, prompt) for chunk in chunks]
|
||||
chunk_results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
# Filter out exceptions
|
||||
valid_results: list[dict] = []
|
||||
for i, cr in enumerate(chunk_results):
|
||||
if isinstance(cr, Exception):
|
||||
logger.error(f"[analyze_dashboard_multimodal] Chunk {i + 1}/{len(chunks)} failed: {cr!s}")
|
||||
valid_results.append({
|
||||
"status": "UNKNOWN",
|
||||
"summary": f"Chunk {i + 1} failed: {cr!s}",
|
||||
"issues": [],
|
||||
})
|
||||
else:
|
||||
valid_results.append(cr)
|
||||
|
||||
result = self._merge_chunk_results(valid_results)
|
||||
result["chunk_count"] = len(chunks)
|
||||
except Exception as e:
|
||||
logger.error(f"[analyze_dashboard_multimodal] Failed to get analysis: {e!s}")
|
||||
return {
|
||||
@@ -1268,6 +1353,8 @@ class LLMClient:
|
||||
"summary": f"Failed to get response from LLM: {e!s}",
|
||||
"issues": [{"severity": "UNKNOWN", "message": "LLM provider returned empty or invalid response"}],
|
||||
}
|
||||
|
||||
return result
|
||||
# endregion LLMClient.analyze_dashboard_multimodal
|
||||
|
||||
# region LLMClient.analyze_dashboard_text_batch [TYPE Function] [C:3]
|
||||
|
||||
@@ -104,6 +104,10 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
|
||||
async def capture_dashboard(self, _dashboard_id, _screenshot_path):
|
||||
return [], []
|
||||
|
||||
@staticmethod
|
||||
def _cleanup_temp_files(_paths):
|
||||
pass
|
||||
|
||||
# endregion _FakeScreenshotService
|
||||
|
||||
# region _FakeLLMClient [TYPE Class]
|
||||
@@ -127,6 +131,22 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
|
||||
"issues": [],
|
||||
}
|
||||
|
||||
async def analyze_dashboard_multimodal(self, *_args, **_kwargs):
|
||||
return {
|
||||
"status": "PASS",
|
||||
"summary": "Dashboard healthy",
|
||||
"issues": [],
|
||||
}
|
||||
|
||||
async def analyze_dashboard_text_batch(self, *_args, **_kwargs):
|
||||
return {
|
||||
"dashboards": [{
|
||||
"status": "PASS",
|
||||
"summary": "Dashboard healthy",
|
||||
"issues": [],
|
||||
}],
|
||||
}
|
||||
|
||||
# endregion _FakeLLMClient
|
||||
|
||||
# region _FakeNotificationService [TYPE Class]
|
||||
|
||||
Reference in New Issue
Block a user