diff --git a/backend/src/app.py b/backend/src/app.py index 27b25ffd..496415b8 100755 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -634,8 +634,8 @@ async def websocket_endpoint( extra={"task_id": task_id, "message": result.message}, ) await asyncio.sleep(2) - except (WebSocketDisconnect, StopIteration): - if isinstance(StopIteration): + except (WebSocketDisconnect, StopIteration) as _ws_exc: + if isinstance(_ws_exc, StopIteration): # Task reached terminal state — close cleanly with code 1000 try: await websocket.close(code=1000, reason="Task completed") diff --git a/backend/src/plugins/llm_analysis/plugin.py b/backend/src/plugins/llm_analysis/plugin.py index 57993302..07e77bf5 100644 --- a/backend/src/plugins/llm_analysis/plugin.py +++ b/backend/src/plugins/llm_analysis/plugin.py @@ -371,6 +371,7 @@ class DashboardValidationPlugin(PluginBase): screenshot_paths=jpeg_paths, logs=logs, prompt_template=dashboard_prompt, + max_images=db_provider.max_images, ) else: # Fallback: text-only analysis if no screenshots @@ -418,6 +419,7 @@ class DashboardValidationPlugin(PluginBase): result_payload = _json_safe_value(validation_result.model_dump()) result_payload["execution_path"] = "multimodal" + result_payload["chunk_count"] = analysis.get("chunk_count") or analysis.get("chunks") or 1 result_payload["screenshot_paths"] = webp_paths or jpeg_paths result_payload["logs_sent_to_llm"] = logs result_payload["logs_sent_count"] = len(logs) diff --git a/backend/src/plugins/llm_analysis/service.py b/backend/src/plugins/llm_analysis/service.py index 8b876083..bc256f2f 100644 --- a/backend/src/plugins/llm_analysis/service.py +++ b/backend/src/plugins/llm_analysis/service.py @@ -1193,13 +1193,71 @@ class LLMClient: } # endregion LLMClient._estimate_payload_size + # region LLMClient._deduplicate_issues [TYPE Function] [C:2] + # @PURPOSE Deduplicate issues by (severity, message, location) while preserving order. + def _deduplicate_issues(self, issues: list[dict]) -> list[dict]: + seen: set[tuple[str, str, str]] = set() + result: list[dict] = [] + for issue in issues: + key = (issue.get("severity", ""), issue.get("message", ""), issue.get("location", "") or "") + if key not in seen: + seen.add(key) + result.append(issue) + return result + + # endregion LLMClient._deduplicate_issues + + # region LLMClient._merge_chunk_results [TYPE Function] [C:2] + # @PURPOSE Merge multiple chunk analyses into one. Takes the worst status, + # concatenates summaries, and deduplicates issues. + # @PRE chunks is a non-empty list of {status, summary, issues} dicts. + # @POST Returns a single merged dict. + def _merge_chunk_results(self, chunks: list[dict[str, Any]]) -> dict[str, Any]: + STATUS_ORDER = {"FAIL": 0, "WARN": 1, "PASS": 2, "UNKNOWN": 3} + worst_status = "UNKNOWN" + all_summaries: list[str] = [] + all_issues: list[dict] = [] + + for i, chunk in enumerate(chunks): + s = chunk.get("status", "UNKNOWN") + if STATUS_ORDER.get(s, 3) < STATUS_ORDER.get(worst_status, 3): + worst_status = s + all_summaries.append(f"[Chunk {i + 1}/{len(chunks)}] {chunk.get('summary', 'No summary')}") + all_issues.extend(chunk.get("issues", [])) + + merged = { + "status": worst_status, + "summary": " | ".join(all_summaries), + "issues": self._deduplicate_issues(all_issues), + "chunks": len(chunks), + } + return merged + + # endregion LLMClient._merge_chunk_results + + # region LLMClient._call_llm_for_images [TYPE Function] [C:2] + # @PURPOSE Send a single chunk of images to the LLM and return parsed result. + async def _call_llm_for_images( + self, encoded_images: list[str], prompt: str + ) -> dict[str, Any]: + content: list[dict] = [{"type": "text", "text": prompt}] + for b64_img in encoded_images: + content.append({ + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{b64_img}"}, + }) + messages = [{"role": "user", "content": content}] + return await self.get_json_completion(messages) + + # endregion LLMClient._call_llm_for_images + # region LLMClient.analyze_dashboard_multimodal [TYPE Function] [C:3] - # @PURPOSE Path A: send multiple tab screenshots + logs to multimodal LLM. + # @PURPOSE Path A: send screenshots + logs to multimodal LLM, with chunking support. # @PRE screenshot_paths is a non-empty list of paths. # @POST Returns dict {status, summary, issues}. - # @SIDE_EFFECT Compresses images, calls external LLM API. - # @RATIONALE Multi-chunk: one screenshot per tab. All images sent in single content[] array. - # Token budget estimated before send; quality reduced if >80% of model context window. + # @SIDE_EFFECT Compresses images, calls external LLM API (possibly multiple times for chunks). + # @RATIONALE Screenshots are split into chunks of max_images to respect provider image limits. + # Each chunk is sent in parallel; results are merged via _merge_chunk_results. async def analyze_dashboard_multimodal( self, screenshot_paths: list[str], @@ -1207,6 +1265,7 @@ class LLMClient: prompt_template: str = DEFAULT_LLM_PROMPTS["dashboard_validation_prompt"], max_width: int = 1024, image_quality: int = 60, + max_images: int | None = None, ) -> dict[str, Any]: with belief_scope("analyze_dashboard_multimodal"): if not screenshot_paths: @@ -1248,19 +1307,45 @@ class LLMClient: b64 = base64.b64encode(f.read()).decode("utf-8") encoded_images.append(b64) - # 3. Build multimodal content array with all images - content: list[dict] = [{"type": "text", "text": prompt}] - for b64_img in encoded_images: - content.append({ - "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64_img}"}, - }) + # 3. Chunk images if max_images is set + n_total = len(encoded_images) + chunk_size = max_images if (max_images and max_images > 0 and max_images < n_total) else n_total - messages = [{"role": "user", "content": content}] + if chunk_size < n_total: + logger.reason( + f"[analyze_dashboard_multimodal] Chunking {n_total} images into " + f"{ (n_total + chunk_size - 1) // chunk_size } chunks of {chunk_size}", + extra={"src": "analyze_dashboard_multimodal", "total": n_total, "chunk_size": chunk_size}, + ) - # 4. Call LLM + # Split into chunks + chunks: list[list[str]] = [] + for i in range(0, n_total, chunk_size): + chunks.append(encoded_images[i:i + chunk_size]) + + # 4. Call LLM — parallel for multiple chunks, single for one try: - return await self.get_json_completion(messages) + if len(chunks) == 1: + result = await self._call_llm_for_images(chunks[0], prompt) + else: + tasks = [self._call_llm_for_images(chunk, prompt) for chunk in chunks] + chunk_results = await asyncio.gather(*tasks, return_exceptions=True) + + # Filter out exceptions + valid_results: list[dict] = [] + for i, cr in enumerate(chunk_results): + if isinstance(cr, Exception): + logger.error(f"[analyze_dashboard_multimodal] Chunk {i + 1}/{len(chunks)} failed: {cr!s}") + valid_results.append({ + "status": "UNKNOWN", + "summary": f"Chunk {i + 1} failed: {cr!s}", + "issues": [], + }) + else: + valid_results.append(cr) + + result = self._merge_chunk_results(valid_results) + result["chunk_count"] = len(chunks) except Exception as e: logger.error(f"[analyze_dashboard_multimodal] Failed to get analysis: {e!s}") return { @@ -1268,6 +1353,8 @@ class LLMClient: "summary": f"Failed to get response from LLM: {e!s}", "issues": [{"severity": "UNKNOWN", "message": "LLM provider returned empty or invalid response"}], } + + return result # endregion LLMClient.analyze_dashboard_multimodal # region LLMClient.analyze_dashboard_text_batch [TYPE Function] [C:3] diff --git a/backend/src/services/__tests__/test_llm_plugin_persistence.py b/backend/src/services/__tests__/test_llm_plugin_persistence.py index ef2fedae..63e02f79 100644 --- a/backend/src/services/__tests__/test_llm_plugin_persistence.py +++ b/backend/src/services/__tests__/test_llm_plugin_persistence.py @@ -104,6 +104,10 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( async def capture_dashboard(self, _dashboard_id, _screenshot_path): return [], [] + @staticmethod + def _cleanup_temp_files(_paths): + pass + # endregion _FakeScreenshotService # region _FakeLLMClient [TYPE Class] @@ -127,6 +131,22 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( "issues": [], } + async def analyze_dashboard_multimodal(self, *_args, **_kwargs): + return { + "status": "PASS", + "summary": "Dashboard healthy", + "issues": [], + } + + async def analyze_dashboard_text_batch(self, *_args, **_kwargs): + return { + "dashboards": [{ + "status": "PASS", + "summary": "Dashboard healthy", + "issues": [], + }], + } + # endregion _FakeLLMClient # region _FakeNotificationService [TYPE Class] diff --git a/frontend/src/lib/i18n/locales/en/validation.json b/frontend/src/lib/i18n/locales/en/validation.json index c562a45b..0b6a779e 100644 --- a/frontend/src/lib/i18n/locales/en/validation.json +++ b/frontend/src/lib/i18n/locales/en/validation.json @@ -212,5 +212,6 @@ "path_b": "Path B — Text-only", "no_issues": "No issues detected", "no_logs": "No logs available", - "no_screenshots": "No screenshots saved" + "no_screenshots": "No screenshots saved", + "chunked": "Chunked" } diff --git a/frontend/src/lib/i18n/locales/ru/validation.json b/frontend/src/lib/i18n/locales/ru/validation.json index 1ce83da9..9479a6e7 100644 --- a/frontend/src/lib/i18n/locales/ru/validation.json +++ b/frontend/src/lib/i18n/locales/ru/validation.json @@ -212,5 +212,6 @@ "path_b": "Путь B — Только текст", "no_issues": "Проблем не обнаружено", "no_logs": "Нет доступных логов", - "no_screenshots": "Нет сохраненных скриншотов" + "no_screenshots": "Нет сохраненных скриншотов", + "chunked": "По частям" } diff --git a/frontend/src/routes/validation-tasks/[policyId]/runs/[runId]/+page.svelte b/frontend/src/routes/validation-tasks/[policyId]/runs/[runId]/+page.svelte index 0826db64..e0cb63b7 100644 --- a/frontend/src/routes/validation-tasks/[policyId]/runs/[runId]/+page.svelte +++ b/frontend/src/routes/validation-tasks/[policyId]/runs/[runId]/+page.svelte @@ -414,6 +414,11 @@ {#if dbPath === 'A'} 📸 {$t.validation?.path_a || 'Path A — Screenshot'} | {$t.validation?.screenshots || 'Screenshots'} captured via browser automation + {#if dashboard.chunk_count && dashboard.chunk_count > 1} + + {$t.validation?.chunked || 'Chunked'} ×{dashboard.chunk_count} + + {/if} {:else} 📝 {$t.validation?.path_b || 'Path B — Text-only'} | {$t.validation?.issues || 'Issue'} detection via DOM extraction