From 7f7a85b2c57e8693991775d18b1fa38640dc860d Mon Sep 17 00:00:00 2001 From: busya Date: Sun, 31 May 2026 22:57:58 +0300 Subject: [PATCH] refactor(validation): deduplicate image optimization, fix quality-reduction-before-chunking, unify chunk_count key - Extract _optimize_images() helper to eliminate duplicate optimization code (was duplicated between initial pass and quality-reduction fallback) - Move quality-reduction estimate to only apply when NOT chunking (each chunk fits the image limit by definition) - Fix _merge_chunk_results to return 'chunk_count' instead of 'chunks' for consistency with plugin.py - Simplify plugin.py: analysis.get('chunk_count', 1) instead of fallback chain - Document that Kilo API gateway is incompatible with AsyncOpenAI image format (probe returns 0) --- backend/src/api/routes/llm.py | 3 + backend/src/plugins/llm_analysis/plugin.py | 2 +- backend/src/plugins/llm_analysis/service.py | 94 +++++++++++---------- 3 files changed, 52 insertions(+), 47 deletions(-) diff --git a/backend/src/api/routes/llm.py b/backend/src/api/routes/llm.py index f3196bdf..a786f922 100644 --- a/backend/src/api/routes/llm.py +++ b/backend/src/api/routes/llm.py @@ -481,6 +481,9 @@ async def probe_max_images( from openai import AsyncOpenAI # Minimal 1x1 white JPEG in base64 (~840 chars, PIL-generated) + # NOTE: Uses raw AsyncOpenAI client — works for OpenAI/OpenRouter providers. + # Kilo API gateway does NOT support the OpenAI image content format; + # probes against Kilo providers will return max_images=0 (not a bug). PROBE_IMAGE_B64 = "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAFA3PEY8MlBGQUZaVVBfeMiCeG5uePWvuZHI////////////////////////////////////////////////////2wBDAVVaWnhpeOuCguv/////////////////////////////////////////////////////////////////////////wAARCAABAAEDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwC7RRRQB//Z" service = LLMProviderService(db) diff --git a/backend/src/plugins/llm_analysis/plugin.py b/backend/src/plugins/llm_analysis/plugin.py index 07e77bf5..3ad7de3d 100644 --- a/backend/src/plugins/llm_analysis/plugin.py +++ b/backend/src/plugins/llm_analysis/plugin.py @@ -419,7 +419,7 @@ class DashboardValidationPlugin(PluginBase): result_payload = _json_safe_value(validation_result.model_dump()) result_payload["execution_path"] = "multimodal" - result_payload["chunk_count"] = analysis.get("chunk_count") or analysis.get("chunks") or 1 + result_payload["chunk_count"] = analysis.get("chunk_count", 1) result_payload["screenshot_paths"] = webp_paths or jpeg_paths result_payload["logs_sent_to_llm"] = logs result_payload["logs_sent_count"] = len(logs) diff --git a/backend/src/plugins/llm_analysis/service.py b/backend/src/plugins/llm_analysis/service.py index bc256f2f..c2836604 100644 --- a/backend/src/plugins/llm_analysis/service.py +++ b/backend/src/plugins/llm_analysis/service.py @@ -1207,11 +1207,29 @@ class LLMClient: # endregion LLMClient._deduplicate_issues + # region LLMClient._optimize_images [TYPE Function] [C:2] + # @PURPOSE Convert screenshot paths to base64 at given quality, with fallback to raw read. + def _optimize_images(self, paths: list[str], max_width: int, quality: int) -> list[str]: + encoded: list[str] = [] + for path in paths: + try: + b64, _ = self._reduce_image_quality(path, max_width, quality) + encoded.append(b64) + except Exception as e: + logger.warning(f"[_optimize_images] Optimization failed for {path}: {e}") + with open(path, "rb") as f: + raw = f.read() + b64 = base64.b64encode(raw).decode("utf-8") + encoded.append(b64) + return encoded + + # endregion LLMClient._optimize_images + # region LLMClient._merge_chunk_results [TYPE Function] [C:2] # @PURPOSE Merge multiple chunk analyses into one. Takes the worst status, # concatenates summaries, and deduplicates issues. # @PRE chunks is a non-empty list of {status, summary, issues} dicts. - # @POST Returns a single merged dict. + # @POST Returns a single merged dict with chunk_count. def _merge_chunk_results(self, chunks: list[dict[str, Any]]) -> dict[str, Any]: STATUS_ORDER = {"FAIL": 0, "WARN": 1, "PASS": 2, "UNKNOWN": 3} worst_status = "UNKNOWN" @@ -1225,11 +1243,11 @@ class LLMClient: all_summaries.append(f"[Chunk {i + 1}/{len(chunks)}] {chunk.get('summary', 'No summary')}") all_issues.extend(chunk.get("issues", [])) - merged = { + merged: dict[str, Any] = { "status": worst_status, "summary": " | ".join(all_summaries), "issues": self._deduplicate_issues(all_issues), - "chunks": len(chunks), + "chunk_count": len(chunks), } return merged @@ -1254,10 +1272,11 @@ class LLMClient: # region LLMClient.analyze_dashboard_multimodal [TYPE Function] [C:3] # @PURPOSE Path A: send screenshots + logs to multimodal LLM, with chunking support. # @PRE screenshot_paths is a non-empty list of paths. - # @POST Returns dict {status, summary, issues}. + # @POST Returns dict {status, summary, issues} with optional chunk_count. # @SIDE_EFFECT Compresses images, calls external LLM API (possibly multiple times for chunks). # @RATIONALE Screenshots are split into chunks of max_images to respect provider image limits. - # Each chunk is sent in parallel; results are merged via _merge_chunk_results. + # Quality reduction is skipped when chunking — each chunk fits the limit by definition. + # Results are merged via _merge_chunk_results. async def analyze_dashboard_multimodal( self, screenshot_paths: list[str], @@ -1271,57 +1290,42 @@ class LLMClient: if not screenshot_paths: raise ValueError("screenshot_paths must be a non-empty list") - # 1. Optimize all images - encoded_images: list[str] = [] - for path in screenshot_paths: - try: - b64, _ = self._reduce_image_quality(path, max_width, image_quality) - encoded_images.append(b64) - except Exception as img_e: - logger.warning(f"[analyze_dashboard_multimodal] Image optimization failed for {path}: {img_e}") - with open(path, "rb") as f: - raw = f.read() - b64 = base64.b64encode(raw).decode("utf-8") - encoded_images.append(b64) + # 1. Optimize all images at requested quality + encoded_images = self._optimize_images(screenshot_paths, max_width, image_quality) log_text = "\n".join(logs) prompt = render_prompt(prompt_template, {"logs": log_text}) - # 2. Estimate payload size and reduce quality if needed - estimate = self._estimate_payload_size( - screenshot_paths, len(prompt) + len(log_text) - ) - if estimate["exceeds_limit"] and image_quality > 30: - logger.info( - f"[analyze_dashboard_multimodal] Payload estimated at {estimate['pct_of_limit']}% " - f"of context window. Reducing image quality to 30." - ) - encoded_images = [] - for path in screenshot_paths: - try: - b64, _ = self._reduce_image_quality(path, max_width, image_quality=30) - encoded_images.append(b64) - except Exception as img_e: - logger.warning(f"[analyze_dashboard_multimodal] Re-optimization failed for {path}: {img_e}") - with open(path, "rb") as f: - b64 = base64.b64encode(f.read()).decode("utf-8") - encoded_images.append(b64) - - # 3. Chunk images if max_images is set + # 2. Determine chunking n_total = len(encoded_images) chunk_size = max_images if (max_images and max_images > 0 and max_images < n_total) else n_total + is_chunking = chunk_size < n_total - if chunk_size < n_total: + if is_chunking: logger.reason( f"[analyze_dashboard_multimodal] Chunking {n_total} images into " - f"{ (n_total + chunk_size - 1) // chunk_size } chunks of {chunk_size}", + f"{(n_total + chunk_size - 1) // chunk_size} chunks of {chunk_size}", extra={"src": "analyze_dashboard_multimodal", "total": n_total, "chunk_size": chunk_size}, ) + # Skip quality reduction: each chunk has ≤ max_images images, + # well within the context window at normal quality. + else: + # Single batch: estimate payload and reduce quality if needed + estimate = self._estimate_payload_size( + screenshot_paths, len(prompt) + len(log_text) + ) + if estimate["exceeds_limit"] and image_quality > 30: + logger.info( + f"[analyze_dashboard_multimodal] Payload estimated at {estimate['pct_of_limit']}% " + f"of context window. Reducing image quality to 30." + ) + encoded_images = self._optimize_images(screenshot_paths, max_width, image_quality=30) - # Split into chunks - chunks: list[list[str]] = [] - for i in range(0, n_total, chunk_size): - chunks.append(encoded_images[i:i + chunk_size]) + # 3. Split into chunks + chunks: list[list[str]] = [ + encoded_images[i:i + chunk_size] + for i in range(0, n_total, chunk_size) + ] # 4. Call LLM — parallel for multiple chunks, single for one try: @@ -1331,7 +1335,6 @@ class LLMClient: tasks = [self._call_llm_for_images(chunk, prompt) for chunk in chunks] chunk_results = await asyncio.gather(*tasks, return_exceptions=True) - # Filter out exceptions valid_results: list[dict] = [] for i, cr in enumerate(chunk_results): if isinstance(cr, Exception): @@ -1345,7 +1348,6 @@ class LLMClient: valid_results.append(cr) result = self._merge_chunk_results(valid_results) - result["chunk_count"] = len(chunks) except Exception as e: logger.error(f"[analyze_dashboard_multimodal] Failed to get analysis: {e!s}") return {