busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(validation): deduplicate image optimization, fix quality-reduction-before-chunking, unify chunk_count key

Browse Source 
- Extract _optimize_images() helper to eliminate duplicate
  optimization code (was duplicated between initial pass and
  quality-reduction fallback)
- Move quality-reduction estimate to only apply when NOT chunking
  (each chunk fits the image limit by definition)
- Fix _merge_chunk_results to return 'chunk_count' instead of 'chunks'
  for consistency with plugin.py
- Simplify plugin.py: analysis.get('chunk_count', 1) instead of
  fallback chain
- Document that Kilo API gateway is incompatible with AsyncOpenAI
  image format (probe returns 0)
This commit is contained in:
busya
2026-05-31 22:57:58 +03:00
parent 2760fa09ea
commit 7f7a85b2c5
3 changed files with 52 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/api/routes/llm.py
View File

@@ -481,6 +481,9 @@ async def probe_max_images(
from openai import AsyncOpenAI
# Minimal 1x1 white JPEG in base64 (~840 chars, PIL-generated)
# NOTE: Uses raw AsyncOpenAI client — works for OpenAI/OpenRouter providers.
# Kilo API gateway does NOT support the OpenAI image content format;
# probes against Kilo providers will return max_images=0 (not a bug).
PROBE_IMAGE_B64 = "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAFA3PEY8MlBGQUZaVVBfeMiCeG5uePWvuZHI////////////////////////////////////////////////////2wBDAVVaWnhpeOuCguv/////////////////////////////////////////////////////////////////////////wAARCAABAAEDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwC7RRRQB//Z"
service = LLMProviderService(db)

2
backend/src/plugins/llm_analysis/plugin.py
View File

@@ -419,7 +419,7 @@ class DashboardValidationPlugin(PluginBase):
result_payload = _json_safe_value(validation_result.model_dump())
result_payload["execution_path"] = "multimodal"
result_payload["chunk_count"] = analysis.get("chunk_count") or analysis.get("chunks") or 1
result_payload["chunk_count"] = analysis.get("chunk_count", 1)
result_payload["screenshot_paths"] = webp_paths or jpeg_paths
result_payload["logs_sent_to_llm"] = logs
result_payload["logs_sent_count"] = len(logs)

94
backend/src/plugins/llm_analysis/service.py
View File

@@ -1207,11 +1207,29 @@ class LLMClient:
# endregion LLMClient._deduplicate_issues
# region LLMClient._optimize_images [TYPE Function] [C:2]
# @PURPOSE Convert screenshot paths to base64 at given quality, with fallback to raw read.
def _optimize_images(self, paths: list[str], max_width: int, quality: int) -> list[str]:
encoded: list[str] = []
for path in paths:
try:
b64, _ = self._reduce_image_quality(path, max_width, quality)
encoded.append(b64)
except Exception as e:
logger.warning(f"[_optimize_images] Optimization failed for {path}: {e}")
with open(path, "rb") as f:
raw = f.read()
b64 = base64.b64encode(raw).decode("utf-8")
encoded.append(b64)
return encoded
# endregion LLMClient._optimize_images
# region LLMClient._merge_chunk_results [TYPE Function] [C:2]
# @PURPOSE Merge multiple chunk analyses into one. Takes the worst status,
# concatenates summaries, and deduplicates issues.
# @PRE chunks is a non-empty list of {status, summary, issues} dicts.
# @POST Returns a single merged dict.
# @POST Returns a single merged dict with chunk_count.
def _merge_chunk_results(self, chunks: list[dict[str, Any]]) -> dict[str, Any]:
STATUS_ORDER = {"FAIL": 0, "WARN": 1, "PASS": 2, "UNKNOWN": 3}
worst_status = "UNKNOWN"
@@ -1225,11 +1243,11 @@ class LLMClient:
all_summaries.append(f"[Chunk {i + 1}/{len(chunks)}] {chunk.get('summary', 'No summary')}")
all_issues.extend(chunk.get("issues", []))
merged = {
merged: dict[str, Any] = {
"status": worst_status,
"summary": " | ".join(all_summaries),
"issues": self._deduplicate_issues(all_issues),
"chunks": len(chunks),
"chunk_count": len(chunks),
}
return merged
@@ -1254,10 +1272,11 @@ class LLMClient:
# region LLMClient.analyze_dashboard_multimodal [TYPE Function] [C:3]
# @PURPOSE Path A: send screenshots + logs to multimodal LLM, with chunking support.
# @PRE screenshot_paths is a non-empty list of paths.
# @POST Returns dict {status, summary, issues}.
# @POST Returns dict {status, summary, issues} with optional chunk_count.
# @SIDE_EFFECT Compresses images, calls external LLM API (possibly multiple times for chunks).
# @RATIONALE Screenshots are split into chunks of max_images to respect provider image limits.
# Each chunk is sent in parallel; results are merged via _merge_chunk_results.
# Quality reduction is skipped when chunking — each chunk fits the limit by definition.
# Results are merged via _merge_chunk_results.
async def analyze_dashboard_multimodal(
self,
screenshot_paths: list[str],
@@ -1271,57 +1290,42 @@ class LLMClient:
if not screenshot_paths:
raise ValueError("screenshot_paths must be a non-empty list")
# 1. Optimize all images
encoded_images: list[str] = []
for path in screenshot_paths:
try:
b64, _ = self._reduce_image_quality(path, max_width, image_quality)
encoded_images.append(b64)
except Exception as img_e:
logger.warning(f"[analyze_dashboard_multimodal] Image optimization failed for {path}: {img_e}")
with open(path, "rb") as f:
raw = f.read()
b64 = base64.b64encode(raw).decode("utf-8")
encoded_images.append(b64)
# 1. Optimize all images at requested quality
encoded_images = self._optimize_images(screenshot_paths, max_width, image_quality)
log_text = "\n".join(logs)
prompt = render_prompt(prompt_template, {"logs": log_text})
# 2. Estimate payload size and reduce quality if needed
estimate = self._estimate_payload_size(
screenshot_paths, len(prompt) + len(log_text)
)
if estimate["exceeds_limit"] and image_quality > 30:
logger.info(
f"[analyze_dashboard_multimodal] Payload estimated at {estimate['pct_of_limit']}% "
f"of context window. Reducing image quality to 30."
)
encoded_images = []
for path in screenshot_paths:
try:
b64, _ = self._reduce_image_quality(path, max_width, image_quality=30)
encoded_images.append(b64)
except Exception as img_e:
logger.warning(f"[analyze_dashboard_multimodal] Re-optimization failed for {path}: {img_e}")
with open(path, "rb") as f:
b64 = base64.b64encode(f.read()).decode("utf-8")
encoded_images.append(b64)
# 3. Chunk images if max_images is set
# 2. Determine chunking
n_total = len(encoded_images)
chunk_size = max_images if (max_images and max_images > 0 and max_images < n_total) else n_total
is_chunking = chunk_size < n_total
if chunk_size < n_total:
if is_chunking:
logger.reason(
f"[analyze_dashboard_multimodal] Chunking {n_total} images into "
f"{ (n_total + chunk_size - 1) // chunk_size } chunks of {chunk_size}",
f"{(n_total + chunk_size - 1) // chunk_size} chunks of {chunk_size}",
extra={"src": "analyze_dashboard_multimodal", "total": n_total, "chunk_size": chunk_size},
)
# Skip quality reduction: each chunk has ≤ max_images images,
# well within the context window at normal quality.
else:
# Single batch: estimate payload and reduce quality if needed
estimate = self._estimate_payload_size(
screenshot_paths, len(prompt) + len(log_text)
)
if estimate["exceeds_limit"] and image_quality > 30:
logger.info(
f"[analyze_dashboard_multimodal] Payload estimated at {estimate['pct_of_limit']}% "
f"of context window. Reducing image quality to 30."
)
encoded_images = self._optimize_images(screenshot_paths, max_width, image_quality=30)
# Split into chunks
chunks: list[list[str]] = []
for i in range(0, n_total, chunk_size):
chunks.append(encoded_images[i:i + chunk_size])
# 3. Split into chunks
chunks: list[list[str]] = [
encoded_images[i:i + chunk_size]
for i in range(0, n_total, chunk_size)
]
# 4. Call LLM — parallel for multiple chunks, single for one
try:
@@ -1331,7 +1335,6 @@ class LLMClient:
tasks = [self._call_llm_for_images(chunk, prompt) for chunk in chunks]
chunk_results = await asyncio.gather(*tasks, return_exceptions=True)
# Filter out exceptions
valid_results: list[dict] = []
for i, cr in enumerate(chunk_results):
if isinstance(cr, Exception):
@@ -1345,7 +1348,6 @@ class LLMClient:
valid_results.append(cr)
result = self._merge_chunk_results(valid_results)
result["chunk_count"] = len(chunks)
except Exception as e:
logger.error(f"[analyze_dashboard_multimodal] Failed to get analysis: {e!s}")
return {