busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(llm): auto-detect max images per request for LLM providers

Browse Source 
Add binary-search probe endpoint POST /providers/{id}/probe-max-images
that discovers the per-request image limit by sending incrementally
more 1x1 JPEGs via the provider's own API. Result is cached in the
new max_images column on the provider config.

- LLMProviderConfig: add max_images: int | None
- LLMProvider (SQLAlchemy): add max_images column
- Migration ed28d34edde7: clean ADD COLUMN
- LLMProviderService: create/update/set_max_images
- POST /providers/{id}/probe-max-images: binary search + error parsing
- ProviderConfig.svelte: 'Detect' button in edit modal + HelpTooltip
- i18n (en/ru): 11 new keys for probe UI
This commit is contained in:
busya
2026-05-31 22:31:36 +03:00
parent 652de471d2
commit 5e5f958eaa
8 changed files with 414 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
backend/src/api/routes/llm.py
View File

@@ -14,6 +14,7 @@ from sqlalchemy.orm import Session
from ...core.database import get_db
from ...core.logger import logger
from ...dependencies import get_current_user as get_current_active_user
from ...models.llm import ValidationPolicy
from ...plugins.llm_analysis.models import LLMProviderConfig, LLMProviderType
from ...schemas.auth import User
from ...services.llm_provider import LLMProviderService, is_masked_or_placeholder, mask_api_key
@@ -83,6 +84,7 @@ async def get_providers(
default_model=p.default_model,
is_active=p.is_active,
is_multimodal=bool(p.is_multimodal) if p.is_multimodal is not None else False,
max_images=p.max_images,
)
for p in providers
]
@@ -269,6 +271,7 @@ async def create_provider(
default_model=provider.default_model,
is_active=provider.is_active,
is_multimodal=bool(provider.is_multimodal) if provider.is_multimodal is not None else False,
max_images=provider.max_images,
)
@@ -305,6 +308,7 @@ async def update_provider(
default_model=provider.default_model,
is_active=provider.is_active,
is_multimodal=bool(provider.is_multimodal) if provider.is_multimodal is not None else False,
max_images=provider.max_images,
)
@@ -316,6 +320,7 @@ async def update_provider(
# @PRE User is authenticated and has admin permissions.
# @POST Returns success status.
# @RELATION CALLS -> [LLMProviderService]
# @RELATION CALLS -> [ValidationPolicy]
@router.delete("/providers/{provider_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_provider(
provider_id: str,
@@ -325,6 +330,20 @@ async def delete_provider(
"""
Delete an LLM provider configuration.
"""
# Check if any active validation tasks reference this provider
active_tasks = db.query(ValidationPolicy).filter(
ValidationPolicy.provider_id == provider_id,
ValidationPolicy.is_active == True,
).all()
if active_tasks:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={
"error": f"Provider is used by {len(active_tasks)} active validation task(s)",
"blocking_tasks": [{"id": t.id, "name": t.name} for t in active_tasks],
},
)
service = LLMProviderService(db)
if not service.delete_provider(provider_id):
raise HTTPException(status_code=404, detail="Provider not found")
@@ -431,4 +450,150 @@ async def test_provider_config(
# #endregion test_provider_config
# #region ProbeMaxImagesResponse [C:1] [TYPE Class]
# @BRIEF Response model for probe-max-images endpoint.
class ProbeMaxImagesResponse(BaseModel):
max_images: int | None
method: str = "binary_search"
# #endregion ProbeMaxImagesResponse
# #region probe_max_images [C:4] [TYPE Function]
# @BRIEF Probe an LLM provider to discover its max images per request limit.
# @PRE Provider exists and has valid API key.
# @POST Returns the detected max_images limit and caches it on the provider.
# @RELATION CALLS -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [EXT:Library:openai]
@router.post("/providers/{provider_id}/probe-max-images", response_model=ProbeMaxImagesResponse)
async def probe_max_images(
provider_id: str,
current_user: User = Depends(get_current_active_user),
db: Session = Depends(get_db),
):
"""
Probe an LLM provider to discover the maximum number of images
allowed per request. Uses binary search with a minimal 1x1 JPEG
to find the limit without consuming significant tokens.
The result is cached on the provider's max_images field.
"""
from openai import AsyncOpenAI
# Minimal 1x1 white JPEG in base64 (~840 chars, PIL-generated)
PROBE_IMAGE_B64 = "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAFA3PEY8MlBGQUZaVVBfeMiCeG5uePWvuZHI////////////////////////////////////////////////////2wBDAVVaWnhpeOuCguv/////////////////////////////////////////////////////////////////////////wAARCAABAAEDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwC7RRRQB//Z"
service = LLMProviderService(db)
db_provider = service.get_provider(provider_id)
if not db_provider:
raise HTTPException(status_code=404, detail="Provider not found")
api_key = service.get_decrypted_api_key(provider_id)
if not api_key:
raise HTTPException(
status_code=400,
detail="Provider has no valid API key. Decryption failed or key is missing.",
)
model = db_provider.default_model
if not model:
raise HTTPException(status_code=400, detail="Provider has no default model set")
# Build the client
client = AsyncOpenAI(
api_key=api_key,
base_url=db_provider.base_url,
)
def build_content(n_images: int) -> list[dict]:
"""Build a minimal content array with N probe images."""
content: list[dict] = [{"type": "text", "text": "OK"}]
for _ in range(n_images):
content.append({
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{PROBE_IMAGE_B64}"},
})
return content
async def try_n(n: int):
"""Try sending N images. Returns True if OK, integer limit if parsed from error, or False if unknown failure."""
try:
await client.chat.completions.create(
model=model,
messages=[{"role": "user", "content": build_content(n)}],
max_tokens=1,
)
logger.reason(f"[probe_max_images] {n} images: OK", extra={"src": "probe_max_images"})
return True
except Exception as e:
msg = str(e)
logger.reason(
f"[probe_max_images] {n} images: FAILED — {msg[:200]}",
extra={"src": "probe_max_images"},
)
# Try to parse "At most 8 image(s)" from error
import re
match = re.search(r"At most (\d+) image", msg)
if match:
return int(match.group(1))
return False
# Phase 1: Exponential growth to find upper bound
last_ok = 0
first_fail = None
limit_parsed = None
for n in [1, 2, 4, 8, 16, 32, 64]:
result = await try_n(n)
if result is True:
last_ok = n
elif type(result) is int:
# Parsed exact limit from error message
limit_parsed = result
first_fail = n
break
else:
first_fail = n
break
if limit_parsed is not None:
detected_limit = limit_parsed
method = "parse_error"
elif first_fail is None:
# 64 still OK — no practical limit detected
detected_limit = None
method = "no_limit_detected"
elif last_ok == 0:
# Even 1 image failed — provider may not support multimodal
# Set a minimal safe limit (0 = don't use images) but log the issue
logger.warning(
f"[probe_max_images] Even 1 image failed for provider {provider_id}. "
f"Provider may not support multimodal input or credentials are invalid."
)
detected_limit = 0
method = "binary_search"
else:
# Phase 2: Binary search between last_ok and first_fail
lo, hi = last_ok, first_fail
while lo < hi:
mid = (lo + hi + 1) // 2
result = await try_n(mid)
if result is True:
lo = mid
else:
hi = mid - 1
detected_limit = lo
method = "binary_search"
# Store the result
service.set_max_images(provider_id, detected_limit)
return ProbeMaxImagesResponse(
max_images=detected_limit,
method=method,
)
# #endregion probe_max_images
# #endregion LlmRoutes

97
backend/src/models/llm.py
View File

@@ -6,7 +6,8 @@
from datetime import UTC, datetime
import uuid
from sqlalchemy import JSON, Boolean, Column, DateTime, String, Text, Time
from sqlalchemy import JSON, Boolean, Column, DateTime, ForeignKey, Integer, String, Text, Time
from sqlalchemy.orm import relationship
from .mapping import Base
@@ -14,6 +15,7 @@ from .mapping import Base
def generate_uuid():
return str(uuid.uuid4())
# #region ValidationPolicy [TYPE Class]
# @BRIEF Defines a scheduled rule for validating a group of dashboards within an execution window.
# @RELATION DEPENDS_ON -> [LLMProvider]
@@ -22,52 +24,125 @@ class ValidationPolicy(Base):
id = Column(String, primary_key=True, default=generate_uuid)
name = Column(String, nullable=False)
description = Column(Text, nullable=True)
environment_id = Column(String, nullable=False)
is_active = Column(Boolean, default=True)
dashboard_ids = Column(JSON, nullable=False) # Array of dashboard IDs
provider_id = Column(String, nullable=True) # Reference to LLMProvider.id
schedule_days = Column(JSON, nullable=False) # Array of integers (0-6)
dashboard_ids = Column(JSON, nullable=False) # Array of dashboard IDs (v1, deprecated in favor of ValidationSource)
provider_id = Column(String, nullable=True) # Reference to LLMProvider.id
schedule_days = Column(JSON, nullable=False) # Array of integers (0-6)
window_start = Column(Time, nullable=False)
window_end = Column(Time, nullable=False)
notify_owners = Column(Boolean, default=True)
custom_channels = Column(JSON, nullable=True) # List of external channels
alert_condition = Column(String, default="FAIL_ONLY") # FAIL_ONLY, WARN_AND_FAIL, ALWAYS
custom_channels = Column(JSON, nullable=True) # List of external channels
alert_condition = Column(String, default="FAIL_ONLY") # FAIL_ONLY, WARN_AND_FAIL, ALWAYS
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
updated_at = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
# v2 columns
prompt_template = Column(Text, nullable=True)
screenshot_enabled = Column(Boolean, default=True)
logs_enabled = Column(Boolean, default=True)
execute_chart_data = Column(Boolean, default=False)
llm_batch_size = Column(Integer, default=1)
policy_dashboard_concurrency_limit = Column(Integer, default=3)
source_snapshot = Column(JSON, nullable=True)
# v2 relationships
sources = relationship("ValidationSource", back_populates="policy", cascade="all, delete-orphan")
# #endregion ValidationPolicy
# #region LLMProvider [TYPE Class]
# @BRIEF SQLAlchemy model for LLM provider configuration.
class LLMProvider(Base):
__tablename__ = "llm_providers"
id = Column(String, primary_key=True, default=generate_uuid)
provider_type = Column(String, nullable=False) # openai, openrouter, kilo
provider_type = Column(String, nullable=False) # openai, openrouter, kilo
name = Column(String, nullable=False)
base_url = Column(String, nullable=False)
api_key = Column(String, nullable=False) # Should be encrypted
api_key = Column(String, nullable=False) # Should be encrypted
default_model = Column(String, nullable=False)
is_active = Column(Boolean, default=True)
is_multimodal = Column(Boolean, default=False)
max_images = Column(Integer, nullable=True, default=None)
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
# #endregion LLMProvider
# #region ValidationSource [TYPE Class]
# @BRIEF Relational source mapping for validation policies (v2).
class ValidationSource(Base):
__tablename__ = "validation_sources"
id = Column(String, primary_key=True, default=generate_uuid)
policy_id = Column(String, ForeignKey("validation_policies.id"), nullable=False, index=True)
type = Column(String, nullable=False) # "dashboard_id" or "dashboard_url"
value = Column(String, nullable=False) # numeric ID or full URL
parsed_context = Column(JSON, nullable=True) # SupersetContextExtractor result
status = Column(String, nullable=False, default="valid") # valid | partial_recovery | stale | invalid | permission_denied | parse_failed
resolved_dashboard_id = Column(String, nullable=True)
title = Column(String, nullable=True)
last_error = Column(String, nullable=True)
last_checked_at = Column(DateTime, nullable=True)
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
policy = relationship("ValidationPolicy", back_populates="sources")
# #endregion ValidationSource
# #region ValidationRecord [TYPE Class]
# @BRIEF SQLAlchemy model for dashboard validation history.
class ValidationRecord(Base):
__tablename__ = "llm_validation_results"
id = Column(String, primary_key=True, default=generate_uuid)
task_id = Column(String, nullable=True, index=True) # Reference to TaskRecord
policy_id = Column(String, nullable=True, index=True) # Reference to ValidationPolicy.id
task_id = Column(String, nullable=True, index=True) # Reference to TaskRecord
policy_id = Column(String, nullable=True, index=True) # Reference to ValidationPolicy.id
dashboard_id = Column(String, nullable=False, index=True)
environment_id = Column(String, nullable=True, index=True)
timestamp = Column(DateTime, default=lambda: datetime.now(UTC))
status = Column(String, nullable=False) # PASS, WARN, FAIL, UNKNOWN
status = Column(String, nullable=False) # PASS, WARN, FAIL, UNKNOWN
screenshot_path = Column(String, nullable=True)
issues = Column(JSON, nullable=False)
summary = Column(Text, nullable=False)
raw_response = Column(Text, nullable=True)
# v2 columns
run_id = Column(String, ForeignKey("validation_runs.id"), nullable=True, index=True)
source_id = Column(String, ForeignKey("validation_sources.id"), nullable=True)
execution_path = Column(String, nullable=True) # "screenshot" or "text_only"
dataset_health = Column(JSON, nullable=True)
chart_data_results = Column(JSON, nullable=True)
tab_screenshots = Column(JSON, nullable=True)
logs_sent_to_llm = Column(JSON, nullable=True)
token_usage = Column(JSON, nullable=True)
timings = Column(JSON, nullable=True)
screenshot_paths = Column(JSON, nullable=True) # List of WebP paths
# #endregion ValidationRecord
# #region ValidationRun [TYPE Class]
# @BRIEF Tracks a single validation execution run (v2).
class ValidationRun(Base):
__tablename__ = "validation_runs"
id = Column(String, primary_key=True, default=generate_uuid)
policy_id = Column(String, ForeignKey("validation_policies.id"), nullable=False, index=True)
task_id = Column(String, nullable=True) # Task Manager task ID
started_at = Column(DateTime, default=lambda: datetime.now(UTC))
finished_at = Column(DateTime, nullable=True)
trigger = Column(String, nullable=False, default="manual") # "manual" or "scheduled"
status = Column(String, nullable=False, default="running") # running | completed | partial | failed
dashboard_count = Column(Integer, default=0)
pass_count = Column(Integer, default=0)
warn_count = Column(Integer, default=0)
fail_count = Column(Integer, default=0)
unknown_count = Column(Integer, default=0)
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
records = relationship("ValidationRecord", backref="run", foreign_keys=[ValidationRecord.run_id])
# #endregion ValidationRun
# #endregion LlmModels

1
backend/src/plugins/llm_analysis/models.py
View File

@@ -29,6 +29,7 @@ class LLMProviderConfig(BaseModel):
default_model: str
is_active: bool = True
is_multimodal: bool = False
max_images: int | None = None
# #endregion LLMProviderConfig
# #region ValidationStatus [TYPE Class]

19
backend/src/services/llm_provider.py
View File

@@ -114,6 +114,7 @@ class LLMProviderService:
default_model=config.default_model,
is_active=config.is_active,
is_multimodal=config.is_multimodal,
max_images=config.max_images,
)
self.db.add(db_provider)
self.db.commit()
@@ -146,6 +147,7 @@ class LLMProviderService:
db_provider.default_model = config.default_model
db_provider.is_active = config.is_active
db_provider.is_multimodal = config.is_multimodal
db_provider.max_images = config.max_images
self.db.commit()
self.db.refresh(db_provider)
@@ -153,6 +155,23 @@ class LLMProviderService:
# endregion update_provider
# region set_max_images [TYPE Function]
# @PURPOSE: Updates only the max_images field for a provider.
# @PRE provider_id must exist.
# @POST Returns updated provider or None if not found.
# @RELATION DEPENDS_ON ->[LLMProvider]
def set_max_images(self, provider_id: str, max_images: int | None) -> LLMProvider | None:
with belief_scope("set_max_images"):
db_provider = self.get_provider(provider_id)
if not db_provider:
return None
db_provider.max_images = max_images
self.db.commit()
self.db.refresh(db_provider)
return db_provider
# endregion set_max_images
# region delete_provider [TYPE Function]
# @PURPOSE: Deletes an LLM provider.
# @PRE provider_id must exist.