diff --git a/backend/alembic/versions/ed28d34edde7_add_max_images_to_llm_providers.py b/backend/alembic/versions/ed28d34edde7_add_max_images_to_llm_providers.py new file mode 100644 index 00000000..9c3f4139 --- /dev/null +++ b/backend/alembic/versions/ed28d34edde7_add_max_images_to_llm_providers.py @@ -0,0 +1,26 @@ +"""add_max_images_to_llm_providers + +Revision ID: ed28d34edde7 +Revises: 7703bbc038bd +Create Date: 2026-05-31 22:19:21.922928 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = 'ed28d34edde7' +down_revision: Union[str, Sequence[str], None] = '7703bbc038bd' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column('llm_providers', sa.Column('max_images', sa.Integer(), nullable=True)) + + +def downgrade() -> None: + op.drop_column('llm_providers', 'max_images') diff --git a/backend/src/api/routes/llm.py b/backend/src/api/routes/llm.py index 62c288b3..f3196bdf 100644 --- a/backend/src/api/routes/llm.py +++ b/backend/src/api/routes/llm.py @@ -14,6 +14,7 @@ from sqlalchemy.orm import Session from ...core.database import get_db from ...core.logger import logger from ...dependencies import get_current_user as get_current_active_user +from ...models.llm import ValidationPolicy from ...plugins.llm_analysis.models import LLMProviderConfig, LLMProviderType from ...schemas.auth import User from ...services.llm_provider import LLMProviderService, is_masked_or_placeholder, mask_api_key @@ -83,6 +84,7 @@ async def get_providers( default_model=p.default_model, is_active=p.is_active, is_multimodal=bool(p.is_multimodal) if p.is_multimodal is not None else False, + max_images=p.max_images, ) for p in providers ] @@ -269,6 +271,7 @@ async def create_provider( default_model=provider.default_model, is_active=provider.is_active, is_multimodal=bool(provider.is_multimodal) if provider.is_multimodal is not None else False, + max_images=provider.max_images, ) @@ -305,6 +308,7 @@ async def update_provider( default_model=provider.default_model, is_active=provider.is_active, is_multimodal=bool(provider.is_multimodal) if provider.is_multimodal is not None else False, + max_images=provider.max_images, ) @@ -316,6 +320,7 @@ async def update_provider( # @PRE User is authenticated and has admin permissions. # @POST Returns success status. # @RELATION CALLS -> [LLMProviderService] +# @RELATION CALLS -> [ValidationPolicy] @router.delete("/providers/{provider_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_provider( provider_id: str, @@ -325,6 +330,20 @@ async def delete_provider( """ Delete an LLM provider configuration. """ + # Check if any active validation tasks reference this provider + active_tasks = db.query(ValidationPolicy).filter( + ValidationPolicy.provider_id == provider_id, + ValidationPolicy.is_active == True, + ).all() + if active_tasks: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "error": f"Provider is used by {len(active_tasks)} active validation task(s)", + "blocking_tasks": [{"id": t.id, "name": t.name} for t in active_tasks], + }, + ) + service = LLMProviderService(db) if not service.delete_provider(provider_id): raise HTTPException(status_code=404, detail="Provider not found") @@ -431,4 +450,150 @@ async def test_provider_config( # #endregion test_provider_config + +# #region ProbeMaxImagesResponse [C:1] [TYPE Class] +# @BRIEF Response model for probe-max-images endpoint. +class ProbeMaxImagesResponse(BaseModel): + max_images: int | None + method: str = "binary_search" + # #endregion ProbeMaxImagesResponse + + +# #region probe_max_images [C:4] [TYPE Function] +# @BRIEF Probe an LLM provider to discover its max images per request limit. +# @PRE Provider exists and has valid API key. +# @POST Returns the detected max_images limit and caches it on the provider. +# @RELATION CALLS -> [LLMProviderService] +# @RELATION DEPENDS_ON -> [EXT:Library:openai] +@router.post("/providers/{provider_id}/probe-max-images", response_model=ProbeMaxImagesResponse) +async def probe_max_images( + provider_id: str, + current_user: User = Depends(get_current_active_user), + db: Session = Depends(get_db), +): + """ + Probe an LLM provider to discover the maximum number of images + allowed per request. Uses binary search with a minimal 1x1 JPEG + to find the limit without consuming significant tokens. + + The result is cached on the provider's max_images field. + """ + from openai import AsyncOpenAI + + # Minimal 1x1 white JPEG in base64 (~840 chars, PIL-generated) + PROBE_IMAGE_B64 = "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAFA3PEY8MlBGQUZaVVBfeMiCeG5uePWvuZHI////////////////////////////////////////////////////2wBDAVVaWnhpeOuCguv/////////////////////////////////////////////////////////////////////////wAARCAABAAEDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwC7RRRQB//Z" + + service = LLMProviderService(db) + db_provider = service.get_provider(provider_id) + if not db_provider: + raise HTTPException(status_code=404, detail="Provider not found") + + api_key = service.get_decrypted_api_key(provider_id) + if not api_key: + raise HTTPException( + status_code=400, + detail="Provider has no valid API key. Decryption failed or key is missing.", + ) + + model = db_provider.default_model + if not model: + raise HTTPException(status_code=400, detail="Provider has no default model set") + + # Build the client + client = AsyncOpenAI( + api_key=api_key, + base_url=db_provider.base_url, + ) + + def build_content(n_images: int) -> list[dict]: + """Build a minimal content array with N probe images.""" + content: list[dict] = [{"type": "text", "text": "OK"}] + for _ in range(n_images): + content.append({ + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{PROBE_IMAGE_B64}"}, + }) + return content + + async def try_n(n: int): + """Try sending N images. Returns True if OK, integer limit if parsed from error, or False if unknown failure.""" + try: + await client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": build_content(n)}], + max_tokens=1, + ) + logger.reason(f"[probe_max_images] {n} images: OK", extra={"src": "probe_max_images"}) + return True + except Exception as e: + msg = str(e) + logger.reason( + f"[probe_max_images] {n} images: FAILED — {msg[:200]}", + extra={"src": "probe_max_images"}, + ) + # Try to parse "At most 8 image(s)" from error + import re + match = re.search(r"At most (\d+) image", msg) + if match: + return int(match.group(1)) + return False + + # Phase 1: Exponential growth to find upper bound + last_ok = 0 + first_fail = None + limit_parsed = None + + for n in [1, 2, 4, 8, 16, 32, 64]: + result = await try_n(n) + if result is True: + last_ok = n + elif type(result) is int: + # Parsed exact limit from error message + limit_parsed = result + first_fail = n + break + else: + first_fail = n + break + + if limit_parsed is not None: + detected_limit = limit_parsed + method = "parse_error" + elif first_fail is None: + # 64 still OK — no practical limit detected + detected_limit = None + method = "no_limit_detected" + elif last_ok == 0: + # Even 1 image failed — provider may not support multimodal + # Set a minimal safe limit (0 = don't use images) but log the issue + logger.warning( + f"[probe_max_images] Even 1 image failed for provider {provider_id}. " + f"Provider may not support multimodal input or credentials are invalid." + ) + detected_limit = 0 + method = "binary_search" + else: + # Phase 2: Binary search between last_ok and first_fail + lo, hi = last_ok, first_fail + while lo < hi: + mid = (lo + hi + 1) // 2 + result = await try_n(mid) + if result is True: + lo = mid + else: + hi = mid - 1 + detected_limit = lo + method = "binary_search" + + # Store the result + service.set_max_images(provider_id, detected_limit) + + return ProbeMaxImagesResponse( + max_images=detected_limit, + method=method, + ) + + +# #endregion probe_max_images + # #endregion LlmRoutes diff --git a/backend/src/models/llm.py b/backend/src/models/llm.py index 5962e1fc..0060881c 100644 --- a/backend/src/models/llm.py +++ b/backend/src/models/llm.py @@ -6,7 +6,8 @@ from datetime import UTC, datetime import uuid -from sqlalchemy import JSON, Boolean, Column, DateTime, String, Text, Time +from sqlalchemy import JSON, Boolean, Column, DateTime, ForeignKey, Integer, String, Text, Time +from sqlalchemy.orm import relationship from .mapping import Base @@ -14,6 +15,7 @@ from .mapping import Base def generate_uuid(): return str(uuid.uuid4()) + # #region ValidationPolicy [TYPE Class] # @BRIEF Defines a scheduled rule for validating a group of dashboards within an execution window. # @RELATION DEPENDS_ON -> [LLMProvider] @@ -22,52 +24,125 @@ class ValidationPolicy(Base): id = Column(String, primary_key=True, default=generate_uuid) name = Column(String, nullable=False) + description = Column(Text, nullable=True) environment_id = Column(String, nullable=False) is_active = Column(Boolean, default=True) - dashboard_ids = Column(JSON, nullable=False) # Array of dashboard IDs - provider_id = Column(String, nullable=True) # Reference to LLMProvider.id - schedule_days = Column(JSON, nullable=False) # Array of integers (0-6) + dashboard_ids = Column(JSON, nullable=False) # Array of dashboard IDs (v1, deprecated in favor of ValidationSource) + provider_id = Column(String, nullable=True) # Reference to LLMProvider.id + schedule_days = Column(JSON, nullable=False) # Array of integers (0-6) window_start = Column(Time, nullable=False) window_end = Column(Time, nullable=False) notify_owners = Column(Boolean, default=True) - custom_channels = Column(JSON, nullable=True) # List of external channels - alert_condition = Column(String, default="FAIL_ONLY") # FAIL_ONLY, WARN_AND_FAIL, ALWAYS + custom_channels = Column(JSON, nullable=True) # List of external channels + alert_condition = Column(String, default="FAIL_ONLY") # FAIL_ONLY, WARN_AND_FAIL, ALWAYS created_at = Column(DateTime, default=lambda: datetime.now(UTC)) updated_at = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) + + # v2 columns + prompt_template = Column(Text, nullable=True) + screenshot_enabled = Column(Boolean, default=True) + logs_enabled = Column(Boolean, default=True) + execute_chart_data = Column(Boolean, default=False) + llm_batch_size = Column(Integer, default=1) + policy_dashboard_concurrency_limit = Column(Integer, default=3) + source_snapshot = Column(JSON, nullable=True) + + # v2 relationships + sources = relationship("ValidationSource", back_populates="policy", cascade="all, delete-orphan") # #endregion ValidationPolicy + # #region LLMProvider [TYPE Class] # @BRIEF SQLAlchemy model for LLM provider configuration. class LLMProvider(Base): __tablename__ = "llm_providers" id = Column(String, primary_key=True, default=generate_uuid) - provider_type = Column(String, nullable=False) # openai, openrouter, kilo + provider_type = Column(String, nullable=False) # openai, openrouter, kilo name = Column(String, nullable=False) base_url = Column(String, nullable=False) - api_key = Column(String, nullable=False) # Should be encrypted + api_key = Column(String, nullable=False) # Should be encrypted default_model = Column(String, nullable=False) is_active = Column(Boolean, default=True) is_multimodal = Column(Boolean, default=False) + max_images = Column(Integer, nullable=True, default=None) created_at = Column(DateTime, default=lambda: datetime.now(UTC)) # #endregion LLMProvider + +# #region ValidationSource [TYPE Class] +# @BRIEF Relational source mapping for validation policies (v2). +class ValidationSource(Base): + __tablename__ = "validation_sources" + + id = Column(String, primary_key=True, default=generate_uuid) + policy_id = Column(String, ForeignKey("validation_policies.id"), nullable=False, index=True) + type = Column(String, nullable=False) # "dashboard_id" or "dashboard_url" + value = Column(String, nullable=False) # numeric ID or full URL + parsed_context = Column(JSON, nullable=True) # SupersetContextExtractor result + status = Column(String, nullable=False, default="valid") # valid | partial_recovery | stale | invalid | permission_denied | parse_failed + resolved_dashboard_id = Column(String, nullable=True) + title = Column(String, nullable=True) + last_error = Column(String, nullable=True) + last_checked_at = Column(DateTime, nullable=True) + created_at = Column(DateTime, default=lambda: datetime.now(UTC)) + + policy = relationship("ValidationPolicy", back_populates="sources") +# #endregion ValidationSource + + # #region ValidationRecord [TYPE Class] # @BRIEF SQLAlchemy model for dashboard validation history. class ValidationRecord(Base): __tablename__ = "llm_validation_results" id = Column(String, primary_key=True, default=generate_uuid) - task_id = Column(String, nullable=True, index=True) # Reference to TaskRecord - policy_id = Column(String, nullable=True, index=True) # Reference to ValidationPolicy.id + task_id = Column(String, nullable=True, index=True) # Reference to TaskRecord + policy_id = Column(String, nullable=True, index=True) # Reference to ValidationPolicy.id dashboard_id = Column(String, nullable=False, index=True) environment_id = Column(String, nullable=True, index=True) timestamp = Column(DateTime, default=lambda: datetime.now(UTC)) - status = Column(String, nullable=False) # PASS, WARN, FAIL, UNKNOWN + status = Column(String, nullable=False) # PASS, WARN, FAIL, UNKNOWN screenshot_path = Column(String, nullable=True) issues = Column(JSON, nullable=False) summary = Column(Text, nullable=False) raw_response = Column(Text, nullable=True) + + # v2 columns + run_id = Column(String, ForeignKey("validation_runs.id"), nullable=True, index=True) + source_id = Column(String, ForeignKey("validation_sources.id"), nullable=True) + execution_path = Column(String, nullable=True) # "screenshot" or "text_only" + dataset_health = Column(JSON, nullable=True) + chart_data_results = Column(JSON, nullable=True) + tab_screenshots = Column(JSON, nullable=True) + logs_sent_to_llm = Column(JSON, nullable=True) + token_usage = Column(JSON, nullable=True) + timings = Column(JSON, nullable=True) + screenshot_paths = Column(JSON, nullable=True) # List of WebP paths # #endregion ValidationRecord + +# #region ValidationRun [TYPE Class] +# @BRIEF Tracks a single validation execution run (v2). +class ValidationRun(Base): + __tablename__ = "validation_runs" + + id = Column(String, primary_key=True, default=generate_uuid) + policy_id = Column(String, ForeignKey("validation_policies.id"), nullable=False, index=True) + task_id = Column(String, nullable=True) # Task Manager task ID + started_at = Column(DateTime, default=lambda: datetime.now(UTC)) + finished_at = Column(DateTime, nullable=True) + trigger = Column(String, nullable=False, default="manual") # "manual" or "scheduled" + status = Column(String, nullable=False, default="running") # running | completed | partial | failed + dashboard_count = Column(Integer, default=0) + pass_count = Column(Integer, default=0) + warn_count = Column(Integer, default=0) + fail_count = Column(Integer, default=0) + unknown_count = Column(Integer, default=0) + created_at = Column(DateTime, default=lambda: datetime.now(UTC)) + + records = relationship("ValidationRecord", backref="run", foreign_keys=[ValidationRecord.run_id]) +# #endregion ValidationRun + + # #endregion LlmModels diff --git a/backend/src/plugins/llm_analysis/models.py b/backend/src/plugins/llm_analysis/models.py index 1601d96a..4ea53a95 100644 --- a/backend/src/plugins/llm_analysis/models.py +++ b/backend/src/plugins/llm_analysis/models.py @@ -29,6 +29,7 @@ class LLMProviderConfig(BaseModel): default_model: str is_active: bool = True is_multimodal: bool = False + max_images: int | None = None # #endregion LLMProviderConfig # #region ValidationStatus [TYPE Class] diff --git a/backend/src/services/llm_provider.py b/backend/src/services/llm_provider.py index 98a4867f..1b812353 100644 --- a/backend/src/services/llm_provider.py +++ b/backend/src/services/llm_provider.py @@ -114,6 +114,7 @@ class LLMProviderService: default_model=config.default_model, is_active=config.is_active, is_multimodal=config.is_multimodal, + max_images=config.max_images, ) self.db.add(db_provider) self.db.commit() @@ -146,6 +147,7 @@ class LLMProviderService: db_provider.default_model = config.default_model db_provider.is_active = config.is_active db_provider.is_multimodal = config.is_multimodal + db_provider.max_images = config.max_images self.db.commit() self.db.refresh(db_provider) @@ -153,6 +155,23 @@ class LLMProviderService: # endregion update_provider + # region set_max_images [TYPE Function] + # @PURPOSE: Updates only the max_images field for a provider. + # @PRE provider_id must exist. + # @POST Returns updated provider or None if not found. + # @RELATION DEPENDS_ON ->[LLMProvider] + def set_max_images(self, provider_id: str, max_images: int | None) -> LLMProvider | None: + with belief_scope("set_max_images"): + db_provider = self.get_provider(provider_id) + if not db_provider: + return None + db_provider.max_images = max_images + self.db.commit() + self.db.refresh(db_provider) + return db_provider + + # endregion set_max_images + # region delete_provider [TYPE Function] # @PURPOSE: Deletes an LLM provider. # @PRE provider_id must exist. diff --git a/frontend/src/components/llm/ProviderConfig.svelte b/frontend/src/components/llm/ProviderConfig.svelte index 02fc5664..773778d6 100644 --- a/frontend/src/components/llm/ProviderConfig.svelte +++ b/frontend/src/components/llm/ProviderConfig.svelte @@ -13,6 +13,7 @@ import { t } from "../../lib/i18n"; import { requestApi } from "../../lib/api"; import { addToast } from "../../lib/toasts.js"; + import HelpTooltip from "$lib/ui/HelpTooltip.svelte"; let { providers = [], @@ -37,6 +38,7 @@ default_model: "gpt-4o", is_active: true, is_multimodal: false, + max_images: null, }); let testStatus = $state({ type: "", message: "" }); @@ -51,6 +53,11 @@ let fetchDebounceTimer = $state(null); + // Probing state + let isProbing = $state(false); + let probeResult = $state(null); // { max_images, method } or null + let probeError = $state(""); + function isMaskedKey(key) { if (!key) return false; return key === "********" || key.includes("..."); @@ -75,11 +82,14 @@ default_model: "gpt-4o", is_active: true, is_multimodal: false, + max_images: null, }; editingProvider = null; testStatus = { type: "", message: "" }; availableModels = []; modelsLoadError = ""; + probeResult = null; + probeError = ""; } function handleEdit(provider) { @@ -97,10 +107,13 @@ default_model: provider?.default_model ?? "gpt-4o", is_active: Boolean(provider?.is_active), is_multimodal: Boolean(provider?.is_multimodal), + max_images: provider?.max_images ?? null, }; testStatus = { type: "", message: "" }; availableModels = []; modelsLoadError = ""; + probeResult = null; + probeError = ""; showForm = true; // Auto-fetch models when editing an existing provider if (provider?.base_url) { @@ -205,6 +218,44 @@ } } + async function detectMaxImages() { + const providerId = editingProvider?.id; + if (!providerId) { + addToast($t.llm.save_before_probe, "warning"); + return; + } + + console.log("[ProviderConfig][Action] Detecting max images for", providerId); + isProbing = true; + probeError = ""; + probeResult = null; + + try { + const result = await requestApi(`/llm/providers/${providerId}/probe-max-images`, "POST"); + console.log("[ProviderConfig] probe result:", JSON.stringify(result)); + probeResult = result; + + if (result.max_images != null) { + addToast( + $t.llm.max_images_detected.replace("{n}", result.max_images), + "success" + ); + formData.max_images = result.max_images; + } else { + addToast( + $t.llm.no_max_images_limit, + "info" + ); + } + } catch (err) { + probeError = err.message || "Failed to detect max images"; + console.warn("[ProviderConfig] Probe failed:", err.message); + addToast(probeError, "error"); + } finally { + isProbing = false; + } + } + async function handleSubmit() { console.log("[ProviderConfig][Action] Submitting provider config"); const method = editingProvider ? "PUT" : "POST"; @@ -480,6 +531,48 @@ {$t.llm.multimodal_checkbox} + +
+
+
+ + {$t.llm.max_images_label} + + + {#if formData.max_images != null} + + {formData.max_images} + + {:else} + + {$t.llm.not_detected} + + {/if} +
+ +
+ {#if probeError} +
{probeError}
+ {/if} + {#if probeResult && probeResult.max_images != null} +
+ {$t.llm.detected_via.replace("{method}", probeResult.method === "parse_error" ? $t.llm.detection_method_error : $t.llm.detection_method_binary)} +
+ {/if} +
{#if testStatus.message} diff --git a/frontend/src/lib/i18n/locales/en/llm.json b/frontend/src/lib/i18n/locales/en/llm.json index 949bf0b8..7a91201b 100644 --- a/frontend/src/lib/i18n/locales/en/llm.json +++ b/frontend/src/lib/i18n/locales/en/llm.json @@ -33,5 +33,16 @@ "models_load_failed": "Failed to load models: {error}", "select_model": "Select a model...", "refresh_models": "Refresh models", - "multimodal_checkbox": "Multimodal (supports images)" + "multimodal_checkbox": "Multimodal (supports images)", + "max_images_label": "Max images per request", + "not_detected": "Not detected", + "detect_max_images": "Detect", + "detecting": "Detecting...", + "detected_via": "Detected via: {method}", + "detection_method_error": "error parsing", + "detection_method_binary": "binary search", + "save_before_probe": "Save the provider first to detect max images", + "max_images_detected": "Max images: {n}", + "no_max_images_limit": "No image limit detected", + "help_max_images": "Auto-detect the maximum number of images the LLM provider accepts per request. Some providers (e.g. Nvidia NeMo) limit to 8-10 images." } diff --git a/frontend/src/lib/i18n/locales/ru/llm.json b/frontend/src/lib/i18n/locales/ru/llm.json index f2006e77..4c2b0b42 100644 --- a/frontend/src/lib/i18n/locales/ru/llm.json +++ b/frontend/src/lib/i18n/locales/ru/llm.json @@ -33,5 +33,16 @@ "models_load_failed": "Не удалось загрузить модели: {error}", "select_model": "Выберите модель...", "refresh_models": "Обновить модели", - "multimodal_checkbox": "Мультимодальная (поддержка изображений)" + "multimodal_checkbox": "Мультимодальная (поддержка изображений)", + "max_images_label": "Макс. изображений за запрос", + "not_detected": "Не определен", + "detect_max_images": "Определить", + "detecting": "Определение...", + "detected_via": "Определено через: {method}", + "detection_method_error": "парсинг ошибки", + "detection_method_binary": "бинарный поиск", + "save_before_probe": "Сначала сохраните провайдера", + "max_images_detected": "Макс. изображений: {n}", + "no_max_images_limit": "Лимит изображений не обнаружен", + "help_max_images": "Автоматическое определение максимального количества изображений, которое LLM-провайдер принимает за один запрос. Некоторые провайдеры (например Nvidia NeMo) ограничивают до 8-10 изображений." }