feat(validation): v2 LLM dashboard validation — plugin, routes, services

Core implementation of the v2 LLM dashboard validation pipeline:
- LLM plugin with Path A (screenshots) and Path B (logs-only) execution
- Validation task management (CRUD, schedule, run)
- WebSocket task progress with Python 3.13 asyncio fix
- Cross-task runs listing (GET /validation-tasks/runs/all)
- RecordResponse schema for validation records
- JSON prompt helper, per-dashboard status aggregation
- Prompt templates with docs/git-commit/validation presets
- Migration: v2 validation models + description column
- Tests: plugin persistence, prompt templates, batch, payload, url
This commit is contained in:
2026-05-31 22:32:20 +03:00
parent 963843c2f6
commit cd2ff6c745
29 changed files with 5276 additions and 863 deletions

View File

@@ -1,9 +1,14 @@
# #region AssistantToolLlmValidation [C:3] [TYPE Module] [SEMANTICS assistant, tool, llm, validation]
# @BRIEF Handler for the "run_llm_validation" tool — run LLM dashboard validation.
# #region AssistantToolLlmValidation [C:3] [TYPE Module] [SEMANTICS assistant, tool, llm, validation, task]
# @BRIEF Handler for the "run_llm_validation" tool — create a validation task (policy) and trigger a run.
# @LAYER API
# @RELATION DEPENDS_ON -> [AssistantToolRegistry]
# @RELATION DEPENDS_ON -> [ValidationTaskService]
# @RELATION DEPENDS_ON -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [TaskManager]
# @RATIONALE v2 replaces ad-hoc task dispatch with persistent validation tasks (policies) managed via
# ValidationTaskService. This makes validations repeatable, schedulable, and auditable.
# @REJECTED Ad-hoc task dispatch rejected — single-use tasks cannot be re-triggered, scheduled, or
# tracked across runs. Policy+Run model provides run history and aggregate status.
from __future__ import annotations
@@ -16,7 +21,9 @@ from src.core.config_manager import ConfigManager
from src.core.logger import belief_scope, logger
from src.core.task_manager import TaskManager
from src.schemas.auth import User
from src.schemas.validation import ValidationTaskCreate
from src.services.llm_provider import LLMProviderService
from src.services.validation_service import ValidationTaskService
from ._resolvers import (
_resolve_dashboard_id_entity,
@@ -27,11 +34,11 @@ from ._schemas import AssistantAction
from ._tool_registry import _check_any_permission, assistant_tool
# #region handle_run_llm_validation [C:3] [TYPE Function]
# #region handle_run_llm_validation [C:4] [TYPE Function]
@assistant_tool(
operation="run_llm_validation",
domain="llm",
description="Run LLM dashboard validation by dashboard id/slug/title",
description="Create and run an LLM dashboard validation task — creates a validation policy and triggers an immediate run",
optional_entities=["dashboard_ref", "environment", "provider"],
risk_level="guarded",
requires_confirmation=False,
@@ -45,7 +52,7 @@ async def handle_run_llm_validation(
config_manager: ConfigManager,
db: Session,
) -> tuple[str, str | None, list[AssistantAction]]:
"""Run LLM dashboard validation."""
"""Create a validation task (policy) and trigger an immediate run."""
_check_any_permission(current_user, [("plugin:llm_dashboard_validation", "EXECUTE")])
entities = intent.get("entities", {})
env_token = entities.get("environment")
@@ -66,26 +73,43 @@ async def handle_run_llm_validation(
status_code=422,
detail="Missing dashboard_id/environment/provider. Укажите ID/slug дашборда или окружение.",
)
provider = LLMProviderService(db).get_provider(provider_id)
if not provider or not bool(provider.is_multimodal):
raise HTTPException(
status_code=422,
detail="Selected provider model is not multimodal for dashboard validation. Выберите мультимодальную модель (например, gpt-4o).",
)
task = await task_manager.create_task(
plugin_id="llm_dashboard_validation",
params={
"dashboard_id": str(dashboard_id),
"environment_id": env_id,
"provider_id": provider_id,
},
user_id=current_user.id,
# Create a validation task (policy) via ValidationTaskService
validation_svc = ValidationTaskService(
db=db, config_manager=config_manager, username=current_user.username
)
task_payload = ValidationTaskCreate(
name=f"LLM Validation - Dashboard {dashboard_id}",
environment_id=env_id,
dashboard_ids=[str(dashboard_id)],
provider_id=provider_id,
screenshot_enabled=True,
)
try:
task = validation_svc.create_task(task_payload)
except ValueError as e:
raise HTTPException(status_code=422, detail=str(e))
logger.info(
"[AssistantToolLlmValidation] Created validation task '%s' (id=%s)",
task.name, task.id,
)
# Trigger an immediate run
try:
result = await validation_svc.trigger_run(task.id, task_manager)
except ValueError as e:
raise HTTPException(status_code=422, detail=str(e))
spawned_task_id = result["task_id"]
logger.info(
"[AssistantToolLlmValidation] Triggered run for task '%s' spawned=%s run=%s",
task.name, spawned_task_id, result.get("run_id"),
)
return (
f"LLM-валидация запущена. task_id={task.id}",
task.id,
f"LLM-валидация запущена. task_id={spawned_task_id}",
spawned_task_id,
[
AssistantAction(type="open_task", label="Open Task", target=task.id),
AssistantAction(type="open_task", label="Open Task", target=spawned_task_id),
AssistantAction(
type="open_reports", label="Open Reports", target="/reports"
),