feat(validation): v2 LLM dashboard validation — plugin, routes, services
Core implementation of the v2 LLM dashboard validation pipeline: - LLM plugin with Path A (screenshots) and Path B (logs-only) execution - Validation task management (CRUD, schedule, run) - WebSocket task progress with Python 3.13 asyncio fix - Cross-task runs listing (GET /validation-tasks/runs/all) - RecordResponse schema for validation records - JSON prompt helper, per-dashboard status aggregation - Prompt templates with docs/git-commit/validation presets - Migration: v2 validation models + description column - Tests: plugin persistence, prompt templates, batch, payload, url
This commit is contained in:
@@ -1,9 +1,14 @@
|
||||
# #region AssistantToolLlmValidation [C:3] [TYPE Module] [SEMANTICS assistant, tool, llm, validation]
|
||||
# @BRIEF Handler for the "run_llm_validation" tool — run LLM dashboard validation.
|
||||
# #region AssistantToolLlmValidation [C:3] [TYPE Module] [SEMANTICS assistant, tool, llm, validation, task]
|
||||
# @BRIEF Handler for the "run_llm_validation" tool — create a validation task (policy) and trigger a run.
|
||||
# @LAYER API
|
||||
# @RELATION DEPENDS_ON -> [AssistantToolRegistry]
|
||||
# @RELATION DEPENDS_ON -> [ValidationTaskService]
|
||||
# @RELATION DEPENDS_ON -> [LLMProviderService]
|
||||
# @RELATION DEPENDS_ON -> [TaskManager]
|
||||
# @RATIONALE v2 replaces ad-hoc task dispatch with persistent validation tasks (policies) managed via
|
||||
# ValidationTaskService. This makes validations repeatable, schedulable, and auditable.
|
||||
# @REJECTED Ad-hoc task dispatch rejected — single-use tasks cannot be re-triggered, scheduled, or
|
||||
# tracked across runs. Policy+Run model provides run history and aggregate status.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -16,7 +21,9 @@ from src.core.config_manager import ConfigManager
|
||||
from src.core.logger import belief_scope, logger
|
||||
from src.core.task_manager import TaskManager
|
||||
from src.schemas.auth import User
|
||||
from src.schemas.validation import ValidationTaskCreate
|
||||
from src.services.llm_provider import LLMProviderService
|
||||
from src.services.validation_service import ValidationTaskService
|
||||
|
||||
from ._resolvers import (
|
||||
_resolve_dashboard_id_entity,
|
||||
@@ -27,11 +34,11 @@ from ._schemas import AssistantAction
|
||||
from ._tool_registry import _check_any_permission, assistant_tool
|
||||
|
||||
|
||||
# #region handle_run_llm_validation [C:3] [TYPE Function]
|
||||
# #region handle_run_llm_validation [C:4] [TYPE Function]
|
||||
@assistant_tool(
|
||||
operation="run_llm_validation",
|
||||
domain="llm",
|
||||
description="Run LLM dashboard validation by dashboard id/slug/title",
|
||||
description="Create and run an LLM dashboard validation task — creates a validation policy and triggers an immediate run",
|
||||
optional_entities=["dashboard_ref", "environment", "provider"],
|
||||
risk_level="guarded",
|
||||
requires_confirmation=False,
|
||||
@@ -45,7 +52,7 @@ async def handle_run_llm_validation(
|
||||
config_manager: ConfigManager,
|
||||
db: Session,
|
||||
) -> tuple[str, str | None, list[AssistantAction]]:
|
||||
"""Run LLM dashboard validation."""
|
||||
"""Create a validation task (policy) and trigger an immediate run."""
|
||||
_check_any_permission(current_user, [("plugin:llm_dashboard_validation", "EXECUTE")])
|
||||
entities = intent.get("entities", {})
|
||||
env_token = entities.get("environment")
|
||||
@@ -66,26 +73,43 @@ async def handle_run_llm_validation(
|
||||
status_code=422,
|
||||
detail="Missing dashboard_id/environment/provider. Укажите ID/slug дашборда или окружение.",
|
||||
)
|
||||
provider = LLMProviderService(db).get_provider(provider_id)
|
||||
if not provider or not bool(provider.is_multimodal):
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail="Selected provider model is not multimodal for dashboard validation. Выберите мультимодальную модель (например, gpt-4o).",
|
||||
)
|
||||
task = await task_manager.create_task(
|
||||
plugin_id="llm_dashboard_validation",
|
||||
params={
|
||||
"dashboard_id": str(dashboard_id),
|
||||
"environment_id": env_id,
|
||||
"provider_id": provider_id,
|
||||
},
|
||||
user_id=current_user.id,
|
||||
|
||||
# Create a validation task (policy) via ValidationTaskService
|
||||
validation_svc = ValidationTaskService(
|
||||
db=db, config_manager=config_manager, username=current_user.username
|
||||
)
|
||||
task_payload = ValidationTaskCreate(
|
||||
name=f"LLM Validation - Dashboard {dashboard_id}",
|
||||
environment_id=env_id,
|
||||
dashboard_ids=[str(dashboard_id)],
|
||||
provider_id=provider_id,
|
||||
screenshot_enabled=True,
|
||||
)
|
||||
try:
|
||||
task = validation_svc.create_task(task_payload)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=422, detail=str(e))
|
||||
logger.info(
|
||||
"[AssistantToolLlmValidation] Created validation task '%s' (id=%s)",
|
||||
task.name, task.id,
|
||||
)
|
||||
|
||||
# Trigger an immediate run
|
||||
try:
|
||||
result = await validation_svc.trigger_run(task.id, task_manager)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=422, detail=str(e))
|
||||
spawned_task_id = result["task_id"]
|
||||
logger.info(
|
||||
"[AssistantToolLlmValidation] Triggered run for task '%s' spawned=%s run=%s",
|
||||
task.name, spawned_task_id, result.get("run_id"),
|
||||
)
|
||||
|
||||
return (
|
||||
f"LLM-валидация запущена. task_id={task.id}",
|
||||
task.id,
|
||||
f"LLM-валидация запущена. task_id={spawned_task_id}",
|
||||
spawned_task_id,
|
||||
[
|
||||
AssistantAction(type="open_task", label="Open Task", target=task.id),
|
||||
AssistantAction(type="open_task", label="Open Task", target=spawned_task_id),
|
||||
AssistantAction(
|
||||
type="open_reports", label="Open Reports", target="/reports"
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user