ss-tools/backend/src/api/routes/assistant/_tool_llm_validation.py

# #region AssistantToolLlmValidation [C:3] [TYPE Module] [SEMANTICS assistant, tool, llm, validation, task]
# @defgroup AssistantApi Module group.
# @BRIEF Handler for the "run_llm_validation" tool — create a validation task (policy) and trigger a run.
# @LAYER API
# @RELATION DEPENDS_ON -> [AssistantToolRegistry]
# @RELATION DEPENDS_ON -> [ValidationTaskService]
# @RELATION DEPENDS_ON -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [TaskManager]
# @RATIONALE v2 replaces ad-hoc task dispatch with persistent validation tasks (policies) managed via
#            ValidationTaskService. This makes validations repeatable, schedulable, and auditable.
# @REJECTED Ad-hoc task dispatch rejected — single-use tasks cannot be re-triggered, scheduled, or
#           tracked across runs. Policy+Run model provides run history and aggregate status.

from __future__ import annotations

from typing import Any

from fastapi import HTTPException
from sqlalchemy.orm import Session

from src.core.config_manager import ConfigManager
from src.core.logger import belief_scope, logger
from src.core.task_manager import TaskManager
from src.schemas.auth import User
from src.schemas.validation import ValidationTaskCreate
from src.services.llm_provider import LLMProviderService
from src.services.validation_service import ValidationTaskService

from ._resolvers import (
    _resolve_dashboard_id_entity,
    _resolve_env_id,
    _resolve_provider_id,
)
from ._schemas import AssistantAction
from ._tool_registry import _check_any_permission, assistant_tool


# #region handle_run_llm_validation [C:4] [TYPE Function]
# @ingroup AssistantApi
@assistant_tool(
    operation="run_llm_validation",
    domain="llm",
    description="Create and run an LLM dashboard validation task — creates a validation policy and triggers an immediate run",
    optional_entities=["dashboard_ref", "environment", "provider"],
    risk_level="guarded",
    requires_confirmation=False,
    permission_checks=[("plugin:llm_dashboard_validation", "EXECUTE")],
)
@belief_scope("run_llm_validation")
async def handle_run_llm_validation(
    intent: dict[str, Any],
    current_user: User,
    task_manager: TaskManager,
    config_manager: ConfigManager,
    db: Session,
) -> tuple[str, str | None, list[AssistantAction]]:
    """Create a validation task (policy) and trigger an immediate run."""
    _check_any_permission(current_user, [("plugin:llm_dashboard_validation", "EXECUTE")])
    entities = intent.get("entities", {})
    env_token = entities.get("environment")
    env_id = _resolve_env_id(env_token, config_manager) or _resolve_env_id(
        None, config_manager
    )
    dashboard_id = await _resolve_dashboard_id_entity(
        entities, config_manager, env_hint=env_token
    )
    provider_id = _resolve_provider_id(
        entities.get("provider"),
        db,
        config_manager=config_manager,
        task_key="dashboard_validation",
    )
    if not dashboard_id or not env_id or (not provider_id):
        raise HTTPException(
            status_code=422,
            detail="Missing dashboard_id/environment/provider. Укажите ID/slug дашборда или окружение.",
        )

    # Create a validation task (policy) via ValidationTaskService
    validation_svc = ValidationTaskService(
        db=db, config_manager=config_manager, username=current_user.username
    )
    task_payload = ValidationTaskCreate(
        name=f"LLM Validation - Dashboard {dashboard_id}",
        environment_id=env_id,
        dashboard_ids=[str(dashboard_id)],
        provider_id=provider_id,
        screenshot_enabled=True,
    )
    try:
        task = validation_svc.create_task(task_payload)
    except ValueError as e:
        raise HTTPException(status_code=422, detail=str(e))
    logger.info(
        "[AssistantToolLlmValidation] Created validation task '%s' (id=%s)",
        task.name, task.id,
    )

    # Trigger an immediate run
    try:
        result = await validation_svc.trigger_run(task.id, task_manager)
    except ValueError as e:
        raise HTTPException(status_code=422, detail=str(e))
    spawned_task_id = result["task_id"]
    logger.info(
        "[AssistantToolLlmValidation] Triggered run for task '%s' spawned=%s run=%s",
        task.name, spawned_task_id, result.get("run_id"),
    )

    return (
        f"LLM-валидация запущена. task_id={spawned_task_id}",
        spawned_task_id,
        [
            AssistantAction(type="open_task", label="Open Task", target=spawned_task_id),
            AssistantAction(
                type="open_reports", label="Open Reports", target="/reports"
            ),
        ],
    )


# #endregion handle_run_llm_validation
# #endregion AssistantToolLlmValidation