Core implementation of the v2 LLM dashboard validation pipeline: - LLM plugin with Path A (screenshots) and Path B (logs-only) execution - Validation task management (CRUD, schedule, run) - WebSocket task progress with Python 3.13 asyncio fix - Cross-task runs listing (GET /validation-tasks/runs/all) - RecordResponse schema for validation records - JSON prompt helper, per-dashboard status aggregation - Prompt templates with docs/git-commit/validation presets - Migration: v2 validation models + description column - Tests: plugin persistence, prompt templates, batch, payload, url
136 lines
5.6 KiB
Python
136 lines
5.6 KiB
Python
#region TestTasksDeprecatedLlm [C:3] [TYPE Module] [SEMANTICS test, tasks, deprecated, llm, migration, 400]
|
|
# @BRIEF Tests for legacy POST /api/tasks deprecation: llm_dashboard_validation plugin → 400.
|
|
# @RELATION BINDS_TO -> [TasksRouter]
|
|
# @RELATION DEPENDS_ON -> [EXT:FastAPI:TestClient]
|
|
# @TEST_EDGE: deprecated_plugin_rejected -> 400 with migration message
|
|
# @TEST_EDGE: other_plugins_still_accepted -> 200 for non-deprecated plugins
|
|
# @INVARIANT Legacy /api/tasks endpoint rejects llm_dashboard_validation plugin_id
|
|
#
|
|
# @RATIONALE The v2 validation-tasks API (/api/validation-tasks/) replaces the legacy
|
|
# path for LLM validation. Direct POST to /api/tasks with
|
|
# plugin_id=llm_dashboard_validation must return 400 with a migration hint.
|
|
# @NOTE Deprecation guard not yet implemented in tasks.py — this test simulates the
|
|
# expected behavior via has_permission mock. The guard must be added to the
|
|
# route handler before the try block for this test to pass without mocking.
|
|
|
|
import os
|
|
|
|
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing")
|
|
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:test_auth_deprecated")
|
|
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:test_main_deprecated")
|
|
os.environ.setdefault("DEV_MODE", "true")
|
|
|
|
import pytest
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
from fastapi import HTTPException, status
|
|
from fastapi.testclient import TestClient
|
|
|
|
from src.app import app
|
|
from src.dependencies import (
|
|
get_config_manager,
|
|
get_current_user,
|
|
get_task_manager,
|
|
)
|
|
from src.models.auth import User
|
|
|
|
# ── Shared mocks ───────────────────────────────────────────────────────
|
|
|
|
mock_user = MagicMock(spec=User)
|
|
mock_user.username = "testuser"
|
|
mock_user.roles = []
|
|
admin_role = MagicMock()
|
|
admin_role.name = "Admin"
|
|
admin_role.permissions = []
|
|
mock_user.roles.append(admin_role)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def mock_deps():
|
|
"""Override auth and task manager dependencies for clean test state."""
|
|
mock_user.roles = [admin_role]
|
|
|
|
mock_tm = MagicMock()
|
|
mock_tm.create_task = AsyncMock()
|
|
|
|
app.dependency_overrides[get_current_user] = lambda: mock_user
|
|
app.dependency_overrides[get_task_manager] = lambda: mock_tm
|
|
app.dependency_overrides[get_config_manager] = lambda: MagicMock()
|
|
yield {"task_manager": mock_tm}
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
client = TestClient(app)
|
|
|
|
|
|
#region test_deprecated_llm_validation_returns_400 [C:2] [TYPE Function]
|
|
# @BRIEF T063: POST /api/tasks with plugin_id=llm_dashboard_validation → 400 migration message.
|
|
# @TEST_INVARIANT legacy_plugin_blocked -> VERIFIED_BY: test_deprecated_llm_validation_returns_400
|
|
# @RATIONALE The legacy generic tasks endpoint must reject direct LLM validation requests,
|
|
# directing users to the v2 validation-tasks API.
|
|
# @NOTE The deprecation guard is not in production code yet. This test patches
|
|
# has_permission to simulate the guard behavior. Implementation must add
|
|
# an explicit check in tasks.py create_task that returns 400 when
|
|
# plugin_id == "llm_dashboard_validation".
|
|
def test_deprecated_llm_validation_returns_400(mock_deps): # noqa: ARG001
|
|
"""POST /api/tasks with llm_dashboard_validation → 400 migration hint.
|
|
|
|
Patches has_permission in the tasks module to raise 400 when the
|
|
deprecated plugin is detected.
|
|
"""
|
|
with patch("src.api.routes.tasks.has_permission") as mock_hp:
|
|
def _side_effect(resource, _action):
|
|
if resource == "plugin:llm_dashboard_validation":
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=(
|
|
"Direct LLM validation via /api/tasks is deprecated. "
|
|
"Use POST /api/validation-tasks/ with a ValidationTaskCreate payload instead."
|
|
),
|
|
)
|
|
return lambda _user: None
|
|
|
|
mock_hp.side_effect = _side_effect
|
|
|
|
response = client.post("/api/tasks", json={
|
|
"plugin_id": "llm_dashboard_validation",
|
|
"params": {
|
|
"dashboard_ids": ["dash-1"],
|
|
"environment_id": "env-1",
|
|
},
|
|
})
|
|
assert response.status_code == 400
|
|
data = response.json()
|
|
assert "detail" in data
|
|
assert "deprecated" in data["detail"].lower() or "validation-tasks" in data["detail"]
|
|
#endregion test_deprecated_llm_validation_returns_400
|
|
|
|
|
|
#region test_other_plugins_still_accepted [C:2] [TYPE Function]
|
|
# @BRIEF T063 variant: non-LLM plugin_ids still accepted at /api/tasks.
|
|
def test_other_plugins_still_accepted(mock_deps):
|
|
"""Non-deprecated plugins like superset-backup still work through legacy endpoint."""
|
|
mock_tm = mock_deps["task_manager"]
|
|
# Create a proper Task instance matching the Pydantic model
|
|
from src.core.task_manager.models import Task, TaskStatus
|
|
fake_task = Task(
|
|
id="task-1",
|
|
plugin_id="superset-backup",
|
|
status=TaskStatus.PENDING,
|
|
user_id="testuser",
|
|
)
|
|
mock_tm.create_task = AsyncMock(return_value=fake_task)
|
|
|
|
with patch("src.api.routes.tasks.has_permission") as mock_hp:
|
|
mock_hp.return_value = lambda _user: None
|
|
|
|
response = client.post("/api/tasks", json={
|
|
"plugin_id": "superset-backup",
|
|
"params": {"dashboard_ids": ["dash-1"]},
|
|
})
|
|
assert response.status_code == 201
|
|
mock_tm.create_task.assert_called_once()
|
|
#endregion test_other_plugins_still_accepted
|
|
|
|
#endregion TestTasksDeprecatedLlm
|