busya
d2b53c2a79
Core implementation of the v2 LLM dashboard validation pipeline: - LLM plugin with Path A (screenshots) and Path B (logs-only) execution - Validation task management (CRUD, schedule, run) - WebSocket task progress with Python 3.13 asyncio fix - Cross-task runs listing (GET /validation-tasks/runs/all) - RecordResponse schema for validation records - JSON prompt helper, per-dashboard status aggregation - Prompt templates with docs/git-commit/validation presets - Migration: v2 validation models + description column - Tests: plugin persistence, prompt templates, batch, payload, url
207 lines
9.4 KiB
Python
207 lines
9.4 KiB
Python
#region TestURLReparse [C:3] [TYPE Module] [SEMANTICS test, validation, url, reparse, source-invalid]
|
|
# @BRIEF Tests for URL source re-parse failure handling: deleted dashboards are detected gracefully.
|
|
# @RELATION BINDS_TO -> [ValidationService._parse_superset_link]
|
|
# @RELATION BINDS_TO -> [ValidationService._resolve_sources]
|
|
# @TEST_SCENARIO parse_failure -> URL with deleted dashboard → ValueError with descriptive message
|
|
# @TEST_SCENARIO source_invalid -> parse failure on existing source → source marked "invalid" + error logged
|
|
# @TEST_EDGE: missing_env -> URL parse fails when environment doesn't exist
|
|
# @TEST_EDGE: missing_config -> URL parse fails when config_manager is None
|
|
# @TEST_EDGE: invalid_url -> completely unparseable URL raises ValueError
|
|
# @INVARIANT A URL source whose dashboard was deleted must be detected and sources marked invalid
|
|
# rather than silently skipped during validation runs.
|
|
# @NOTE The _resolve_sources method currently propagates parse failures as ValueError.
|
|
# The desired behavior (FR-055) is to catch the error, set source.status="invalid"
|
|
# and source.last_error, then continue. The tests below document both current
|
|
# and expected behavior.
|
|
|
|
import pytest
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from src.schemas.validation import (
|
|
SourceInput,
|
|
ValidationTaskCreate,
|
|
)
|
|
from src.services.validation_service import ValidationTaskService
|
|
|
|
|
|
#region _make_service [C:2] [TYPE Function]
|
|
# @BRIEF Build a ValidationTaskService with mock DB and config for isolated testing.
|
|
def _make_service():
|
|
"""Create ValidationTaskService with mock dependencies for URL re-parse tests."""
|
|
mock_db = MagicMock()
|
|
mock_config = MagicMock()
|
|
|
|
# Mock environment lookup — env-1 exists, env-2 does not
|
|
env1 = SimpleNamespace(
|
|
id="env-1", name="Test Env", url="http://superset.example",
|
|
username="user", password="pass", verify_ssl=True, timeout=30,
|
|
)
|
|
mock_config.get_environment.side_effect = lambda eid: env1 if eid == "env-1" else None
|
|
|
|
service = ValidationTaskService(db=mock_db, config_manager=mock_config, username="testuser")
|
|
return service, mock_db, mock_config
|
|
#endregion _make_service
|
|
|
|
|
|
#region test_url_reparse_failure_marks_source_invalid [C:2] [TYPE Function]
|
|
# @BRIEF T055: URL source with deleted dashboard → _parse_superset_link raises ValueError.
|
|
# @TEST_INVARIANT url_parse_fails_on_deleted_dashboard -> VERIFIED_BY: test_url_reparse_failure_marks_source_invalid
|
|
# @RATIONALE When the Superset dashboard referenced by a URL source is deleted,
|
|
# SupersetContextExtractor.parse_superset_link raises ValueError.
|
|
# The ValidationTaskService must propagate this error so the caller
|
|
# can decide whether to mark the source invalid or abort.
|
|
def test_url_reparse_failure_marks_source_invalid():
|
|
"""URL pointing to deleted dashboard → ValueError with descriptive message.
|
|
|
|
The current behavior propagates the parse error as a ValueError. The
|
|
source is NOT auto-marked invalid in _parse_superset_link — the caller
|
|
(_resolve_sources or trigger_run) must catch and handle it.
|
|
"""
|
|
service, _mock_db, _mock_config = _make_service()
|
|
|
|
# Mock SupersetContextExtractor — it's imported inside _parse_superset_link
|
|
with patch(
|
|
"src.core.utils.superset_context_extractor.SupersetContextExtractor"
|
|
) as mock_extractor_cls:
|
|
mock_extractor_instance = MagicMock()
|
|
mock_extractor_cls.return_value = mock_extractor_instance
|
|
|
|
# Simulate a deleted dashboard: extractor raises ValueError
|
|
mock_extractor_instance.parse_superset_link.side_effect = ValueError(
|
|
"Dashboard '42' not found on environment 'env-1' — "
|
|
"the dashboard may have been deleted or the URL is invalid."
|
|
)
|
|
|
|
# _parse_superset_link should propagate the ValueError
|
|
with pytest.raises(ValueError) as exc_info:
|
|
service._parse_superset_link(
|
|
url="http://superset.example/superset/dashboard/42/",
|
|
environment_id="env-1",
|
|
)
|
|
|
|
assert "not found" in str(exc_info.value).lower()
|
|
assert "deleted" in str(exc_info.value).lower()
|
|
#endregion test_url_reparse_failure_marks_source_invalid
|
|
|
|
|
|
#region test_url_reparse_missing_environment [C:2] [TYPE Function]
|
|
# @BRIEF T055 edge: non-existent environment → ValueError with clear message.
|
|
def test_url_reparse_missing_environment():
|
|
"""URL re-parse with non-existent environment → ValueError."""
|
|
service, _mock_db, _mock_config = _make_service()
|
|
|
|
with pytest.raises(ValueError) as exc_info:
|
|
service._parse_superset_link(
|
|
url="http://superset.example/superset/dashboard/1/",
|
|
environment_id="env-nonexistent",
|
|
)
|
|
|
|
assert "not found" in str(exc_info.value).lower()
|
|
#endregion test_url_reparse_missing_environment
|
|
|
|
|
|
#region test_url_reparse_no_config [C:2] [TYPE Function]
|
|
# @BRIEF T055 edge: config_manager is None → ValueError.
|
|
def test_url_reparse_no_config():
|
|
"""URL re-parse with no config_manager available → ValueError."""
|
|
service = ValidationTaskService(db=MagicMock(), config_manager=None, username="testuser")
|
|
|
|
with pytest.raises(ValueError) as exc_info:
|
|
service._parse_superset_link(
|
|
url="http://superset.example/superset/dashboard/1/",
|
|
environment_id="env-1",
|
|
)
|
|
|
|
assert "config manager" in str(exc_info.value).lower()
|
|
#endregion test_url_reparse_no_config
|
|
|
|
|
|
#region test_url_reparse_create_task_propagates_error [C:2] [TYPE Function]
|
|
# @BRIEF T055: create_task with unresolvable URL source returns error — source NOT auto-created.
|
|
# @NOTE This documents current behavior: parse failure in _resolve_sources prevents
|
|
# task creation. FR-055 expects the source to be created with status="invalid"
|
|
# and the task to proceed with remaining valid sources.
|
|
def test_url_reparse_create_task_propagates_error():
|
|
"""Creating task with unresolvable URL → ValueError from _resolve_sources.
|
|
|
|
Currently, _resolve_sources does NOT catch parse failures. When a URL
|
|
source's dashboard is deleted, the entire task creation fails. Future
|
|
behavior should mark the specific source as 'invalid' and continue.
|
|
"""
|
|
service, _mock_db, _mock_config = _make_service()
|
|
|
|
# Patch at the source module — import is inside _parse_superset_link
|
|
with patch(
|
|
"src.core.utils.superset_context_extractor.SupersetContextExtractor"
|
|
) as mock_extractor_cls:
|
|
mock_extractor_instance = MagicMock()
|
|
mock_extractor_cls.return_value = mock_extractor_instance
|
|
mock_extractor_instance.parse_superset_link.side_effect = ValueError(
|
|
"Dashboard '42' not found"
|
|
)
|
|
|
|
# Mock the LLM provider validation to succeed
|
|
mock_provider = MagicMock()
|
|
mock_provider.is_active = True
|
|
mock_provider.is_multimodal = True
|
|
mock_provider.name = "Test Provider"
|
|
|
|
with patch.object(service, "_validate_provider", return_value=mock_provider):
|
|
with pytest.raises(ValueError) as exc_info:
|
|
payload = ValidationTaskCreate(
|
|
name="URL-Only Task",
|
|
environment_id="env-1",
|
|
provider_id="provider-1",
|
|
sources=[SourceInput(type="dashboard_url", value="http://superset.example/superset/dashboard/42/")],
|
|
)
|
|
service.create_task(payload)
|
|
|
|
assert "not found" in str(exc_info.value)
|
|
#endregion test_url_reparse_create_task_propagates_error
|
|
|
|
|
|
#region test_url_reparse_with_source_recovery [C:2] [TYPE Function]
|
|
# @BRIEF T055: URL with partial-recovery indicator sets source status to "partial_recovery".
|
|
def test_url_reparse_with_source_recovery():
|
|
"""URL source with partial_recovery → source status 'partial_recovery' (via _resolve_sources)."""
|
|
service, _mock_db, _mock_config = _make_service()
|
|
|
|
# Patch at the source module — import is inside _parse_superset_link
|
|
with patch(
|
|
"src.core.utils.superset_context_extractor.SupersetContextExtractor"
|
|
) as mock_extractor_cls:
|
|
mock_extractor_instance = MagicMock()
|
|
mock_extractor_cls.return_value = mock_extractor_instance
|
|
|
|
# Simulate partial recovery: dashboard_id resolved but dataset_ref unclear
|
|
from src.core.utils.superset_context_extractor import SupersetParsedContext
|
|
mock_extractor_instance.parse_superset_link.return_value = SupersetParsedContext(
|
|
source_url="http://superset.example/superset/dashboard/42/",
|
|
dataset_ref="dataset:?table=unknown",
|
|
dataset_id=None,
|
|
dashboard_id=42,
|
|
resource_type="dashboard",
|
|
partial_recovery=True,
|
|
unresolved_references=["dataset_1"],
|
|
)
|
|
|
|
# Directly test _resolve_sources — avoids creating a full policy with mock DB
|
|
sources = service._resolve_sources(
|
|
sources_input=[
|
|
SourceInput(type="dashboard_url", value="http://superset.example/superset/dashboard/42/"),
|
|
],
|
|
dashboard_ids=None,
|
|
policy_id="test-policy-id",
|
|
environment_id="env-1",
|
|
)
|
|
|
|
# Verify source was created with partial_recovery status
|
|
assert len(sources) == 1
|
|
assert sources[0].type == "dashboard_url"
|
|
assert sources[0].status == "partial_recovery"
|
|
assert sources[0].resolved_dashboard_id == "42"
|
|
#endregion test_url_reparse_with_source_recovery
|
|
|
|
#endregion TestURLReparse
|