315 lines
15 KiB
Python
315 lines
15 KiB
Python
# #region Test.LLMAnalysisPlugin.Coverage [C:3] [TYPE Module] [SEMANTICS test,llm,plugin,coverage,edge]
|
|
# @BRIEF Coverage tests for plugin.py — Path A edge cases, DocumentationPlugin edge cases.
|
|
# @RELATION BINDS_TO -> [LLMAnalysisPlugin]
|
|
# @TEST_EDGE: path_a_with_context -> TaskContext provided to _execute_path_a
|
|
# @TEST_EDGE: path_a_notification_failure -> Notification failure is non-blocking
|
|
# @TEST_EDGE: docs_plugin_full_flow -> DocumentationPlugin full happy path
|
|
import json
|
|
from datetime import UTC, datetime
|
|
from unittest.mock import AsyncMock, MagicMock, patch, PropertyMock, ANY
|
|
|
|
import pytest
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
# Helpers
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
|
|
def _make_env():
|
|
"""Environment mock with all required attributes."""
|
|
env = MagicMock()
|
|
env.id = "env-1"
|
|
env.url = "https://superset.example.com"
|
|
env.username = "admin"
|
|
env.password = "pass"
|
|
return env
|
|
|
|
|
|
def _make_provider():
|
|
prov = MagicMock()
|
|
prov.id = "prov-1"
|
|
prov.name = "Test"
|
|
prov.provider_type = "openai"
|
|
prov.base_url = "https://api.openai.com"
|
|
prov.default_model = "gpt-4o"
|
|
prov.is_active = True
|
|
prov.is_multimodal = True
|
|
prov.max_images = 10
|
|
return prov
|
|
|
|
|
|
def _make_cm():
|
|
cm = MagicMock()
|
|
cfg = MagicMock()
|
|
cfg.settings.storage.root_path = "/tmp/storage"
|
|
cfg.settings.llm = {"prompts": {
|
|
"dashboard_validation_prompt_text": "Analyze {topology}",
|
|
"documentation_prompt": "Doc {dataset_name}: {columns_json}",
|
|
}}
|
|
cm.get_config.return_value = cfg
|
|
return cm
|
|
|
|
|
|
def _make_log():
|
|
log = MagicMock()
|
|
log.with_source.return_value = log
|
|
return log
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
# DashboardValidationPlugin — Path A edge cases
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
|
|
class TestExecutePathAExtended:
|
|
"""Additional _execute_path_a coverage: with TaskContext, notification failure."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_path_a_with_context(self):
|
|
"""TaskContext provided to _execute_path_a."""
|
|
from src.plugins.llm_analysis.plugin import DashboardValidationPlugin
|
|
|
|
plugin = DashboardValidationPlugin()
|
|
mock_db = MagicMock()
|
|
mock_db.add.return_value = None
|
|
mock_db.commit.return_value = None
|
|
env = _make_env()
|
|
prov = _make_provider()
|
|
cm = _make_cm()
|
|
log = _make_log()
|
|
|
|
mock_ctx = MagicMock()
|
|
mock_ctx.logger = log
|
|
mock_ctx.task_id = "task-42"
|
|
|
|
params = {
|
|
"dashboard_id": "42", "environment_id": "env-1",
|
|
"run_id": "run-42", "policy_id": "policy-1",
|
|
}
|
|
|
|
with patch('src.plugins.llm_analysis.plugin.ScreenshotService') as MockSS:
|
|
MockSS.return_value.capture_dashboard = AsyncMock(
|
|
return_value=(["/tmp/jpg.jpg"], [{"webp_path": "/tmp/archive.webp"}]))
|
|
|
|
with patch('src.plugins.llm_analysis.plugin.LLMClient') as MockLLM:
|
|
MockLLM.return_value.analyze_dashboard_multimodal = AsyncMock(
|
|
return_value={"status": "PASS", "summary": "OK", "issues": [], "chunk_count": 1})
|
|
|
|
with patch.object(plugin, '_fetch_dashboard_logs',
|
|
AsyncMock(return_value=(["log"], {"logs_fetch_duration_ms": 10}))):
|
|
with patch.object(MockSS, '_cleanup_temp_files'):
|
|
with patch('src.plugins.llm_analysis.plugin.NotificationService') as MockNotif:
|
|
MockNotif.return_value.dispatch_report = AsyncMock()
|
|
result = await plugin._execute_path_a(
|
|
params, mock_ctx, mock_db, prov, "sk-key", env, cm, log)
|
|
assert result["execution_path"] == "multimodal"
|
|
assert result["status"] == "PASS"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_path_a_no_jpegs_text_fallback(self):
|
|
"""No JPEGs captured → falls back to text batch."""
|
|
from src.plugins.llm_analysis.plugin import DashboardValidationPlugin
|
|
|
|
plugin = DashboardValidationPlugin()
|
|
mock_db = MagicMock()
|
|
mock_db.add.return_value = None
|
|
mock_db.commit.return_value = None
|
|
env = _make_env()
|
|
prov = _make_provider()
|
|
cm = _make_cm()
|
|
log = _make_log()
|
|
params = {"dashboard_id": "42", "environment_id": "env-1"}
|
|
|
|
with patch('src.plugins.llm_analysis.plugin.ScreenshotService') as MockSS:
|
|
MockSS.return_value.capture_dashboard = AsyncMock(return_value=([], []))
|
|
|
|
with patch('src.plugins.llm_analysis.plugin.LLMClient') as MockLLM:
|
|
MockLLM.return_value.analyze_dashboard_text_batch = AsyncMock(
|
|
return_value={"dashboards": [{"status": "WARN", "summary": "No screenshots", "issues": []}]})
|
|
|
|
with patch.object(plugin, '_fetch_dashboard_logs',
|
|
AsyncMock(return_value=(["log"], {"logs_fetch_duration_ms": 5}))):
|
|
with patch.object(MockSS, '_cleanup_temp_files'):
|
|
with patch('src.plugins.llm_analysis.plugin.NotificationService') as MockNotif:
|
|
MockNotif.return_value.dispatch_report = AsyncMock()
|
|
result = await plugin._execute_path_a(
|
|
params, None, mock_db, prov, "sk-key", env, cm, log)
|
|
assert result["status"] == "WARN"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_path_a_notification_failure_does_not_block(self):
|
|
"""Notification failure is non-blocking."""
|
|
from src.plugins.llm_analysis.plugin import DashboardValidationPlugin
|
|
|
|
plugin = DashboardValidationPlugin()
|
|
mock_db = MagicMock()
|
|
mock_db.add.return_value = None
|
|
mock_db.commit.return_value = None
|
|
env = _make_env()
|
|
prov = _make_provider()
|
|
cm = _make_cm()
|
|
log = _make_log()
|
|
params = {"dashboard_id": "42", "environment_id": "env-1"}
|
|
|
|
with patch('src.plugins.llm_analysis.plugin.ScreenshotService') as MockSS:
|
|
MockSS.return_value.capture_dashboard = AsyncMock(
|
|
return_value=(["/tmp/jpg"], [{"webp_path": "/tmp/webp"}]))
|
|
|
|
with patch('src.plugins.llm_analysis.plugin.LLMClient') as MockLLM:
|
|
MockLLM.return_value.analyze_dashboard_multimodal = AsyncMock(
|
|
return_value={"status": "PASS", "summary": "OK", "issues": [], "chunk_count": 1})
|
|
|
|
with patch.object(plugin, '_fetch_dashboard_logs',
|
|
AsyncMock(return_value=(["log"], {"logs_fetch_duration_ms": 5}))):
|
|
with patch.object(MockSS, '_cleanup_temp_files'):
|
|
with patch('src.plugins.llm_analysis.plugin.NotificationService') as MockNotif:
|
|
MockNotif.return_value.dispatch_report = AsyncMock(
|
|
side_effect=RuntimeError("Notif failed"))
|
|
result = await plugin._execute_path_a(
|
|
params, None, mock_db, prov, "sk-key", env, cm, log)
|
|
assert result["status"] == "PASS"
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
# DocumentationPlugin — full flow
|
|
# ═══════════════════════════════════════════════════════════════════
|
|
|
|
class TestDocumentationPluginExtended:
|
|
"""Verify DocumentationPlugin full flow and edge cases."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_docs_full_flow(self):
|
|
"""DocumentationPlugin.execute full happy path."""
|
|
from src.plugins.llm_analysis.plugin import DocumentationPlugin
|
|
|
|
plugin = DocumentationPlugin()
|
|
mock_db = MagicMock()
|
|
mock_db.add.return_value = None
|
|
mock_db.commit.return_value = None
|
|
mock_db.close.return_value = None
|
|
|
|
params = {"dataset_id": "101", "environment_id": "env-1", "provider_id": "prov-1"}
|
|
|
|
# For DocumentationPlugin, import is inside execute(): from ...core.superset_client import SupersetClient
|
|
with patch('src.plugins.llm_analysis.plugin.SessionLocal', return_value=mock_db):
|
|
with patch('src.dependencies.get_config_manager', return_value=_make_cm()):
|
|
with patch('src.plugins.llm_analysis.plugin.LLMProviderService') as MockPS:
|
|
MockPS.return_value.get_provider.return_value = _make_provider()
|
|
MockPS.return_value.get_decrypted_api_key.return_value = "sk-real-key-that-is-long-enough"
|
|
|
|
with patch('src.core.superset_client.SupersetClient') as MockSC:
|
|
mock_instance = MagicMock()
|
|
mock_instance.get_dataset = AsyncMock(return_value={
|
|
"table_name": "orders",
|
|
"columns": [
|
|
{"id": 1, "column_name": "amount", "type": "numeric", "description": None},
|
|
]})
|
|
mock_instance.update_dataset = AsyncMock()
|
|
MockSC.return_value = mock_instance
|
|
|
|
with patch('src.plugins.llm_analysis.plugin.LLMClient') as MockLLM:
|
|
MockLLM.return_value.get_json_completion = AsyncMock(return_value={
|
|
"dataset_description": "Orders dataset",
|
|
"column_descriptions": [{"name": "amount", "description": "Order amount"}],
|
|
})
|
|
|
|
result = await plugin.execute(params)
|
|
assert "dataset_description" in result
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_docs_env_not_found(self):
|
|
"""Environment not found raises ValueError."""
|
|
from src.plugins.llm_analysis.plugin import DocumentationPlugin
|
|
|
|
plugin = DocumentationPlugin()
|
|
mock_db = MagicMock()
|
|
mock_db.close.return_value = None
|
|
|
|
with patch('src.plugins.llm_analysis.plugin.SessionLocal', return_value=mock_db):
|
|
with patch('src.dependencies.get_config_manager') as mock_cm:
|
|
mock_cm.return_value.get_environment.return_value = None
|
|
with patch('src.plugins.llm_analysis.plugin.LLMProviderService') as MockPS:
|
|
MockPS.return_value.get_provider.return_value = MagicMock()
|
|
with pytest.raises(ValueError, match="env-1 not found"):
|
|
await plugin.execute({
|
|
"dataset_id": "101", "environment_id": "env-1", "provider_id": "prov-1",
|
|
})
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_docs_provider_not_found(self):
|
|
"""Provider not found raises ValueError."""
|
|
from src.plugins.llm_analysis.plugin import DocumentationPlugin
|
|
|
|
plugin = DocumentationPlugin()
|
|
mock_db = MagicMock()
|
|
mock_db.close.return_value = None
|
|
|
|
with patch('src.plugins.llm_analysis.plugin.SessionLocal', return_value=mock_db):
|
|
with patch('src.dependencies.get_config_manager') as mock_cm:
|
|
mock_cm.return_value.get_environment.return_value = MagicMock()
|
|
with patch('src.plugins.llm_analysis.plugin.LLMProviderService') as MockPS:
|
|
MockPS.return_value.get_provider.return_value = None
|
|
with pytest.raises(ValueError, match="prov-1 not found"):
|
|
await plugin.execute({
|
|
"dataset_id": "101", "environment_id": "env-1", "provider_id": "prov-1",
|
|
})
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_docs_invalid_api_key(self):
|
|
"""Invalid API key raises ValueError."""
|
|
from src.plugins.llm_analysis.plugin import DocumentationPlugin
|
|
|
|
plugin = DocumentationPlugin()
|
|
mock_db = MagicMock()
|
|
mock_db.close.return_value = None
|
|
|
|
with patch('src.plugins.llm_analysis.plugin.SessionLocal', return_value=mock_db):
|
|
with patch('src.dependencies.get_config_manager') as mock_cm:
|
|
mock_cm.return_value.get_environment.return_value = MagicMock()
|
|
with patch('src.plugins.llm_analysis.plugin.LLMProviderService') as MockPS:
|
|
MockPS.return_value.get_provider.return_value = MagicMock()
|
|
MockPS.return_value.get_decrypted_api_key.return_value = "********"
|
|
with pytest.raises(ValueError, match="Invalid API key"):
|
|
await plugin.execute({
|
|
"dataset_id": "101", "environment_id": "env-1", "provider_id": "prov-1",
|
|
})
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_docs_with_context(self):
|
|
"""DocumentationPlugin with TaskContext."""
|
|
from src.plugins.llm_analysis.plugin import DocumentationPlugin
|
|
|
|
plugin = DocumentationPlugin()
|
|
mock_db = MagicMock()
|
|
mock_db.add.return_value = None
|
|
mock_db.commit.return_value = None
|
|
mock_db.close.return_value = None
|
|
|
|
mock_ctx = MagicMock()
|
|
mock_ctx.logger = MagicMock()
|
|
mock_ctx.logger.with_source.return_value = mock_ctx.logger
|
|
mock_ctx.task_id = "task-101"
|
|
|
|
params = {"dataset_id": "101", "environment_id": "env-1", "provider_id": "prov-1"}
|
|
|
|
with patch('src.plugins.llm_analysis.plugin.SessionLocal', return_value=mock_db):
|
|
with patch('src.dependencies.get_config_manager', return_value=_make_cm()):
|
|
with patch('src.plugins.llm_analysis.plugin.LLMProviderService') as MockPS:
|
|
MockPS.return_value.get_provider.return_value = _make_provider()
|
|
MockPS.return_value.get_decrypted_api_key.return_value = "sk-real-key-that-is-long-enough"
|
|
|
|
with patch('src.core.superset_client.SupersetClient') as MockSC:
|
|
mock_instance = MagicMock()
|
|
mock_instance.get_dataset = AsyncMock(
|
|
return_value={"table_name": "t", "columns": []})
|
|
mock_instance.update_dataset = AsyncMock()
|
|
MockSC.return_value = mock_instance
|
|
|
|
with patch('src.plugins.llm_analysis.plugin.LLMClient') as MockLLM:
|
|
MockLLM.return_value.get_json_completion = AsyncMock(return_value={
|
|
"dataset_description": "Test", "column_descriptions": [],
|
|
})
|
|
|
|
result = await plugin.execute(params, context=mock_ctx)
|
|
assert "dataset_description" in result
|
|
# #endregion Test.LLMAnalysisPlugin.Coverage
|