496 lines
20 KiB
Python
496 lines
20 KiB
Python
# #region Test.Api.Llm [C:3] [TYPE Module] [SEMANTICS test,llm,routes]
|
|
# @BRIEF Tests for LLM API routes — providers, fetch-models, status, test, probe.
|
|
# @RELATION BINDS_TO -> [LlmRoutes]
|
|
|
|
import os
|
|
|
|
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:")
|
|
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:")
|
|
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-tests")
|
|
os.environ.setdefault("DEV_MODE", "true")
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
|
|
_src = str(Path(__file__).resolve().parent.parent.parent / "src")
|
|
if _src not in sys.path:
|
|
sys.path.insert(0, _src)
|
|
|
|
P = "/api/llm"
|
|
|
|
|
|
def _make_provider(**kw):
|
|
p = MagicMock()
|
|
p.id = kw.get("id", "prov-1")
|
|
p.provider_type = kw.get("provider_type", "openai")
|
|
p.name = kw.get("name", "Test Provider")
|
|
p.base_url = kw.get("base_url", "https://api.openai.com")
|
|
p.api_key = kw.get("api_key", "sk-test")
|
|
p.default_model = kw.get("default_model", "gpt-4")
|
|
p.is_active = kw.get("is_active", True)
|
|
p.is_multimodal = kw.get("is_multimodal", True)
|
|
p.max_images = kw.get("max_images", 10)
|
|
p.context_window = kw.get("context_window", 128000)
|
|
p.max_output_tokens = kw.get("max_output_tokens", 4096)
|
|
return p
|
|
|
|
|
|
def _make_client(overrides: dict | None = None) -> TestClient:
|
|
from src.api.routes.llm import router
|
|
from src.core.database import get_db
|
|
from src.dependencies import get_current_user
|
|
from src.schemas.auth import User, RoleSchema
|
|
|
|
app = FastAPI()
|
|
app.include_router(router, prefix="/api/llm")
|
|
|
|
mock_user = User(
|
|
id="admin-1", username="admin", email="admin@x.com",
|
|
auth_source="LOCAL",
|
|
created_at=__import__("datetime").datetime.now(),
|
|
roles=[RoleSchema(id="r1", name="Admin", description="", permissions=[])],
|
|
)
|
|
|
|
app.dependency_overrides[get_db] = lambda: MagicMock()
|
|
app.dependency_overrides[get_current_user] = lambda: mock_user
|
|
if overrides:
|
|
for dep, fn in overrides.items():
|
|
app.dependency_overrides[dep] = fn
|
|
return TestClient(app)
|
|
|
|
|
|
class TestGetProviders:
|
|
"""GET /api/llm/providers"""
|
|
|
|
def test_success(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_all_providers.return_value = [_make_provider()]
|
|
mock_svc.get_decrypted_api_key.return_value = "sk-real-key"
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.get(f"{P}/providers")
|
|
assert resp.status_code == 200
|
|
|
|
def test_empty(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_all_providers.return_value = []
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.get(f"{P}/providers")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
|
|
class TestFetchModels:
|
|
"""POST /api/llm/providers/fetch-models"""
|
|
|
|
def test_success_with_api_key(self):
|
|
mock_db = MagicMock()
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_models = AsyncMock(return_value=["gpt-4"])
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.plugins.llm_analysis.service.LLMClient", return_value=mock_client):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/fetch-models", json={
|
|
"base_url": "https://api.openai.com", "provider_type": "openai", "api_key": "sk-test",
|
|
})
|
|
assert resp.status_code == 200
|
|
assert "gpt-4" in resp.json()["models"]
|
|
|
|
def test_success_with_provider_id(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_provider.return_value = _make_provider()
|
|
mock_svc.get_decrypted_api_key.return_value = "sk-decrypted"
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_models = AsyncMock(return_value=["gpt-4"])
|
|
|
|
from src.core.database import get_db
|
|
with (
|
|
patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc),
|
|
patch("src.plugins.llm_analysis.service.LLMClient", return_value=mock_client),
|
|
):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/fetch-models", json={"provider_id": "prov-1"})
|
|
assert resp.status_code == 200
|
|
|
|
def test_missing_credentials(self):
|
|
client = _make_client()
|
|
resp = client.post(f"{P}/providers/fetch-models", json={
|
|
"base_url": "https://api.openai.com", "provider_type": "openai",
|
|
})
|
|
assert resp.status_code == 400
|
|
|
|
def test_provider_not_found(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_provider.return_value = None
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/fetch-models", json={"provider_id": "unknown"})
|
|
assert resp.status_code == 404
|
|
|
|
def test_missing_base_url(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_provider.return_value = _make_provider(base_url="")
|
|
mock_svc.get_decrypted_api_key.return_value = "sk-decrypted"
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/fetch-models", json={"provider_id": "prov-1"})
|
|
assert resp.status_code == 400
|
|
|
|
def test_api_failure(self):
|
|
mock_db = MagicMock()
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_models = AsyncMock(side_effect=RuntimeError("API error"))
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.plugins.llm_analysis.service.LLMClient", return_value=mock_client):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/fetch-models", json={
|
|
"base_url": "https://api.openai.com", "provider_type": "openai", "api_key": "sk-test",
|
|
})
|
|
assert resp.status_code == 502
|
|
|
|
|
|
class TestGetLlmStatus:
|
|
"""GET /api/llm/status"""
|
|
|
|
def test_configured(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_all_providers.return_value = [_make_provider()]
|
|
mock_svc.get_decrypted_api_key.return_value = "sk-real-valid-key-16chars-min"
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.get(f"{P}/status")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["configured"] is True
|
|
|
|
def test_no_providers(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_all_providers.return_value = []
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.get(f"{P}/status")
|
|
assert resp.json()["configured"] is False
|
|
|
|
def test_no_active_provider(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_all_providers.return_value = [_make_provider(is_active=False)]
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.get(f"{P}/status")
|
|
assert resp.json()["reason"] == "no_active_provider"
|
|
|
|
def test_invalid_api_key(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_all_providers.return_value = [_make_provider()]
|
|
mock_svc.get_decrypted_api_key.return_value = ""
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.get(f"{P}/status")
|
|
assert resp.json()["reason"] == "invalid_api_key"
|
|
|
|
|
|
class TestCreateProvider:
|
|
"""POST /api/llm/providers"""
|
|
|
|
def test_success(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.create_provider.return_value = _make_provider(id="new-prov")
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers", json={
|
|
"provider_type": "openai", "name": "New", "base_url": "https://x.com",
|
|
"api_key": "sk-new", "default_model": "gpt-4",
|
|
})
|
|
assert resp.status_code == 201
|
|
|
|
|
|
class TestUpdateProvider:
|
|
"""PUT /api/llm/providers/{provider_id}"""
|
|
|
|
def test_success(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.update_provider.return_value = _make_provider(id="prov-1")
|
|
mock_svc.get_decrypted_api_key.return_value = "sk-decrypted"
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.put(f"{P}/providers/prov-1", json={
|
|
"provider_type": "openai", "name": "Updated", "base_url": "https://x.com",
|
|
"api_key": "sk-updated", "default_model": "gpt-4",
|
|
})
|
|
assert resp.status_code == 200
|
|
|
|
def test_not_found(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.update_provider.return_value = None
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.put(f"{P}/providers/unknown", json={
|
|
"provider_type": "openai", "name": "X", "base_url": "https://x.com",
|
|
"api_key": "sk-x", "default_model": "gpt-4",
|
|
})
|
|
assert resp.status_code == 404
|
|
|
|
|
|
class TestDeleteProvider:
|
|
"""DELETE /api/llm/providers/{provider_id}"""
|
|
|
|
def test_success(self):
|
|
mock_db = MagicMock()
|
|
mock_db.query.return_value.filter.return_value.all.return_value = []
|
|
mock_svc = MagicMock()
|
|
mock_svc.delete_provider.return_value = True
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.delete(f"{P}/providers/prov-1")
|
|
assert resp.status_code == 204
|
|
|
|
def test_not_found(self):
|
|
mock_db = MagicMock()
|
|
mock_db.query.return_value.filter.return_value.all.return_value = []
|
|
mock_svc = MagicMock()
|
|
mock_svc.delete_provider.return_value = False
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.delete(f"{P}/providers/unknown")
|
|
assert resp.status_code == 404
|
|
|
|
def test_blocked_by_active_tasks(self):
|
|
mock_db = MagicMock()
|
|
mock_db.query.return_value.filter.return_value.all.return_value = [MagicMock(id="t-1", name="Task")]
|
|
|
|
from src.core.database import get_db
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.delete(f"{P}/providers/prov-1")
|
|
assert resp.status_code == 409
|
|
|
|
|
|
class TestLlmTestConnection:
|
|
"""POST /api/llm/providers/{provider_id}/test"""
|
|
|
|
def test_success(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_provider.return_value = _make_provider()
|
|
mock_svc.get_decrypted_api_key.return_value = "sk-real-key"
|
|
mock_client = MagicMock()
|
|
mock_client.test_runtime_connection = AsyncMock()
|
|
|
|
from src.core.database import get_db
|
|
with (
|
|
patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc),
|
|
patch("src.plugins.llm_analysis.service.LLMClient", return_value=mock_client),
|
|
):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/prov-1/test")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["success"] is True
|
|
|
|
def test_not_found(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_provider.return_value = None
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/unknown/test")
|
|
assert resp.status_code == 404
|
|
|
|
def test_no_api_key(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_provider.return_value = _make_provider()
|
|
mock_svc.get_decrypted_api_key.return_value = ""
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/prov-1/test")
|
|
assert resp.status_code == 500
|
|
|
|
def test_connection_failure(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_provider.return_value = _make_provider()
|
|
mock_svc.get_decrypted_api_key.return_value = "sk-real-key"
|
|
mock_client = MagicMock()
|
|
mock_client.test_runtime_connection = AsyncMock(side_effect=RuntimeError("fail"))
|
|
|
|
from src.core.database import get_db
|
|
with (
|
|
patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc),
|
|
patch("src.plugins.llm_analysis.service.LLMClient", return_value=mock_client),
|
|
):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/prov-1/test")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["success"] is False
|
|
|
|
|
|
class TestTestProviderConfig:
|
|
"""POST /api/llm/providers/test"""
|
|
|
|
def test_success(self):
|
|
mock_client = MagicMock()
|
|
mock_client.test_runtime_connection = AsyncMock()
|
|
|
|
with patch("src.plugins.llm_analysis.service.LLMClient", return_value=mock_client):
|
|
client = _make_client()
|
|
resp = client.post(f"{P}/providers/test", json={
|
|
"provider_type": "openai", "name": "T", "base_url": "https://x.com",
|
|
"api_key": "sk-test", "default_model": "gpt-4",
|
|
})
|
|
assert resp.status_code == 200
|
|
assert resp.json()["success"] is True
|
|
|
|
def test_masked_api_key_rejected(self):
|
|
client = _make_client()
|
|
resp = client.post(f"{P}/providers/test", json={
|
|
"provider_type": "openai", "name": "T", "base_url": "https://x.com",
|
|
"api_key": "********", "default_model": "gpt-4",
|
|
})
|
|
assert resp.status_code == 400
|
|
|
|
|
|
class TestProbeMaxImages:
|
|
"""POST /api/llm/providers/{provider_id}/probe-max-images"""
|
|
|
|
def test_no_limit_detected(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_provider.return_value = _make_provider()
|
|
mock_svc.get_decrypted_api_key.return_value = "sk-real"
|
|
mock_openai = MagicMock()
|
|
mock_openai.chat.completions = MagicMock()
|
|
mock_openai.chat.completions.create = AsyncMock()
|
|
|
|
from src.core.database import get_db
|
|
with (
|
|
patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc),
|
|
patch("src.api.routes.llm.AsyncOpenAI", return_value=mock_openai),
|
|
):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/prov-1/probe-max-images")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["method"] == "no_limit_detected"
|
|
|
|
def test_provider_not_found(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_provider.return_value = None
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/unknown/probe-max-images")
|
|
assert resp.status_code == 404
|
|
|
|
def test_no_api_key(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_provider.return_value = _make_provider()
|
|
mock_svc.get_decrypted_api_key.return_value = ""
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/prov-1/probe-max-images")
|
|
assert resp.status_code == 400
|
|
|
|
def test_no_default_model(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_provider.return_value = _make_provider(default_model="")
|
|
mock_svc.get_decrypted_api_key.return_value = "sk-real"
|
|
|
|
from src.core.database import get_db
|
|
with patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/prov-1/probe-max-images")
|
|
assert resp.status_code == 400
|
|
|
|
def test_limit_from_error_parse(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_provider.return_value = _make_provider()
|
|
mock_svc.get_decrypted_api_key.return_value = "sk-real"
|
|
mock_openai = MagicMock()
|
|
mock_openai.chat.completions = MagicMock()
|
|
mock_openai.chat.completions.create = AsyncMock(side_effect=RuntimeError("At most 5 image(s) per request"))
|
|
|
|
from src.core.database import get_db
|
|
with (
|
|
patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc),
|
|
patch("src.api.routes.llm.AsyncOpenAI", return_value=mock_openai),
|
|
):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/prov-1/probe-max-images")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["max_images"] == 5
|
|
assert resp.json()["method"] == "parse_error"
|
|
|
|
def test_first_image_fails(self):
|
|
mock_db = MagicMock()
|
|
mock_svc = MagicMock()
|
|
mock_svc.get_provider.return_value = _make_provider()
|
|
mock_svc.get_decrypted_api_key.return_value = "sk-real"
|
|
mock_openai = MagicMock()
|
|
mock_openai.chat.completions = MagicMock()
|
|
mock_openai.chat.completions.create = AsyncMock(side_effect=RuntimeError("unsupported"))
|
|
|
|
from src.core.database import get_db
|
|
with (
|
|
patch("src.api.routes.llm.LLMProviderService", return_value=mock_svc),
|
|
patch("src.api.routes.llm.AsyncOpenAI", return_value=mock_openai),
|
|
):
|
|
client = _make_client({get_db: lambda: mock_db})
|
|
resp = client.post(f"{P}/providers/prov-1/probe-max-images")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["max_images"] == 0
|
|
# #endregion Test.Api.Llm
|