busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
51afbee470d3e45ee93c1eec25ee2ba7ad2da95b
ss-tools/backend/src/services/__tests__/test_llm_provider.py
busya 814f2da139 perf(translate): fix slow translation startup — CJK estimation, output budget, provider token config 
Root cause: batch sizing underestimated CJK token density (1.5→1.0 chars/token)
and ignored output budget as primary constraint, causing cascading finish_reason=length.

Changes:
- _token_budget.py: CJK_RATIO 1.5→1.0, OTHER_RATIO 2.2→1.8, safety factors 0.75/0.70
- _token_budget.py: new _compute_max_rows_by_output() — output budget is PRIMARY constraint
- _batch_sizer.py: resolve_provider_config() with DB-level context_window/max_output_tokens
- _batch_sizer.py: INPUT_SAFETY_FACTOR applied, max_rows_by_output used as row cap
- _llm_http.py: log actual usage.prompt_tokens/.completion_tokens from provider
- _llm_call.py: retry only missing rows after finish_reason=length (save partial result)
- models/llm.py + schema: provider-level context_window / max_output_tokens (nullable)
- services/llm_provider.py: get_provider_token_config() helper
- Alembic migration: add columns to llm_providers
- Svelte ProviderConfig: collapsible Advanced: Token Limits section
- 12 new tests (token budget, batch sizer, provider config)
- All 492 tests pass
2026-06-03 23:25:08 +03:00

501 lines
18 KiB
Python
Raw Blame History

# region test_llm_provider [TYPE Module]
# @RELATION BINDS_TO -> [LLMProvider]
# @SEMANTICS: tests, llm-provider, encryption, contract
# @PURPOSE: Contract testing for LLMProviderService and EncryptionManager
# endregion test_llm_provider
import os
import pytest
from unittest.mock import MagicMock
from cryptography.fernet import Fernet
from sqlalchemy.orm import Session
from src.models.llm import LLMProvider
from src.plugins.llm_analysis.models import LLMProviderConfig, LLMProviderType
from src.services.llm_provider import (
EncryptionManager,
LLMProviderService,
is_masked_or_placeholder,
mask_api_key,
)
# region _test_encryption_key_fixture [TYPE Global]
# @PURPOSE: Ensure encryption-dependent provider tests run with a valid Fernet key.
# @RELATION DEPENDS_ON -> [EXT:Library:pytest]
os.environ.setdefault("ENCRYPTION_KEY", Fernet.generate_key().decode())
# endregion _test_encryption_key_fixture
# region test_provider_type_enum_values [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderType]
# @PURPOSE: Regression guard — prevent accidental value drift when enum variants are added or renamed.
# @INVARIANT Every LLMProviderType member must have a value matching its lowercase name.
def test_provider_type_enum_values():
"""Verify all LLMProviderType enum values match their expected strings."""
assert LLMProviderType.OPENAI.value == "openai"
assert LLMProviderType.OPENROUTER.value == "openrouter"
assert LLMProviderType.KILO.value == "kilo"
assert LLMProviderType.LITELLM.value == "litellm"
# If new providers are added, extend this list — enum value drift breaks string comparisons
# across executor.py, preview.py, ProviderConfig.svelte, and the DB layer.
expected_count = 4
assert len(LLMProviderType) == expected_count, (
f"Expected {expected_count} provider types, got {len(LLMProviderType)}. "
"If you added a new one, add its value assertion above and update expected_count."
)
# endregion test_provider_type_enum_values
# @TEST_CONTRACT EncryptionManagerModel -> Invariants
# @TEST_INVARIANT symmetric_encryption
# region test_encryption_cycle [TYPE Function]
# @RELATION BINDS_TO -> [EXT:frontend:test_llm_provider]
# @PURPOSE: Verify EncryptionManager round-trip encryption/decryption invariant for non-empty secrets.
def test_encryption_cycle():
"""Verify encrypted data can be decrypted back to original string."""
manager = EncryptionManager()
original = "secret_api_key_123"
encrypted = manager.encrypt(original)
assert encrypted != original
assert manager.decrypt(encrypted) == original
# @TEST_EDGE empty_string_encryption
# endregion test_encryption_cycle
# region test_empty_string_encryption [TYPE Function]
# @RELATION BINDS_TO -> [EXT:frontend:test_llm_provider]
# @PURPOSE: Verify EncryptionManager preserves empty-string payloads through encrypt/decrypt cycle.
def test_empty_string_encryption():
manager = EncryptionManager()
original = ""
encrypted = manager.encrypt(original)
assert manager.decrypt(encrypted) == ""
# @TEST_EDGE decrypt_invalid_data
# endregion test_empty_string_encryption
# region test_decrypt_invalid_data [TYPE Function]
# @RELATION BINDS_TO -> [EXT:frontend:test_llm_provider]
# @PURPOSE: Ensure decrypt rejects invalid ciphertext input by raising an exception.
def test_decrypt_invalid_data():
manager = EncryptionManager()
with pytest.raises(Exception):
manager.decrypt("not-encrypted-string")
# @TEST_FIXTURE mock_db_session
# endregion test_decrypt_invalid_data
# region mock_db [TYPE Fixture]
# @RELATION BINDS_TO -> [EXT:frontend:test_llm_provider]
# @PURPOSE: MagicMock(spec=Session) fixture providing a constrained DB session double for LLMProviderService tests.
# @INVARIANT Chained calls beyond Session spec create unconstrained intermediate mocks; only top-level query/add/commit are spec-enforced.
@pytest.fixture
def mock_db():
# @RISK: query() returns unconstrained MagicMock — chain beyond query() has no spec protection. Consider create_autospec(Session) for full chain safety.
return MagicMock(spec=Session)
# endregion mock_db
# region service [TYPE Fixture]
# @RELATION BINDS_TO -> [EXT:frontend:test_llm_provider]
# @PURPOSE: LLMProviderService fixture wired to mock_db for provider CRUD tests.
@pytest.fixture
def service(mock_db):
return LLMProviderService(db=mock_db)
# endregion service
# region test_get_all_providers [TYPE Function]
# @RELATION BINDS_TO -> [EXT:frontend:test_llm_provider]
# @PURPOSE: Verify provider list retrieval issues query/all calls on the backing DB session.
def test_get_all_providers(service, mock_db):
service.get_all_providers()
mock_db.query.assert_called()
mock_db.query().all.assert_called()
# endregion test_get_all_providers
# region test_create_provider [TYPE Function]
# @RELATION BINDS_TO -> [EXT:frontend:test_llm_provider]
# @PURPOSE: Ensure provider creation persists entity and stores API key in encrypted form.
def test_create_provider(service, mock_db):
config = LLMProviderConfig(
provider_type=LLMProviderType.OPENAI,
name="Test OpenAI",
base_url="https://api.openai.com",
api_key="sk-test",
default_model="gpt-4",
is_active=True,
)
provider = service.create_provider(config)
mock_db.add.assert_called()
mock_db.commit.assert_called()
# Verify API key was encrypted
assert provider.api_key != "sk-test"
# Decrypt to verify it matches
assert EncryptionManager().decrypt(provider.api_key) == "sk-test"
# endregion test_create_provider
# region test_get_decrypted_api_key [TYPE Function]
# @RELATION BINDS_TO -> [EXT:frontend:test_llm_provider]
# @PURPOSE: Verify service decrypts stored provider API key for an existing provider record.
def test_get_decrypted_api_key(service, mock_db):
# Setup mock provider
encrypted_key = EncryptionManager().encrypt("secret-value")
mock_provider = LLMProvider(id="p1", api_key=encrypted_key)
mock_db.query().filter().first.return_value = mock_provider
key = service.get_decrypted_api_key("p1")
assert key == "secret-value"
# endregion test_get_decrypted_api_key
# region test_get_decrypted_api_key_not_found [TYPE Function]
# @RELATION BINDS_TO -> [EXT:frontend:test_llm_provider]
# @PURPOSE: Verify missing provider lookup returns None instead of attempting decryption.
def test_get_decrypted_api_key_not_found(service, mock_db):
mock_db.query().filter().first.return_value = None
assert service.get_decrypted_api_key("missing") is None
# endregion test_get_decrypted_api_key_not_found
# region test_update_provider_ignores_masked_placeholder_api_key [TYPE Function]
# @RELATION BINDS_TO -> [EXT:frontend:test_llm_provider]
# @PURPOSE: Ensure masked placeholder API keys do not overwrite previously encrypted provider secrets.
def test_update_provider_ignores_masked_placeholder_api_key(service, mock_db):
existing_encrypted = EncryptionManager().encrypt("secret-value")
mock_provider = LLMProvider(
id="p1",
provider_type="openai",
name="Existing",
base_url="https://api.openai.com/v1",
api_key=existing_encrypted,
default_model="gpt-4o",
is_active=True,
)
mock_db.query().filter().first.return_value = mock_provider
config = LLMProviderConfig(
id="p1",
provider_type=LLMProviderType.OPENAI,
name="Existing",
base_url="https://api.openai.com/v1",
api_key="********",
default_model="gpt-4o",
is_active=False,
)
updated = service.update_provider("p1", config)
assert updated is mock_provider
assert updated.api_key == existing_encrypted
assert EncryptionManager().decrypt(updated.api_key) == "secret-value"
assert updated.is_active is False
# endregion test_update_provider_ignores_masked_placeholder_api_key
# region test_mask_api_key [TYPE Function]
# @RELATION BINDS_TO -> [mask_api_key]
# @PURPOSE: Verify mask_api_key produces correct masked strings for various key lengths and edge cases.
# @INVARIANT mask_api_key never reveals more than 4 chars from any position.
# @TEST_EDGE None -> ""
# @TEST_EDGE "" -> ""
# @TEST_EDGE "abcd" -> "****"
# @TEST_EDGE "abcdef" -> "ab...ef"
# @TEST_EDGE "abcdefgh" -> "ab...gh"
# @TEST_EDGE "sk-test-key-1234abcd" -> "sk-t...abcd"
def test_mask_api_key():
assert mask_api_key(None) == ""
assert mask_api_key("") == ""
assert mask_api_key("abcd") == "****"
assert mask_api_key("abcdef") == "ab...ef"
assert mask_api_key("abcdefgh") == "ab...gh"
assert mask_api_key("sk-test-key-1234abcd") == "sk-t...abcd"
# endregion test_mask_api_key
# region test_is_masked_or_placeholder [TYPE Function]
# @RELATION BINDS_TO -> [is_masked_or_placeholder]
# @PURPOSE: Verify predicate correctly identifies all forms of masked/placeholder API keys.
# @INVARIANT Real API keys (no "..." pattern) always return False.
# @TEST_EDGE None -> True
# @TEST_EDGE "" -> True
# @TEST_EDGE "********" -> True
# @TEST_EDGE "sk-...abcd" -> True
# @TEST_EDGE "...xyz" -> True
# @TEST_EDGE "sk-real-key-1234" -> False
# @TEST_EDGE "short" -> False
def test_is_masked_or_placeholder():
assert is_masked_or_placeholder(None) is True
assert is_masked_or_placeholder("") is True
assert is_masked_or_placeholder("********") is True
assert is_masked_or_placeholder("sk-...abcd") is True
assert is_masked_or_placeholder("...xyz") is True
assert is_masked_or_placeholder("sk-real-key-1234") is False
assert is_masked_or_placeholder("short") is False
# endregion test_is_masked_or_placeholder
# region test_create_provider_with_multimodal_flag [TYPE Function]
# @RELATION BINDS_TO -> [EXT:frontend:test_llm_provider]
# @PURPOSE: Verify provider creation persists the is_multimodal flag.
def test_create_provider_with_multimodal_flag(service, mock_db):
"""Verify is_multimodal=True is stored during provider creation."""
config = LLMProviderConfig(
provider_type=LLMProviderType.OPENAI,
name="Vision Provider",
base_url="https://api.openai.com",
api_key="sk-vision-key",
default_model="gpt-4o",
is_active=True,
is_multimodal=True,
)
provider = service.create_provider(config)
assert provider.is_multimodal is True
mock_db.add.assert_called()
mock_db.commit.assert_called()
# endregion test_create_provider_with_multimodal_flag
# region test_create_provider_without_multimodal_flag [TYPE Function]
# @RELATION BINDS_TO -> [EXT:frontend:test_llm_provider]
# @PURPOSE: Verify is_multimodal defaults to False when not specified.
def test_create_provider_without_multimodal_flag(service, mock_db):
"""Verify is_multimodal defaults to False."""
config = LLMProviderConfig(
provider_type=LLMProviderType.OPENAI,
name="Text Provider",
base_url="https://api.openai.com",
api_key="sk-text-key",
default_model="gpt-4.1-mini",
is_active=True,
)
provider = service.create_provider(config)
assert provider.is_multimodal is False
# endregion test_create_provider_without_multimodal_flag
# region test_update_provider_preserves_is_multimodal [TYPE Function]
# @RELATION BINDS_TO -> [EXT:frontend:test_llm_provider]
# @PURPOSE: Verify updating a provider preserves and correctly sets is_multimodal.
def test_update_provider_preserves_is_multimodal(service, mock_db):
"""Verify is_multimodal is updated correctly."""
existing_encrypted = EncryptionManager().encrypt("secret-value")
mock_provider = LLMProvider(
id="p1",
provider_type="openai",
name="Vision",
base_url="https://api.openai.com/v1",
api_key=existing_encrypted,
default_model="gpt-4o",
is_active=True,
is_multimodal=True,
)
mock_db.query().filter().first.return_value = mock_provider
# Update with is_multimodal=False
config = LLMProviderConfig(
id="p1",
provider_type=LLMProviderType.OPENAI,
name="Vision",
base_url="https://api.openai.com/v1",
api_key="********",
default_model="gpt-4o",
is_active=True,
is_multimodal=False,
)
updated = service.update_provider("p1", config)
assert updated.is_multimodal is False
# endregion test_update_provider_preserves_is_multimodal
# region test_llm_provider_config_multimodal_default [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderConfig]
# @PURPOSE: Verify LLMProviderConfig.is_multimodal defaults to False for schema stability.
def test_llm_provider_config_multimodal_default():
"""Verify default is_multimodal is False in schema."""
config = LLMProviderConfig(
provider_type=LLMProviderType.OPENAI,
name="Default Test",
base_url="https://api.openai.com/v1",
api_key="sk-test",
default_model="gpt-4",
)
assert config.is_multimodal is False
# endregion test_llm_provider_config_multimodal_default
# region test_llm_provider_config_multimodal_explicit [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderConfig]
# @PURPOSE: Verify LLMProviderConfig accepts explicit is_multimodal=True.
def test_llm_provider_config_multimodal_explicit():
"""Verify setting is_multimodal=True explicitly works."""
config = LLMProviderConfig(
provider_type=LLMProviderType.OPENAI,
name="Vision Test",
base_url="https://api.openai.com/v1",
api_key="sk-test",
default_model="gpt-4o",
is_multimodal=True,
)
assert config.is_multimodal is True
# endregion test_llm_provider_config_multimodal_explicit
# region test_llm_provider_config_context_window_default [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderConfig]
# @PURPOSE: Verify LLMProviderConfig.context_window defaults to None.
def test_llm_provider_config_context_window_default():
"""Verify default context_window is None in schema."""
config = LLMProviderConfig(
provider_type=LLMProviderType.OPENAI,
name="Default Test",
base_url="https://api.openai.com/v1",
api_key="sk-test",
default_model="gpt-4",
)
assert config.context_window is None
# endregion test_llm_provider_config_context_window_default
# region test_llm_provider_config_context_window_explicit [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderConfig]
# @PURPOSE: Verify LLMProviderConfig accepts explicit context_window value.
def test_llm_provider_config_context_window_explicit():
"""Verify setting context_window explicitly works."""
config = LLMProviderConfig(
provider_type=LLMProviderType.OPENAI,
name="Test",
base_url="https://api.openai.com/v1",
api_key="sk-test",
default_model="gpt-4",
context_window=128000,
max_output_tokens=16384,
)
assert config.context_window == 128000
assert config.max_output_tokens == 16384
# endregion test_llm_provider_config_context_window_explicit
# region test_get_provider_token_config_no_provider [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderService]
# @PURPOSE: Verify get_provider_token_config returns all-None when provider not found.
def test_get_provider_token_config_no_provider():
"""When provider_id doesn't exist, returns all values as None."""
db = MagicMock(spec=Session)
db.query.return_value.filter.return_value.first.return_value = None
service = LLMProviderService(db)
result = service.get_provider_token_config("nonexistent-id")
assert result == {"model": None, "context_window": None, "max_output_tokens": None}
# endregion test_get_provider_token_config_no_provider
# region test_get_provider_token_config_with_values [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderService]
# @PURPOSE: Verify get_provider_token_config returns provider token limits from DB.
def test_get_provider_token_config_with_values():
"""Provider with context_window and max_output_tokens returns them."""
db = MagicMock(spec=Session)
mock_provider = MagicMock(spec=LLMProvider)
mock_provider.default_model = "gpt-4o-mini"
mock_provider.context_window = 128000
mock_provider.max_output_tokens = 16384
db.query.return_value.filter.return_value.first.return_value = mock_provider
service = LLMProviderService(db)
result = service.get_provider_token_config("provider-1")
assert result["model"] == "gpt-4o-mini"
assert result["context_window"] == 128000
assert result["max_output_tokens"] == 16384
# endregion test_get_provider_token_config_with_values
# region test_get_provider_token_config_null_limits [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderService]
# @PURPOSE: Verify get_provider_token_config returns None for null DB fields.
def test_get_provider_token_config_null_limits():
"""Provider with NULL token limits returns None values (signal to use defaults)."""
db = MagicMock(spec=Session)
mock_provider = MagicMock(spec=LLMProvider)
mock_provider.default_model = "qwen-flash"
mock_provider.context_window = None
mock_provider.max_output_tokens = None
db.query.return_value.filter.return_value.first.return_value = mock_provider
service = LLMProviderService(db)
result = service.get_provider_token_config("provider-2")
assert result["model"] == "qwen-flash"
assert result["context_window"] is None
assert result["max_output_tokens"] is None
# endregion test_get_provider_token_config_null_limits
# region test_provider_token_config_default_model_fallback [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderService]
# @PURPOSE: Verify get_provider_token_config falls back to "gpt-4o-mini" when default_model is None.
def test_provider_token_config_default_model_fallback():
"""Provider without explicit default_model uses 'gpt-4o-mini' fallback."""
db = MagicMock(spec=Session)
mock_provider = MagicMock(spec=LLMProvider)
mock_provider.default_model = None
mock_provider.context_window = None
mock_provider.max_output_tokens = None
db.query.return_value.filter.return_value.first.return_value = mock_provider
service = LLMProviderService(db)
result = service.get_provider_token_config("provider-3")
assert result["model"] == "gpt-4o-mini"
assert result["context_window"] is None
assert result["max_output_tokens"] is None
# endregion test_provider_token_config_default_model_fallback
Reference in New Issue View Git Blame Copy Permalink