busya
ce0369ae5c
Subagents delivered tests across all uncovered backend modules: Schemas (100%): agent, auth, health, profile, settings, validation Services (98-100%): auth, profile, health, llm, mapping, resource, security, git, superset_lookup, sql_table_extractor, rbac API routes (new): auth, admin, health, environments, plugins, dashboards (helpers, projection, actions, listing), git (config, deps, env, helpers) Clean Release (100%): DTO, facade, policy_engine, stages, repos, preparation, source_isolation, compliance Git services: base, remote_providers Agent module: app, run, middleware, langgraph_setup Core: trace, cleanup, ws_log_handler, timezone, auth (config/oauth/security), matching Reports: normalizer, report_service, type_profiles Notifications: service, providers Also: - .gitignore: add .coverage, *.cover, coverage-* dirs - src/schemas/auth.py: fix AD group DN regex (comma in CN=...) - Remove co-located src/services/__tests__/ (caused pytest module collision)
282 lines
11 KiB
Python
282 lines
11 KiB
Python
# #region Test.LLMPromptTemplates [C:3] [TYPE Module] [SEMANTICS test,llm,prompt,templates,normalization]
|
|
# @BRIEF Tests for services/llm_prompt_templates.py — DEFAULT_LLM_PROMPTS, normalize_llm_settings,
|
|
# is_multimodal_model, resolve_bound_provider_id, render_prompt.
|
|
# @RELATION BINDS_TO -> [llm_prompt_templates]
|
|
# @TEST_EDGE: missing_field -> None/empty settings produce defaults
|
|
# @TEST_EDGE: custom_overrides -> user-provided prompts preserved
|
|
# @TEST_EDGE: multimodal_detection -> known vision/text models classified correctly
|
|
|
|
import warnings
|
|
from unittest.mock import patch
|
|
import pytest
|
|
|
|
from src.services.llm_prompt_templates import (
|
|
DEFAULT_LLM_ASSISTANT_SETTINGS,
|
|
DEFAULT_LLM_PROMPTS,
|
|
DEFAULT_LLM_PROVIDER_BINDINGS,
|
|
is_multimodal_model,
|
|
normalize_llm_settings,
|
|
render_prompt,
|
|
resolve_bound_provider_id,
|
|
)
|
|
|
|
|
|
class TestDefaults:
|
|
"""DEFAULT_LLM_PROMPTS, DEFAULT_LLM_PROVIDER_BINDINGS, DEFAULT_LLM_ASSISTANT_SETTINGS."""
|
|
|
|
def test_all_prompts_present(self):
|
|
required_keys = {
|
|
"dashboard_validation_prompt",
|
|
"dashboard_validation_prompt_multimodal",
|
|
"dashboard_validation_prompt_text",
|
|
"documentation_prompt",
|
|
"git_commit_prompt",
|
|
}
|
|
assert required_keys.issubset(DEFAULT_LLM_PROMPTS.keys())
|
|
|
|
def test_all_prompts_are_strings(self):
|
|
for key, value in DEFAULT_LLM_PROMPTS.items():
|
|
assert isinstance(value, str), f"{key} is not a string"
|
|
assert len(value) > 0, f"{key} is empty"
|
|
|
|
def test_provider_bindings_present(self):
|
|
assert "documentation" in DEFAULT_LLM_PROVIDER_BINDINGS
|
|
assert "git_commit" in DEFAULT_LLM_PROVIDER_BINDINGS
|
|
|
|
def test_assistant_settings_present(self):
|
|
assert "assistant_planner_provider" in DEFAULT_LLM_ASSISTANT_SETTINGS
|
|
assert "assistant_planner_model" in DEFAULT_LLM_ASSISTANT_SETTINGS
|
|
|
|
|
|
class TestNormalizeLLMSettings:
|
|
"""normalize_llm_settings — ensure stable schema with defaults."""
|
|
|
|
def test_none_input_returns_defaults(self):
|
|
normalized = normalize_llm_settings(None)
|
|
assert normalized["providers"] == []
|
|
assert normalized["default_provider"] == ""
|
|
assert "prompts" in normalized
|
|
assert "provider_bindings" in normalized
|
|
|
|
def test_empty_dict_returns_defaults(self):
|
|
normalized = normalize_llm_settings({})
|
|
assert normalized["providers"] == []
|
|
assert normalized["default_provider"] == ""
|
|
|
|
def test_keeps_valid_keys(self):
|
|
normalized = normalize_llm_settings({
|
|
"providers": [{"id": "p1"}],
|
|
"default_provider": "p1",
|
|
"unknown_key": "should_be_ignored",
|
|
})
|
|
assert normalized["default_provider"] == "p1"
|
|
assert "unknown_key" not in normalized
|
|
|
|
def test_merges_custom_prompts(self):
|
|
normalized = normalize_llm_settings({
|
|
"prompts": {"documentation_prompt": "Custom doc prompt {dataset_name}"},
|
|
})
|
|
assert normalized["prompts"]["documentation_prompt"] == "Custom doc prompt {dataset_name}"
|
|
|
|
def test_ignores_empty_custom_prompts(self):
|
|
normalized = normalize_llm_settings({
|
|
"prompts": {"documentation_prompt": ""},
|
|
})
|
|
# Should use default
|
|
assert normalized["prompts"]["documentation_prompt"] == DEFAULT_LLM_PROMPTS["documentation_prompt"]
|
|
|
|
def test_ignores_whitespace_only_custom_prompts(self):
|
|
normalized = normalize_llm_settings({
|
|
"prompts": {"documentation_prompt": " "},
|
|
})
|
|
assert normalized["prompts"]["documentation_prompt"] == DEFAULT_LLM_PROMPTS["documentation_prompt"]
|
|
|
|
def test_keeps_custom_bindings(self):
|
|
normalized = normalize_llm_settings({
|
|
"provider_bindings": {"documentation": "doc-provider"},
|
|
})
|
|
assert normalized["provider_bindings"]["documentation"] == "doc-provider"
|
|
|
|
def test_assistant_settings_preserved(self):
|
|
normalized = normalize_llm_settings({
|
|
"assistant_planner_provider": "planner-1",
|
|
"assistant_planner_model": "gpt-4o",
|
|
})
|
|
assert normalized["assistant_planner_provider"] == "planner-1"
|
|
assert normalized["assistant_planner_model"] == "gpt-4o"
|
|
|
|
def test_assistant_settings_default_when_missing(self):
|
|
normalized = normalize_llm_settings({})
|
|
assert normalized["assistant_planner_provider"] == ""
|
|
assert normalized["assistant_planner_model"] == ""
|
|
|
|
def test_non_dict_prompts_field_uses_defaults(self):
|
|
normalized = normalize_llm_settings({"prompts": "not_a_dict"})
|
|
for key in DEFAULT_LLM_PROMPTS:
|
|
assert key in normalized["prompts"]
|
|
|
|
def test_non_dict_bindings_field_uses_defaults(self):
|
|
normalized = normalize_llm_settings({"provider_bindings": None})
|
|
for key in DEFAULT_LLM_PROVIDER_BINDINGS:
|
|
assert key in normalized["provider_bindings"]
|
|
|
|
def test_strips_assistant_settings_values(self):
|
|
normalized = normalize_llm_settings({
|
|
"assistant_planner_provider": " provider-a ",
|
|
})
|
|
assert normalized["assistant_planner_provider"] == "provider-a"
|
|
|
|
|
|
class TestIsMultimodalModel:
|
|
"""is_multimodal_model — heuristic detection of vision-capable models."""
|
|
|
|
def test_gpt_4o_is_multimodal(self):
|
|
with pytest.warns(DeprecationWarning):
|
|
assert is_multimodal_model("gpt-4o") is True
|
|
|
|
def test_gpt_4_1_is_multimodal(self):
|
|
with pytest.warns(DeprecationWarning):
|
|
assert is_multimodal_model("gpt-4.1-nano") is True
|
|
|
|
def test_claude_3_is_multimodal(self):
|
|
with pytest.warns(DeprecationWarning):
|
|
assert is_multimodal_model("claude-3-5-sonnet") is True
|
|
|
|
def test_claude_sonnet_4_is_multimodal(self):
|
|
with pytest.warns(DeprecationWarning):
|
|
assert is_multimodal_model("claude-sonnet-4-20250514") is True
|
|
|
|
def test_gemini_is_multimodal(self):
|
|
with pytest.warns(DeprecationWarning):
|
|
assert is_multimodal_model("gemini-2.0-flash") is True
|
|
|
|
def test_vision_model_is_multimodal(self):
|
|
with pytest.warns(DeprecationWarning):
|
|
assert is_multimodal_model("llava-v1.6") is True
|
|
|
|
def test_pixtral_is_multimodal(self):
|
|
with pytest.warns(DeprecationWarning):
|
|
assert is_multimodal_model("pixtral-12b") is True
|
|
|
|
def test_qwen_vl_is_multimodal(self):
|
|
with pytest.warns(DeprecationWarning):
|
|
assert is_multimodal_model("qwen2-vl-72b") is True
|
|
|
|
def test_text_only_markers_return_false(self):
|
|
with pytest.warns(DeprecationWarning):
|
|
assert is_multimodal_model("text-only-embedding") is False
|
|
assert is_multimodal_model("whisper-1") is False
|
|
assert is_multimodal_model("rerank-model") is False
|
|
assert is_multimodal_model("tts-model") is False
|
|
assert is_multimodal_model("transcribe") is False
|
|
|
|
def test_empty_model_name_returns_false(self):
|
|
with pytest.warns(DeprecationWarning):
|
|
assert is_multimodal_model("") is False
|
|
|
|
def test_none_model_name_returns_false(self):
|
|
with pytest.warns(DeprecationWarning):
|
|
assert is_multimodal_model(None) is False
|
|
|
|
def test_case_insensitive_matching(self):
|
|
with pytest.warns(DeprecationWarning):
|
|
assert is_multimodal_model("GPT-4O") is True
|
|
assert is_multimodal_model("CLAUDE-3-OPUS") is True
|
|
|
|
def test_provider_type_ignored_by_function(self):
|
|
with pytest.warns(DeprecationWarning):
|
|
# provider_type parameter exists but function ignores it
|
|
result = is_multimodal_model("gpt-4o", provider_type="openai")
|
|
assert result is True
|
|
|
|
|
|
class TestResolveBoundProviderId:
|
|
"""resolve_bound_provider_id — binding resolution with fallback."""
|
|
|
|
def test_uses_binding_when_present(self):
|
|
settings = {
|
|
"default_provider": "default-1",
|
|
"provider_bindings": {"documentation": "doc-provider"},
|
|
}
|
|
assert resolve_bound_provider_id(settings, "documentation") == "doc-provider"
|
|
|
|
def test_falls_back_to_default(self):
|
|
settings = {
|
|
"default_provider": "default-1",
|
|
"provider_bindings": {},
|
|
}
|
|
assert resolve_bound_provider_id(settings, "documentation") == "default-1"
|
|
|
|
def test_empty_binding_uses_default(self):
|
|
settings = {
|
|
"default_provider": "default-1",
|
|
"provider_bindings": {"documentation": ""},
|
|
}
|
|
assert resolve_bound_provider_id(settings, "documentation") == "default-1"
|
|
|
|
def test_whitespace_binding_uses_default(self):
|
|
settings = {
|
|
"default_provider": "default-1",
|
|
"provider_bindings": {"documentation": " "},
|
|
}
|
|
assert resolve_bound_provider_id(settings, "documentation") == "default-1"
|
|
|
|
def test_no_default_returns_empty(self):
|
|
settings = {"provider_bindings": {}}
|
|
assert resolve_bound_provider_id(settings, "documentation") == ""
|
|
|
|
def test_raw_settings_normalized(self):
|
|
result = resolve_bound_provider_id(None, "documentation")
|
|
assert result == ""
|
|
|
|
def test_strips_bound_value(self):
|
|
settings = {
|
|
"default_provider": "default-1",
|
|
"provider_bindings": {"documentation": " bound-1 "},
|
|
}
|
|
assert resolve_bound_provider_id(settings, "documentation") == "bound-1"
|
|
|
|
|
|
class TestRenderPrompt:
|
|
"""render_prompt — template rendering with placeholder substitution."""
|
|
|
|
def test_simple_replacement(self):
|
|
result = render_prompt("Hello {name}", {"name": "World"})
|
|
assert result == "Hello World"
|
|
|
|
def test_multiple_replacements(self):
|
|
result = render_prompt("{a} + {b} = {c}", {"a": "1", "b": "2", "c": "3"})
|
|
assert result == "1 + 2 = 3"
|
|
|
|
def test_missing_placeholders_warned(self):
|
|
with patch("src.services.llm_prompt_templates.logger") as mock_logger:
|
|
result = render_prompt("Hello {name}, you are {age}", {"name": "Bob"})
|
|
assert result == "Hello Bob, you are {age}"
|
|
mock_logger.warning.assert_called_once()
|
|
assert "unfilled" in mock_logger.warning.call_args[0][0].lower()
|
|
|
|
def test_no_placeholders(self):
|
|
result = render_prompt("Static text", {})
|
|
assert result == "Static text"
|
|
|
|
def test_empty_template(self):
|
|
result = render_prompt("", {"key": "value"})
|
|
assert result == ""
|
|
|
|
def test_empty_variables(self):
|
|
result = render_prompt("Hello {name}", {})
|
|
assert result == "Hello {name}"
|
|
|
|
def test_non_string_values_converted(self):
|
|
result = render_prompt("Count: {count}", {"count": 42})
|
|
assert result == "Count: 42"
|
|
|
|
def test_multiple_unfilled_placeholders(self):
|
|
with patch("src.services.llm_prompt_templates.logger") as mock_logger:
|
|
result = render_prompt("{a} {b} {c}", {"a": "x"})
|
|
assert result == "x {b} {c}"
|
|
warning_msg = mock_logger.warning.call_args[0][0]
|
|
assert "b" in warning_msg
|
|
assert "c" in warning_msg
|
|
# #endregion Test.LLMPromptTemplates
|