# #region Test.LLMPromptTemplates [C:3] [TYPE Module] [SEMANTICS test,llm,prompt,templates,normalization] # @BRIEF Tests for services/llm_prompt_templates.py — DEFAULT_LLM_PROMPTS, normalize_llm_settings, # is_multimodal_model, resolve_bound_provider_id, render_prompt. # @RELATION BINDS_TO -> [llm_prompt_templates] # @TEST_EDGE: missing_field -> None/empty settings produce defaults # @TEST_EDGE: custom_overrides -> user-provided prompts preserved # @TEST_EDGE: multimodal_detection -> known vision/text models classified correctly import warnings from unittest.mock import patch import pytest from src.services.llm_prompt_templates import ( DEFAULT_LLM_ASSISTANT_SETTINGS, DEFAULT_LLM_PROMPTS, DEFAULT_LLM_PROVIDER_BINDINGS, is_multimodal_model, normalize_llm_settings, render_prompt, resolve_bound_provider_id, ) class TestDefaults: """DEFAULT_LLM_PROMPTS, DEFAULT_LLM_PROVIDER_BINDINGS, DEFAULT_LLM_ASSISTANT_SETTINGS.""" def test_all_prompts_present(self): required_keys = { "dashboard_validation_prompt", "dashboard_validation_prompt_multimodal", "dashboard_validation_prompt_text", "documentation_prompt", "git_commit_prompt", } assert required_keys.issubset(DEFAULT_LLM_PROMPTS.keys()) def test_all_prompts_are_strings(self): for key, value in DEFAULT_LLM_PROMPTS.items(): assert isinstance(value, str), f"{key} is not a string" assert len(value) > 0, f"{key} is empty" def test_provider_bindings_present(self): assert "documentation" in DEFAULT_LLM_PROVIDER_BINDINGS assert "git_commit" in DEFAULT_LLM_PROVIDER_BINDINGS def test_assistant_settings_present(self): assert "assistant_planner_provider" in DEFAULT_LLM_ASSISTANT_SETTINGS assert "assistant_planner_model" in DEFAULT_LLM_ASSISTANT_SETTINGS class TestNormalizeLLMSettings: """normalize_llm_settings — ensure stable schema with defaults.""" def test_none_input_returns_defaults(self): normalized = normalize_llm_settings(None) assert normalized["providers"] == [] assert normalized["default_provider"] == "" assert "prompts" in normalized assert "provider_bindings" in normalized def test_empty_dict_returns_defaults(self): normalized = normalize_llm_settings({}) assert normalized["providers"] == [] assert normalized["default_provider"] == "" def test_keeps_valid_keys(self): normalized = normalize_llm_settings({ "providers": [{"id": "p1"}], "default_provider": "p1", "unknown_key": "should_be_ignored", }) assert normalized["default_provider"] == "p1" assert "unknown_key" not in normalized def test_merges_custom_prompts(self): normalized = normalize_llm_settings({ "prompts": {"documentation_prompt": "Custom doc prompt {dataset_name}"}, }) assert normalized["prompts"]["documentation_prompt"] == "Custom doc prompt {dataset_name}" def test_ignores_empty_custom_prompts(self): normalized = normalize_llm_settings({ "prompts": {"documentation_prompt": ""}, }) # Should use default assert normalized["prompts"]["documentation_prompt"] == DEFAULT_LLM_PROMPTS["documentation_prompt"] def test_ignores_whitespace_only_custom_prompts(self): normalized = normalize_llm_settings({ "prompts": {"documentation_prompt": " "}, }) assert normalized["prompts"]["documentation_prompt"] == DEFAULT_LLM_PROMPTS["documentation_prompt"] def test_keeps_custom_bindings(self): normalized = normalize_llm_settings({ "provider_bindings": {"documentation": "doc-provider"}, }) assert normalized["provider_bindings"]["documentation"] == "doc-provider" def test_assistant_settings_preserved(self): normalized = normalize_llm_settings({ "assistant_planner_provider": "planner-1", "assistant_planner_model": "gpt-4o", }) assert normalized["assistant_planner_provider"] == "planner-1" assert normalized["assistant_planner_model"] == "gpt-4o" def test_assistant_settings_default_when_missing(self): normalized = normalize_llm_settings({}) assert normalized["assistant_planner_provider"] == "" assert normalized["assistant_planner_model"] == "" def test_non_dict_prompts_field_uses_defaults(self): normalized = normalize_llm_settings({"prompts": "not_a_dict"}) for key in DEFAULT_LLM_PROMPTS: assert key in normalized["prompts"] def test_non_dict_bindings_field_uses_defaults(self): normalized = normalize_llm_settings({"provider_bindings": None}) for key in DEFAULT_LLM_PROVIDER_BINDINGS: assert key in normalized["provider_bindings"] def test_strips_assistant_settings_values(self): normalized = normalize_llm_settings({ "assistant_planner_provider": " provider-a ", }) assert normalized["assistant_planner_provider"] == "provider-a" class TestIsMultimodalModel: """is_multimodal_model — heuristic detection of vision-capable models.""" def test_gpt_4o_is_multimodal(self): with pytest.warns(DeprecationWarning): assert is_multimodal_model("gpt-4o") is True def test_gpt_4_1_is_multimodal(self): with pytest.warns(DeprecationWarning): assert is_multimodal_model("gpt-4.1-nano") is True def test_claude_3_is_multimodal(self): with pytest.warns(DeprecationWarning): assert is_multimodal_model("claude-3-5-sonnet") is True def test_claude_sonnet_4_is_multimodal(self): with pytest.warns(DeprecationWarning): assert is_multimodal_model("claude-sonnet-4-20250514") is True def test_gemini_is_multimodal(self): with pytest.warns(DeprecationWarning): assert is_multimodal_model("gemini-2.0-flash") is True def test_vision_model_is_multimodal(self): with pytest.warns(DeprecationWarning): assert is_multimodal_model("llava-v1.6") is True def test_pixtral_is_multimodal(self): with pytest.warns(DeprecationWarning): assert is_multimodal_model("pixtral-12b") is True def test_qwen_vl_is_multimodal(self): with pytest.warns(DeprecationWarning): assert is_multimodal_model("qwen2-vl-72b") is True def test_text_only_markers_return_false(self): with pytest.warns(DeprecationWarning): assert is_multimodal_model("text-only-embedding") is False assert is_multimodal_model("whisper-1") is False assert is_multimodal_model("rerank-model") is False assert is_multimodal_model("tts-model") is False assert is_multimodal_model("transcribe") is False def test_empty_model_name_returns_false(self): with pytest.warns(DeprecationWarning): assert is_multimodal_model("") is False def test_none_model_name_returns_false(self): with pytest.warns(DeprecationWarning): assert is_multimodal_model(None) is False def test_case_insensitive_matching(self): with pytest.warns(DeprecationWarning): assert is_multimodal_model("GPT-4O") is True assert is_multimodal_model("CLAUDE-3-OPUS") is True def test_provider_type_ignored_by_function(self): with pytest.warns(DeprecationWarning): # provider_type parameter exists but function ignores it result = is_multimodal_model("gpt-4o", provider_type="openai") assert result is True class TestResolveBoundProviderId: """resolve_bound_provider_id — binding resolution with fallback.""" def test_uses_binding_when_present(self): settings = { "default_provider": "default-1", "provider_bindings": {"documentation": "doc-provider"}, } assert resolve_bound_provider_id(settings, "documentation") == "doc-provider" def test_falls_back_to_default(self): settings = { "default_provider": "default-1", "provider_bindings": {}, } assert resolve_bound_provider_id(settings, "documentation") == "default-1" def test_empty_binding_uses_default(self): settings = { "default_provider": "default-1", "provider_bindings": {"documentation": ""}, } assert resolve_bound_provider_id(settings, "documentation") == "default-1" def test_whitespace_binding_uses_default(self): settings = { "default_provider": "default-1", "provider_bindings": {"documentation": " "}, } assert resolve_bound_provider_id(settings, "documentation") == "default-1" def test_no_default_returns_empty(self): settings = {"provider_bindings": {}} assert resolve_bound_provider_id(settings, "documentation") == "" def test_raw_settings_normalized(self): result = resolve_bound_provider_id(None, "documentation") assert result == "" def test_strips_bound_value(self): settings = { "default_provider": "default-1", "provider_bindings": {"documentation": " bound-1 "}, } assert resolve_bound_provider_id(settings, "documentation") == "bound-1" class TestRenderPrompt: """render_prompt — template rendering with placeholder substitution.""" def test_simple_replacement(self): result = render_prompt("Hello {name}", {"name": "World"}) assert result == "Hello World" def test_multiple_replacements(self): result = render_prompt("{a} + {b} = {c}", {"a": "1", "b": "2", "c": "3"}) assert result == "1 + 2 = 3" def test_missing_placeholders_warned(self): with patch("src.services.llm_prompt_templates.logger") as mock_logger: result = render_prompt("Hello {name}, you are {age}", {"name": "Bob"}) assert result == "Hello Bob, you are {age}" mock_logger.warning.assert_called_once() assert "unfilled" in mock_logger.warning.call_args[0][0].lower() def test_no_placeholders(self): result = render_prompt("Static text", {}) assert result == "Static text" def test_empty_template(self): result = render_prompt("", {"key": "value"}) assert result == "" def test_empty_variables(self): result = render_prompt("Hello {name}", {}) assert result == "Hello {name}" def test_non_string_values_converted(self): result = render_prompt("Count: {count}", {"count": 42}) assert result == "Count: 42" def test_multiple_unfilled_placeholders(self): with patch("src.services.llm_prompt_templates.logger") as mock_logger: result = render_prompt("{a} {b} {c}", {"a": "x"}) assert result == "x {b} {c}" warning_msg = mock_logger.warning.call_args[0][0] assert "b" in warning_msg assert "c" in warning_msg # #endregion Test.LLMPromptTemplates