# #region Test.LLMTranslationService [C:3] [TYPE Module] [SEMANTICS test, llm, translation, retry, parse] # @BRIEF Verify LLMTranslationService contracts — call_llm_for_batch, retry, split, recovery, failure handling. # @RELATION BINDS_TO -> [LLMTranslationService] # @TEST_EDGE: batch_empty -> Handled gracefully # @TEST_EDGE: llm_all_retries_exhausted -> Returns failure counts # @TEST_EDGE: parse_failure -> Returns skipped counts # @TEST_EDGE: truncation_recovery -> Recovers partial rows # @TEST_EDGE: no_provider -> Raises ValueError import json import pytest from unittest.mock import AsyncMock, MagicMock, patch from datetime import UTC, datetime from src.models.translate import ( Base, TranslationBatch, TranslationJob, TranslationLanguage, TranslationRecord, ) from src.plugins.translate._llm_call import LLMTranslationService, MAX_RETRIES_PER_BATCH from .conftest import JOB_ID, RUN_ID def _make_job(session, provider_id="test-provider", target_langs=None): job = session.query(TranslationJob).filter(TranslationJob.id == JOB_ID).first() if job is None: job = TranslationJob( id=JOB_ID, name="LLM Test", source_dialect="en", target_dialect=target_langs[0] if target_langs else "fr", target_languages=target_langs or ["fr"], status="ACTIVE", provider_id=provider_id, ) session.add(job) else: if provider_id: job.provider_id = provider_id if target_langs: job.target_languages = target_langs job.target_dialect = target_langs[0] session.commit() return job def _batch_rows(count=2, detected_lang="en"): return [ { "row_index": str(i), "source_text": f"Hello world {i}", "source_data": {"table": "test"}, "_detected_lang": detected_lang, } for i in range(count) ] class TestInit: """Verify basic initialization.""" def test_init(self, db_session): svc = LLMTranslationService(db_session) assert svc.db is db_session class TestBuildDictionarySection: """Verify _build_dictionary_section.""" def test_no_dict_matches(self, db_session): svc = LLMTranslationService(db_session) result = svc._build_dictionary_section([], _batch_rows(1)) assert result == "" def test_with_dict_matches(self, db_session): svc = LLMTranslationService(db_session) matches = [{ "entry_id": "e1", "source_term": "hello", "source_term_normalized": "hello", "target_term": "hola", "dictionary_id": "d1", "dictionary_name": "Test", "context_notes": "", "context_data": None, "has_context": False, "is_regex": False, "context_source": None, "usage_notes": None, "source_language": "en", "target_language": "es", "priority_match": False, "text_index": 0, "source_text": "hello", }] result = svc._build_dictionary_section(matches, _batch_rows(1)) assert "Terminology dictionary" in result assert "hello" in result class TestResolveTargetLanguages: """Verify _resolve_target_languages.""" def test_list_target_languages(self, db_session): svc = LLMTranslationService(db_session) job = _make_job(db_session, target_langs=["fr", "de"]) result = LLMTranslationService._resolve_target_languages(job) assert result == ["fr", "de"] def test_single_target_language(self, db_session): svc = LLMTranslationService(db_session) job = _make_job(db_session, target_langs=["fr"]) result = LLMTranslationService._resolve_target_languages(job) assert result == ["fr"] class TestBuildPrompt: """Verify _build_prompt.""" def test_prompt_contains_rows_and_languages(self, db_session): svc = LLMTranslationService(db_session) job = _make_job(db_session, target_langs=["fr"]) rows = _batch_rows(2) prompt = LLMTranslationService._build_prompt(job, rows, "dict_section\n", ["fr"]) assert "fr" in prompt assert "Hello world 0" in prompt assert "Hello world 1" in prompt assert "dict_section" in prompt class TestCallLlm: """Verify call_llm method (provider routing).""" @pytest.mark.asyncio async def test_no_provider_id(self, db_session): svc = LLMTranslationService(db_session) job = _make_job(db_session, provider_id="") with pytest.raises(ValueError, match="no LLM provider configured"): await svc.call_llm(job, "prompt") @pytest.mark.asyncio async def test_provider_not_found(self, db_session): svc = LLMTranslationService(db_session) job = _make_job(db_session, provider_id="nonexistent") with patch('src.plugins.translate._llm_call.LLMProviderService') as MockSvc: mock_instance = MockSvc.return_value mock_instance.get_provider.return_value = None with pytest.raises(ValueError, match="not found"): await svc.call_llm(job, "prompt") @pytest.mark.asyncio async def test_api_key_not_found(self, db_session): svc = LLMTranslationService(db_session) job = _make_job(db_session, provider_id="test-provider") with patch('src.plugins.translate._llm_call.LLMProviderService') as MockSvc: mock_instance = MockSvc.return_value mock_instance.get_provider.return_value = MagicMock( provider_type="openai", base_url="https://api.openai.com", default_model="gpt-4o-mini", ) mock_instance.get_decrypted_api_key.return_value = None with pytest.raises(ValueError, match="Could not decrypt"): await svc.call_llm(job, "prompt") @pytest.mark.asyncio async def test_unsupported_provider_type(self, db_session): svc = LLMTranslationService(db_session) job = _make_job(db_session, provider_id="test-provider") with patch('src.plugins.translate._llm_call.LLMProviderService') as MockSvc: mock_instance = MockSvc.return_value mock_provider = MagicMock( provider_type="unknown_type", base_url="https://example.com", default_model="test-model", ) mock_instance.get_provider.return_value = mock_provider mock_instance.get_decrypted_api_key.return_value = "sk-test" with pytest.raises(ValueError, match="Unsupported provider type"): await svc.call_llm(job, "prompt") @pytest.mark.asyncio async def test_successful_llm_call(self, db_session): svc = LLMTranslationService(db_session) job = _make_job(db_session, provider_id="test-provider") with patch('src.plugins.translate._llm_call.LLMProviderService') as MockSvc: mock_instance = MockSvc.return_value mock_instance.get_provider.return_value = MagicMock( provider_type="openai", base_url="https://api.openai.com", default_model="gpt-4o-mini", ) mock_instance.get_decrypted_api_key.return_value = "sk-test" with patch('src.plugins.translate._llm_call.call_openai_compatible', new=AsyncMock(return_value=('{"response": "ok"}', "stop"))): response, finish = await svc.call_llm(job, "test prompt") assert response == '{"response": "ok"}' assert finish == "stop" class TestCallLlmForBatch: """Verify call_llm_for_batch orchestration.""" @pytest.mark.asyncio async def test_llm_failure_all_retries(self, db_with_run): """Negative: all retries fail.""" session, run_id = db_with_run svc = LLMTranslationService(session) job = _make_job(session, target_langs=["fr"]) rows = _batch_rows(2) with patch.object(svc, '_call_llm_with_retry', new=AsyncMock(return_value=(None, None, 3, "API error"))): result = await svc.call_llm_for_batch( job=job, run_id=run_id, batch_rows=rows, dict_matches=[], batch_id="batch-1", ) assert result["successful"] == 0 assert result["failed"] == 2 assert result["retries"] == 3 @pytest.mark.asyncio async def test_parse_failure(self, db_with_run): """Negative: LLM response cannot be parsed.""" session, run_id = db_with_run svc = LLMTranslationService(session) job = _make_job(session, target_langs=["fr"]) rows = _batch_rows(1) with patch.object(svc, '_call_llm_with_retry', new=AsyncMock(return_value=("bad response", "stop", 1, None))): with patch('src.plugins.translate._llm_call.parse_llm_response', side_effect=ValueError("Invalid JSON")): result = await svc.call_llm_for_batch( job=job, run_id=run_id, batch_rows=rows, dict_matches=[], batch_id="batch-1", ) assert result["successful"] == 0 assert result["skipped"] == 1 @pytest.mark.asyncio async def test_successful_translation(self, db_with_run): """Happy: successful LLM call with parsed translations.""" session, run_id = db_with_run svc = LLMTranslationService(session) job = _make_job(session, target_langs=["fr"]) rows = _batch_rows(1) with patch.object(svc, '_call_llm_with_retry', new=AsyncMock(return_value=('{"ok": true}', "stop", 1, None))): with patch('src.plugins.translate._llm_call.parse_llm_response', return_value={"0": {"fr": "Bonjour le monde 0", "translation": "Bonjour le monde 0"}}): result = await svc.call_llm_for_batch( job=job, run_id=run_id, batch_rows=rows, dict_matches=[], batch_id="batch-1", ) assert result["successful"] == 1 @pytest.mark.asyncio async def test_truncation_split_and_retry(self, db_with_run): """Edge: truncation triggers binary split.""" session, run_id = db_with_run svc = LLMTranslationService(session) job = _make_job(session, target_langs=["fr"]) rows = _batch_rows(4) with patch.object(svc, '_call_llm_with_retry', new=AsyncMock(return_value=('partial', "length", 1, None))): with patch.object(svc, '_try_recover_partial', return_value=None): with patch.object(svc, '_split_and_retry', new=AsyncMock(return_value={"successful": 2, "failed": 0, "skipped": 0, "retries": 1})): result = await svc.call_llm_for_batch( job=job, run_id=run_id, batch_rows=rows, dict_matches=[], batch_id="batch-1", ) assert result["successful"] == 2 @pytest.mark.asyncio async def test_truncation_recovery_all_rows(self, db_with_run): """Edge: all rows recovered from truncated response.""" session, run_id = db_with_run svc = LLMTranslationService(session) job = _make_job(session, target_langs=["fr"]) rows = _batch_rows(2) with patch.object(svc, '_call_llm_with_retry', new=AsyncMock(return_value=('partial', "length", 1, None))): with patch.object(svc, '_try_recover_partial', return_value={"0", "1"}): result = await svc.call_llm_for_batch( job=job, run_id=run_id, batch_rows=rows, dict_matches=[], batch_id="batch-1", ) assert result["successful"] == 2 @pytest.mark.asyncio async def test_truncation_partial_recovery_then_retry(self, db_with_run): """Edge: partial recovery, then retry remaining.""" session, run_id = db_with_run svc = LLMTranslationService(session) job = _make_job(session, target_langs=["fr"]) rows = _batch_rows(3) with patch.object(svc, '_call_llm_with_retry', new=AsyncMock(return_value=('partial', "length", 1, None))): with patch.object(svc, '_try_recover_partial', return_value={"0"}): with patch.object(svc, '_retry_missing_rows', new=AsyncMock(return_value={"successful": 2, "failed": 0, "skipped": 0, "retries": 1})): result = await svc.call_llm_for_batch( job=job, run_id=run_id, batch_rows=rows, dict_matches=[], batch_id="batch-1", ) assert result["successful"] == 2 @pytest.mark.asyncio async def test_truncation_depth_exceeded(self, db_with_run): """Edge: recursion depth exceeded.""" session, run_id = db_with_run svc = LLMTranslationService(session) job = _make_job(session, target_langs=["fr"]) rows = _batch_rows(2) with patch.object(svc, '_call_llm_with_retry', new=AsyncMock(return_value=('partial', "length", 1, None))): with patch.object(svc, '_try_recover_partial', return_value=None): result = await svc.call_llm_for_batch( job=job, run_id=run_id, batch_rows=rows, dict_matches=[], batch_id="batch-1", _recursion_depth=MAX_RETRIES_PER_BATCH, ) assert result["successful"] == 0 class TestCallLlmWithRetry: """Verify _call_llm_with_retry loop.""" @pytest.mark.asyncio async def test_first_attempt_succeeds(self, db_session): svc = LLMTranslationService(db_session) with patch.object(svc, 'call_llm', new=AsyncMock(return_value=("response", "stop"))): response, finish, retries, last_error = await svc._call_llm_with_retry( MagicMock(), "prompt", "batch-1", 8192, ) assert response == "response" assert finish == "stop" assert retries == 0 @pytest.mark.asyncio async def test_retry_then_succeeds(self, db_session): svc = LLMTranslationService(db_session) call_mock = AsyncMock(side_effect=[ Exception("First fail"), ("response", "stop"), ]) with patch.object(svc, 'call_llm', new=call_mock): response, finish, retries, last_error = await svc._call_llm_with_retry( MagicMock(), "prompt", "batch-1", 8192, ) assert response == "response" assert retries == 1 @pytest.mark.asyncio async def test_all_retries_exhausted(self, db_session): svc = LLMTranslationService(db_session) call_mock = AsyncMock(side_effect=Exception("Always fail")) with patch.object(svc, 'call_llm', new=call_mock): response, finish, retries, last_error = await svc._call_llm_with_retry( MagicMock(), "prompt", "batch-1", 8192, ) assert response is None assert retries == MAX_RETRIES_PER_BATCH class TestHandleLlmFailure: """Verify _handle_llm_failure.""" def test_all_rows_marked_failed(self, db_with_run): session, run_id = db_with_run svc = LLMTranslationService(session) rows = _batch_rows(3) result = svc._handle_llm_failure(rows, run_id, "batch-1", 3, "Timeout") assert result["failed"] == 3 assert result["successful"] == 0 records = session.query(TranslationRecord).all() assert len(records) == 3 assert all(r.status == "FAILED" for r in records) assert all("Timeout" in r.error_message for r in records) class TestHandleParseFailure: """Verify _handle_parse_failure.""" def test_all_rows_marked_skipped(self, db_with_run): session, run_id = db_with_run svc = LLMTranslationService(session) rows = _batch_rows(2) result = svc._handle_parse_failure(rows, run_id, "batch-1", 2, ValueError("Bad parse")) assert result["skipped"] == 2 records = session.query(TranslationRecord).all() assert len(records) == 2 assert all(r.status == "SKIPPED" for r in records) class TestCreateRecordsFromTranslations: """Verify _create_records_from_translations.""" def test_successful_creates_records(self, db_with_run): session, run_id = db_with_run svc = LLMTranslationService(session) rows = _batch_rows(1) translations = {"0": {"fr": "Bonjour", "translation": "Bonjour"}} result = svc._create_records_from_translations( rows, run_id, "batch-1", ["fr"], translations, [], 0, ) assert result["successful"] == 1 records = session.query(TranslationRecord).all() assert len(records) == 1 assert records[0].status == "SUCCESS" langs = session.query(TranslationLanguage).all() assert len(langs) == 1 def test_null_translation_skipped(self, db_with_run): session, run_id = db_with_run svc = LLMTranslationService(session) rows = _batch_rows(1) translations = {} result = svc._create_records_from_translations( rows, run_id, "batch-1", ["fr"], translations, [], 0, ) assert result["skipped"] == 1 def test_empty_plv_skipped(self, db_with_run): session, run_id = db_with_run svc = LLMTranslationService(session) rows = _batch_rows(1) translations = {"0": {"de": "Hallo"}} result = svc._create_records_from_translations( rows, run_id, "batch-1", ["fr"], translations, [], 0, ) assert result["skipped"] == 1 def test_with_dict_enforcement(self, db_with_run): session, run_id = db_with_run svc = LLMTranslationService(session) rows = _batch_rows(1) translations = {"0": {"fr": "Bonjour", "translation": "Bonjour"}} dict_matches = [{"entry_id": "e1", "source_term": "hello", "target_term": "hola"}] result = svc._create_records_from_translations( rows, run_id, "batch-1", ["fr"], translations, dict_matches, 0, ) assert result["successful"] == 1 class TestExtractPerLangValues: """Verify _extract_per_lang_values.""" def test_extract_first_target(self): td = {"fr": "Bonjour", "translation": "Hello"} result = LLMTranslationService._extract_per_lang_values(td, ["fr"]) assert result == {"fr": "Bonjour"} def test_fallback_to_translation(self): td = {"translation": "Hello"} result = LLMTranslationService._extract_per_lang_values(td, ["fr"]) assert result == {"fr": "Hello"} def test_empty_values(self): td = {"fr": "", "translation": ""} result = LLMTranslationService._extract_per_lang_values(td, ["fr"]) assert result == {} def test_multi_language(self): td = {"fr": "Bonjour", "de": "Hallo"} result = LLMTranslationService._extract_per_lang_values(td, ["fr", "de"]) assert result == {"fr": "Bonjour", "de": "Hallo"} class TestAddSkipped: """Verify _add_skipped.""" def test_add_skipped_record(self, db_with_run): session, run_id = db_with_run svc = LLMTranslationService(session) row = _batch_rows(1)[0] svc._add_skipped(row, run_id, "batch-1", "source text", "No translation found") records = session.query(TranslationRecord).all() assert len(records) == 1 assert records[0].status == "SKIPPED" assert records[0].error_message == "No translation found" class TestSplitAndRetry: """Verify _split_and_retry.""" @pytest.mark.asyncio async def test_split_creates_child_batches(self, db_with_run): session, run_id = db_with_run svc = LLMTranslationService(session) job = _make_job(session, target_langs=["fr"]) rows = _batch_rows(4) with patch.object(svc, 'call_llm_for_batch', new=AsyncMock(return_value={"successful": 2, "failed": 0, "skipped": 0, "retries": 0})): result = await svc._split_and_retry( job, run_id, rows, [], "orig-batch", 8192, 1, 0, ) assert result["successful"] == 4 batches = session.query(TranslationBatch).all() assert len(batches) == 2 class TestTryRecoverPartial: """Verify _try_recover_partial.""" def test_successful_recovery(self, db_with_run): session, run_id = db_with_run svc = LLMTranslationService(session) rows = _batch_rows(2) translations_response = json.dumps([ {"row_id": "0", "fr": "Bonjour"}, {"row_id": "1", "fr": "Monde"}, ]) with patch('src.plugins.translate._llm_call.parse_llm_response', return_value={"0": {"fr": "Bonjour"}, "1": {"fr": "Monde"}}): recovered = svc._try_recover_partial( translations_response, rows, run_id, "batch-1", ["fr"], ) assert recovered == {"0", "1"} records = session.query(TranslationRecord).all() assert len(records) == 2 def test_no_rows_recovered(self, db_with_run): session, run_id = db_with_run svc = LLMTranslationService(session) rows = _batch_rows(2) with patch('src.plugins.translate._llm_call.parse_llm_response', return_value={}): recovered = svc._try_recover_partial( "", rows, run_id, "batch-1", ["fr"], ) assert recovered is None def test_parse_error_returns_none(self, db_with_run): session, run_id = db_with_run svc = LLMTranslationService(session) rows = _batch_rows(2) with patch('src.plugins.translate._llm_call.parse_llm_response', side_effect=ValueError("Parse error")): recovered = svc._try_recover_partial( "invalid json", rows, run_id, "batch-1", ["fr"], ) assert recovered is None class TestRetryMissingRows: """Verify _retry_missing_rows.""" @pytest.mark.asyncio async def test_empty_missing(self, db_session): svc = LLMTranslationService(db_session) result = await svc._retry_missing_rows( MagicMock(), "run-1", [], [], "batch-1", 8192, 1, ) assert result["successful"] == 0 @pytest.mark.asyncio async def test_retry_creates_sub_batch(self, db_with_run): session, run_id = db_with_run svc = LLMTranslationService(session) job = _make_job(session, target_langs=["fr"]) rows = _batch_rows(1) with patch.object(svc, 'call_llm_for_batch', new=AsyncMock(return_value={"successful": 0, "failed": 1, "skipped": 0, "retries": 1})): result = await svc._retry_missing_rows( job, run_id, rows, [], "batch-1", 8192, 1, ) assert result["failed"] == 1 batches = session.query(TranslationBatch).all() assert len(batches) == 1 # #endregion Test.LLMTranslationService