busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

semantics

Browse Source
This commit is contained in:
busya
2026-05-26 09:30:41 +03:00
parent 1e7bcecaea
commit 9ffa8af1dc
623 changed files with 28045 additions and 26557 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py
View File

@@ -1,15 +1,15 @@
# region ClickHouseInsertIntegration [TYPE Module]
# @SEMANTICS: test, clickhouse, integration, insert, join
# @PURPOSE: Integration tests for ClickHouse INSERT with timestamp normalization and JOIN verification.
# @LAYER: Test
# @RELATION: BINDS_TO -> [SQLGenerator:Module]
# @RELATION: BINDS_TO -> [TranslationOrchestrator]
# @TEST_CONTRACT: SQLGenerator.generate(clickhouse) -> valid INSERT SQL with YYYY-MM-DD dates
# @TEST_SCENARIO: timestamp_in_date_column -> ClickHouse parses date correctly
# @TEST_SCENARIO: join_after_insert -> financial_comments_translated JOIN financial_arrears works
# @TEST_EDGE: unix_millis_timestamp -> converted to YYYY-MM-DD
# @TEST_EDGE: unix_seconds_timestamp -> converted to YYYY-MM-DD
# @TEST_EDGE: non_timestamp_string -> passed through unchanged
# @LAYER Test
# @RELATION BINDS_TO -> [SQLGenerator]
# @RELATION BINDS_TO -> [TranslationOrchestrator]
# @TEST_CONTRACT SQLGenerator.generate(clickhouse) -> valid INSERT SQL with YYYY-MM-DD dates
# @TEST_SCENARIO timestamp_in_date_column -> ClickHouse parses date correctly
# @TEST_SCENARIO join_after_insert -> financial_comments_translated JOIN financial_arrears works
# @TEST_EDGE unix_millis_timestamp -> converted to YYYY-MM-DD
# @TEST_EDGE unix_seconds_timestamp -> converted to YYYY-MM-DD
# @TEST_EDGE non_timestamp_string -> passed through unchanged
import logging

4
backend/src/plugins/translate/__tests__/test_dictionary.py
View File

@@ -5,7 +5,7 @@
# test_dictionary_filter.py — Batch filter + migration
# test_dictionary_correction.py — Correction context capture
# test_dictionary_prompt_builder.py — Prompt builder operations
# test_dictionary_utils.py — Utility functions
# test_dictionary[EXT:internal:_utils].py — Utility functions
# @RELATION BINDS_TO -> [DictionaryManager]
# @RATIONALE Split monolithic test module into domain-specific files per INV_7.
# @REJECTED Keeping 1199-line test module violates module < 400 lines constraint.
@@ -16,5 +16,5 @@ from .test_dictionary_crud import * # noqa: F401, F403
from .test_dictionary_filter import * # noqa: F401, F403
from .test_dictionary_import import * # noqa: F401, F403
from .test_dictionary_prompt_builder import * # noqa: F401, F403
from .test_dictionary_utils import * # noqa: F401, F403
from .test_dictionary[EXT:internal:_utils] import * # noqa: F401, F403
# #endregion TestDictionaryLegacyHub

36
backend/src/plugins/translate/__tests__/test_dictionary_crud.py
View File

@@ -2,10 +2,10 @@
# @BRIEF Validate DictionaryCRUD and DictionaryEntryCRUD operations.
# @RELATION BINDS_TO -> [DictionaryCRUD]
# @RELATION BINDS_TO -> [DictionaryEntryCRUD]
# @TEST_EDGE: duplicate_entry -> ValueError on repeated (dictionary_id, source_term_norm, source_lang, target_lang)
# @TEST_EDGE: delete_active_job -> ValueError with active/scheduled message
# @TEST_EDGE: same_term_different_lang_pair -> allowed (not duplicate)
# @TEST_INVARIANT: unique_normalized -> verifies: [duplicate_entry, same_term_different_lang_pair allowed]
# @TEST_EDGE duplicate_entry -> ValueError on repeated (dictionary_id, source_term_norm, source_lang, target_lang)
# @TEST_EDGE delete_active_job -> ValueError with active/scheduled message
# @TEST_EDGE same_term_different_lang_pair -> allowed (not duplicate)
# @TEST_INVARIANT unique_normalized -> verifies: [duplicate_entry, same_term_different_lang_pair allowed]
import pytest
@@ -42,7 +42,7 @@ class TestDictionaryCRUD:
"""Verify dictionary-level CRUD operations."""
# region test_create_dictionary [C:2] [TYPE Function]
# @BRIEF: Verify dictionary creation and read-back.
# @BRIEF Verify dictionary creation and read-back.
def test_create_dictionary(self, db_session: Session):
d = DictionaryManager.create_dictionary(
db_session, name="Finance Terms",
@@ -62,7 +62,7 @@ class TestDictionaryCRUD:
# endregion test_create_dictionary
# region test_update_dictionary [C:2] [TYPE Function]
# @BRIEF: Verify dictionary metadata update.
# @BRIEF Verify dictionary metadata update.
def test_update_dictionary(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Old Name", source_dialect="a", target_dialect="b")
updated = DictionaryManager.update_dictionary(
@@ -74,7 +74,7 @@ class TestDictionaryCRUD:
# endregion test_update_dictionary
# region test_delete_dictionary [C:2] [TYPE Function]
# @BRIEF: Verify dictionary deletion also removes entries.
# @BRIEF Verify dictionary deletion also removes entries.
def test_delete_dictionary(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="To Delete", source_dialect="a", target_dialect="b")
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
@@ -85,7 +85,7 @@ class TestDictionaryCRUD:
# endregion test_delete_dictionary
# region test_list_dictionaries [C:2] [TYPE Function]
# @BRIEF: Verify paginated dictionary listing.
# @BRIEF Verify paginated dictionary listing.
def test_list_dictionaries(self, db_session: Session):
for i in range(5):
DictionaryManager.create_dictionary(db_session, name=f"Dict {i}", source_dialect="a", target_dialect="b")
@@ -95,7 +95,7 @@ class TestDictionaryCRUD:
# endregion test_list_dictionaries
# region test_delete_dictionary_blocked_by_active_job [C:2] [TYPE Function]
# @BRIEF: Verify deletion is blocked when attached to active/scheduled jobs.
# @BRIEF Verify deletion is blocked when attached to active/scheduled jobs.
def test_delete_dictionary_blocked_by_active_job(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Test", source_dialect="a", target_dialect="b")
job = TranslationJob(name="Active Job", source_dialect="a", target_dialect="b", status="ACTIVE", created_by="test_user")
@@ -109,7 +109,7 @@ class TestDictionaryCRUD:
# endregion test_delete_dictionary_blocked_by_active_job
# region test_delete_dictionary_allowed_with_completed_job [C:2] [TYPE Function]
# @BRIEF: Verify deletion is allowed when only completed/failed jobs reference the dictionary.
# @BRIEF Verify deletion is allowed when only completed/failed jobs reference the dictionary.
def test_delete_dictionary_allowed_with_completed_job(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Test", source_dialect="a", target_dialect="b")
job = TranslationJob(name="Completed Job", source_dialect="a", target_dialect="b", status="COMPLETED", created_by="test_user")
@@ -128,7 +128,7 @@ class TestDictionaryEntryCRUD:
"""Verify entry-level CRUD operations."""
# region test_add_entry_duplicate [C:2] [TYPE Function]
# @BRIEF: Verify duplicate entry raises ValueError.
# @BRIEF Verify duplicate entry raises ValueError.
def test_add_entry_duplicate(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Test", source_dialect="a", target_dialect="b")
DictionaryManager.add_entry(db_session, d.id, "Hello", "Hola", source_language="en", target_language="es")
@@ -139,7 +139,7 @@ class TestDictionaryEntryCRUD:
# endregion test_add_entry_duplicate
# region test_add_entry_duplicate_per_dictionary [C:2] [TYPE Function]
# @BRIEF: Verify duplicate is per-dictionary (same term in different dicts is OK).
# @BRIEF Verify duplicate is per-dictionary (same term in different dicts is OK).
def test_add_entry_duplicate_per_dictionary(self, db_session: Session):
d1 = DictionaryManager.create_dictionary(db_session, name="Dict1", source_dialect="a", target_dialect="b")
d2 = DictionaryManager.create_dictionary(db_session, name="Dict2", source_dialect="a", target_dialect="b")
@@ -149,7 +149,7 @@ class TestDictionaryEntryCRUD:
# endregion test_add_entry_duplicate_per_dictionary
# region test_edit_entry [C:2] [TYPE Function]
# @BRIEF: Verify entry edit updates fields and enforces uniqueness.
# @BRIEF Verify entry edit updates fields and enforces uniqueness.
def test_edit_entry(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Test", source_dialect="a", target_dialect="b")
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
@@ -161,7 +161,7 @@ class TestDictionaryEntryCRUD:
# endregion test_edit_entry
# region test_delete_entry [C:2] [TYPE Function]
# @BRIEF: Verify entry deletion.
# @BRIEF Verify entry deletion.
def test_delete_entry(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Test", source_dialect="a", target_dialect="b")
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
@@ -171,7 +171,7 @@ class TestDictionaryEntryCRUD:
# endregion test_delete_entry
# region test_clear_entries [C:2] [TYPE Function]
# @BRIEF: Verify clearing all entries for a dictionary.
# @BRIEF Verify clearing all entries for a dictionary.
def test_clear_entries(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Test", source_dialect="a", target_dialect="b")
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
@@ -183,7 +183,7 @@ class TestDictionaryEntryCRUD:
# endregion test_clear_entries
# region test_add_entry_with_language_pair [C:2] [TYPE Function]
# @BRIEF: Verify creating entry with language pair stores correctly.
# @BRIEF Verify creating entry with language pair stores correctly.
def test_add_entry_with_language_pair(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Lang Test", source_dialect="a", target_dialect="b")
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")
@@ -196,7 +196,7 @@ class TestDictionaryEntryCRUD:
# endregion test_add_entry_with_language_pair
# region test_duplicate_same_language_pair [C:2] [TYPE Function]
# @BRIEF: Verify duplicate with same language pair raises conflict.
# @BRIEF Verify duplicate with same language pair raises conflict.
def test_duplicate_same_language_pair(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Dup Test", source_dialect="a", target_dialect="b")
DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")
@@ -205,7 +205,7 @@ class TestDictionaryEntryCRUD:
# endregion test_duplicate_same_language_pair
# region test_same_term_different_language_pair [C:2] [TYPE Function]
# @BRIEF: Verify same term with different language pair is allowed.
# @BRIEF Verify same term with different language pair is allowed.
def test_same_term_different_language_pair(self, db_session: Session):
d = DictionaryManager.create_dictionary(db_session, name="Multi Lang", source_dialect="a", target_dialect="b")
entry1 = DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")

2
backend/src/plugins/translate/__tests__/test_dictionary_import.py
View File

@@ -1,7 +1,7 @@
# #region TestDictionaryImport [C:3] [TYPE Module] [SEMANTICS test, dictionary, import, export]
# @BRIEF Validate DictionaryImportExport operations.
# @RELATION BINDS_TO -> [DictionaryImportExport]
# @TEST_EDGE: import_invalid_format -> ValueError for missing required columns
# @TEST_EDGE import_invalid_format -> ValueError for missing required columns
import csv
import io

4
backend/src/plugins/translate/__tests__/test_dictionary_utils.py
View File

@@ -1,8 +1,8 @@
# #region TestDictionaryUtils [C:3] [TYPE Module] [SEMANTICS test, dictionary, utils, normalization]
# @BRIEF Validate utility functions: _normalize_term and _detect_delimiter.
# @RELATION BINDS_TO -> [_utils]
# @RELATION BINDS_TO -> [[EXT:internal:_utils]]
from src.plugins.translate._utils import _detect_delimiter, _normalize_term
from src.plugins.translate.[EXT:internal:_utils] import _detect_delimiter, _normalize_term
class TestNormalizeTerm:

16
backend/src/plugins/translate/__tests__/test_executor.py
View File

@@ -1,11 +1,11 @@
# region ExecutorTests [TYPE Module]
# @SEMANTICS: test, translate, executor, null-handling, cancellation
# @PURPOSE: Tests for TranslationExecutor: null content handling, cancellation flag during execution.
# @LAYER: Test
# @RELATION: BINDS_TO -> [TranslationExecutor:Module]
# @TEST_CONTRACT: TranslationExecutor -> execute_run, _call_openai_compatible
# @TEST_EDGE: null_llm_content -> raises ValueError instead of TypeError
# @TEST_EDGE: cancellation_flag_during_execution -> stops batch processing
# @LAYER Test
# @RELATION BINDS_TO -> [TranslationExecutor]
# @TEST_CONTRACT TranslationExecutor -> execute_run, _call_openai_compatible
# @TEST_EDGE null_llm_content -> raises ValueError instead of TypeError
# @TEST_EDGE cancellation_flag_during_execution -> stops batch processing
import pytest
from unittest.mock import MagicMock, patch
@@ -267,7 +267,7 @@ class TestCancellationFlag:
# region TestEstimateRowTokens [TYPE Class]
# @PURPOSE: Tests for estimate_row_tokens — per-row token estimation for adaptive batch sizing.
# @RELATION: BINDS_TO -> [estimate_row_tokens]
# @RELATION BINDS_TO -> [estimate_row_tokens]
class TestEstimateRowTokens:
"""Unit tests for estimate_row_tokens()."""
@@ -344,8 +344,8 @@ class TestEstimateRowTokens:
# region TestAutoSizeBatches [TYPE Class]
# @PURPOSE: Tests for _auto_size_batches — variable-sized batch splitting based on content length.
# @RELATION: BINDS_TO -> [TranslationExecutor._auto_size_batches]
# @RELATION: BINDS_TO -> [estimate_token_budget]
# @RELATION BINDS_TO -> [EXT:method:TranslationExecutor._auto_size_batches]
# @RELATION BINDS_TO -> [estimate_token_budget]
class TestAutoSizeBatches:
"""Tests for TranslationExecutor._auto_size_batches()."""

44
backend/src/plugins/translate/__tests__/test_inline_correction.py
View File

@@ -1,15 +1,15 @@
# region InlineCorrectionTests [TYPE Module]
# @SEMANTICS: test, translate, correction, inline, bulk
# @PURPOSE: Tests for inline correction (T109-T116, T117): single correction, dictionary submission, bulk replace.
# @LAYER: Domain
# @RELATION: BINDS_TO -> [InlineCorrectionService:Module]
# @RELATION: BINDS_TO -> [BulkFindReplaceService:Module]
# @TEST_CONTRACT: InlineCorrectionService -> submit_correction, duplicate detection
# @TEST_CONTRACT: BulkFindReplaceService -> preview, apply, atomic
# @TEST_EDGE: single_correction -> Creates dictionary entry with origin tracking
# @TEST_EDGE: duplicate_detection -> Conflict detected for existing term
# @TEST_EDGE: bulk_replace_preview -> Returns accurate affected records
# @TEST_EDGE: bulk_replace_apply -> Updates values and optionally submits to dictionary
# @LAYER Domain
# @RELATION BINDS_TO -> [InlineCorrectionService]
# @RELATION BINDS_TO -> [BulkFindReplaceService]
# @TEST_CONTRACT InlineCorrectionService -> submit_correction, duplicate detection
# @TEST_CONTRACT BulkFindReplaceService -> preview, apply, atomic
# @TEST_EDGE single_correction -> Creates dictionary entry with origin tracking
# @TEST_EDGE duplicate_detection -> Conflict detected for existing term
# @TEST_EDGE bulk_replace_preview -> Returns accurate affected records
# @TEST_EDGE bulk_replace_apply -> Updates values and optionally submits to dictionary
from unittest.mock import MagicMock, patch
@@ -282,19 +282,19 @@ class TestBulkFindReplaceService:
# region TestContextAwareCorrection [TYPE Module]
# @SEMANTICS: test, translate, correction, context, jaccard, priority
# @PURPOSE: Tests for context-aware correction (T132): context capture, Jaccard similarity, priority flagging, truncation, context_source tagging.
# @LAYER: Test
# @RELATION: BINDS_TO -> [InlineCorrectionService:Class]
# @RELATION: BINDS_TO -> [ContextAwarePromptBuilder:Class]
# @TEST_CONTRACT: InlineCorrectionService -> context capture from source row, context editing/removal
# @TEST_CONTRACT: ContextAwarePromptBuilder -> Jaccard similarity, priority flagging, truncation, render_entry
# @TEST_EDGE: context_capture -> Auto-populates context_data from source row columns
# @TEST_EDGE: context_removal -> keep_context=false clears context_data and sets has_context=False
# @TEST_EDGE: jaccard_zero -> 0% overlap returns 0.0
# @TEST_EDGE: jaccard_half -> 50% overlap returns 0.5
# @TEST_EDGE: jaccard_full -> 100% overlap returns 1.0
# @TEST_EDGE: priority_flagging -> Similarity >=0.5 triggers priority prefix
# @TEST_EDGE: truncation_500 -> Context rendering capped at ~500 tokens with annotation
# @TEST_EDGE: context_source_tagging -> auto/bulk/manual tags set correctly
# @LAYER Test
# @RELATION BINDS_TO -> [InlineCorrectionService]
# @RELATION BINDS_TO -> [ContextAwarePromptBuilder]
# @TEST_CONTRACT InlineCorrectionService -> context capture from source row, context editing/removal
# @TEST_CONTRACT ContextAwarePromptBuilder -> Jaccard similarity, priority flagging, truncation, render_entry
# @TEST_EDGE context_capture -> Auto-populates context_data from source row columns
# @TEST_EDGE context_removal -> keep_context=false clears context_data and sets has_context=False
# @TEST_EDGE jaccard_zero -> 0% overlap returns 0.0
# @TEST_EDGE jaccard_half -> 50% overlap returns 0.5
# @TEST_EDGE jaccard_full -> 100% overlap returns 1.0
# @TEST_EDGE priority_flagging -> Similarity >=0.5 triggers priority prefix
# @TEST_EDGE truncation_500 -> Context rendering capped at ~500 tokens with annotation
# @TEST_EDGE context_source_tagging -> auto/bulk/manual tags set correctly
from unittest.mock import MagicMock

18
backend/src/plugins/translate/__tests__/test_orchestrator.py
View File

@@ -1,15 +1,15 @@
# region OrchestratorTests [TYPE Module]
# @SEMANTICS: test, translate, orchestrator, events
# @PURPOSE: Tests for TranslationOrchestrator: run lifecycle, partial failure, batch retry, event invariants, NULL handling.
# @LAYER: Test
# @RELATION: BINDS_TO -> [TranslationOrchestrator:Module]
# @RELATION: BINDS_TO -> [TranslationEventLog:Module]
# @TEST_CONTRACT: TranslationOrchestrator -> start_run, execute_run, cancel_run, retry_failed_batches
# @TEST_FIXTURE: mock_db -> MagicMock SQLAlchemy session
# @TEST_FIXTURE: mock_config_manager -> MagicMock ConfigManager
# @TEST_EDGE: missing_preview -> raises ValueError
# @TEST_EDGE: invalid_run_status -> raises ValueError
# @TEST_EDGE: executor_failure -> run is marked FAILED
# @LAYER Test
# @RELATION BINDS_TO -> [TranslationOrchestrator]
# @RELATION BINDS_TO -> [TranslationEventLog]
# @TEST_CONTRACT TranslationOrchestrator -> start_run, execute_run, cancel_run, retry_failed_batches
# @TEST_FIXTURE mock_db -> MagicMock SQLAlchemy session
# @TEST_FIXTURE mock_config_manager -> MagicMock ConfigManager
# @TEST_EDGE missing_preview -> raises ValueError
# @TEST_EDGE invalid_run_status -> raises ValueError
# @TEST_EDGE executor_failure -> run is marked FAILED
from datetime import UTC, datetime
import json

20
backend/src/plugins/translate/__tests__/test_orthogonal_fixes.py
View File

@@ -1,18 +1,18 @@
# region OrthogonalTranslationFixes [TYPE Module] [SEMANTICS test, translate, orthogonal, verification]
# @BRIEF Orthogonal verification of translation system fixes: cancel lock timeout, null content, periodic commit, async->def migration.
# @LAYER: Test
# @LAYER Test
# @RELATION BINDS_TO -> [TranslationOrchestrator]
# @RELATION BINDS_TO -> [TranslationExecutor]
# @RELATION BINDS_TO -> [TranslateRunRoutesModule]
# @TEST_EDGE: cancel_lock_timeout -> lock_timeout expiry falls back to CANCEL_REQUESTED flag
# @TEST_EDGE: cancel_nonexistent_run -> raises ValueError for missing run
# @TEST_EDGE: cancel_pending_run -> PENDING runs can be cancelled
# @TEST_EDGE: cancel_completed_at_set -> completed_at is populated after cancel
# @TEST_EDGE: cancel_event_log -> RUN_CANCELLED event exists in event log
# @TEST_EDGE: execute_cancelled_after_executor -> orchestrator handles CANCELLED status from executor
# @TEST_EDGE: execute_zero_rows -> orchestrator handles zero-row completion
# @TEST_EDGE: async_to_def_routes -> sync route handlers work with FastAPI TestClient
# @TEST_EDGE: no_await_in_routes -> no accidental await in sync handlers
# @TEST_EDGE cancel_lock_timeout -> lock_timeout expiry falls back to CANCEL_REQUESTED flag
# @TEST_EDGE cancel_nonexistent_run -> raises ValueError for missing run
# @TEST_EDGE cancel_pending_run -> PENDING runs can be cancelled
# @TEST_EDGE cancel_completed_at_set -> completed_at is populated after cancel
# @TEST_EDGE cancel_event_log -> RUN_CANCELLED event exists in event log
# @TEST_EDGE execute_cancelled_after_executor -> orchestrator handles CANCELLED status from executor
# @TEST_EDGE execute_zero_rows -> orchestrator handles zero-row completion
# @TEST_EDGE async_to_def_routes -> sync route handlers work with FastAPI TestClient
# @TEST_EDGE no_await_in_routes -> no accidental await in sync handlers
from datetime import UTC, datetime
import pytest

4
backend/src/plugins/translate/__tests__/test_preview.py
View File

@@ -1,8 +1,8 @@
# region TranslationPreviewTests [TYPE Module]
# @SEMANTICS: test, translate, preview, session
# @PURPOSE: Test TranslationPreview service: preview, approve/edit/reject state transitions, cost estimation, acceptance gate.
# @LAYER: Test
# @RELATION: BINDS_TO -> [TranslationPreview:Module]
# @LAYER Test
# @RELATION BINDS_TO -> [TranslationPreview]
from datetime import UTC, datetime, timedelta
import json

16
backend/src/plugins/translate/__tests__/test_scheduler.py
View File

@@ -1,14 +1,14 @@
# region TestScheduler [TYPE Module]
# @SEMANTICS: test, translate, scheduler, notification
# @PURPOSE: Tests for TranslationScheduler: CRUD, cron validation, trigger dispatch, failure notification.
# @LAYER: Test
# @RELATION: BINDS_TO -> [TranslationScheduler:Module]
# @TEST_CONTRACT: TranslationScheduler -> create, update, delete, get, get_next_executions
# @TEST_CONTRACT: execute_scheduled_translation -> failure notification, concurrency check
# @TEST_EDGE: missing_schedule -> raise ValueError
# @TEST_EDGE: concurrent_run_skip -> skip and log event
# @TEST_EDGE: execution_failure -> NotificationService called
# @TEST_EDGE: execution_success -> NotificationService NOT called
# @LAYER Test
# @RELATION BINDS_TO -> [TranslationScheduler]
# @TEST_CONTRACT TranslationScheduler -> create, update, delete, get, get_next_executions
# @TEST_CONTRACT execute_scheduled_translation -> failure notification, concurrency check
# @TEST_EDGE missing_schedule -> raise ValueError
# @TEST_EDGE concurrent_run_skip -> skip and log event
# @TEST_EDGE execution_failure -> NotificationService called
# @TEST_EDGE execution_success -> NotificationService NOT called
from datetime import UTC, datetime
import pytest

14
backend/src/plugins/translate/__tests__/test_sql_generator.py
View File

@@ -1,13 +1,13 @@
# region SQLGeneratorTests [TYPE Module]
# @SEMANTICS: test, translate, sql_generator
# @PURPOSE: Tests for SQLGenerator: PostgreSQL INSERT + UPSERT, ClickHouse INSERT, dialect quoting, NULL handling, injection safety.
# @LAYER: Test
# @RELATION: BINDS_TO -> [SQLGenerator:Module]
# @TEST_CONTRACT: SQLGenerator.generate -> SQL string, row_count
# @TEST_FIXTURE: sample_rows -> [{"col1": "val1", "col2": 42}, {"col1": "val2", "col2": 99}]
# @TEST_EDGE: empty_rows -> raises ValueError
# @TEST_EDGE: null_values -> properly encoded as NULL
# @TEST_EDGE: sql_injection -> values with single quotes are escaped
# @LAYER Test
# @RELATION BINDS_TO -> [SQLGenerator]
# @TEST_CONTRACT SQLGenerator.generate -> SQL string, row_count
# @TEST_FIXTURE sample_rows -> [{"col1": "val1", "col2": 42}, {"col1": "val2", "col2": 99}]
# @TEST_EDGE empty_rows -> raises ValueError
# @TEST_EDGE null_values -> properly encoded as NULL
# @TEST_EDGE sql_injection -> values with single quotes are escaped
import pytest
from typing import Any

12
backend/src/plugins/translate/__tests__/test_target_schema.py
View File

@@ -1,12 +1,12 @@
# region TargetSchemaValidationTests [TYPE Module]
# @SEMANTICS: test, translate, target-schema, validation
# @PURPOSE: Tests for target table schema validation.
# @LAYER: Test
# @RELATION: BINDS_TO -> [TargetSchemaValidation]
# @TEST_CONTRACT: _build_expected_columns -> list[TargetSchemaColumnInfo]
# @TEST_CONTRACT: _extract_columns_from_rows -> list[dict]
# @TEST_CONTRACT: _parse_sqllab_result -> (list[dict], bool)
# @TEST_CONTRACT: validate_target_table_schema -> TargetSchemaValidationResponse
# @LAYER Test
# @RELATION BINDS_TO -> [TargetSchemaValidation]
# @TEST_CONTRACT _build_expected_columns -> list[TargetSchemaColumnInfo]
# @TEST_CONTRACT _extract_columns_from_rows -> list[dict]
# @TEST_CONTRACT _parse_sqllab_result -> (list[dict], bool)
# @TEST_CONTRACT validate_target_table_schema -> TargetSchemaValidationResponse
import pytest
from unittest.mock import MagicMock, patch

14
backend/src/plugins/translate/__tests__/test_text_cleaner.py
View File

@@ -1,13 +1,13 @@
# #region TestTextCleaner [C:3] [TYPE Module] [SEMANTICS test, text, cleaner, whitespace, truncation]
# @BRIEF Verify text cleaning contracts — normalize_whitespace, truncate_text, clean_text.
# @RELATION BINDS_TO -> [_text_cleaner:Module]
# @TEST_EDGE: empty_string — empty/whitespace-only input returns ""
# @TEST_EDGE: whitespace_variants — multiple spaces, newlines, tabs all collapse to single space
# @TEST_EDGE: truncation_boundary — text shorter than, equal to, and longer than max_length
# @TEST_EDGE: clean_combined — clean_text correctly chains normalize + truncate
# @TEST_EDGE: zero_max_length — max_length=0 forces truncation on any non-empty text
# @RELATION BINDS_TO -> [[EXT:internal:_text_cleaner]]
# @TEST_EDGE empty_string — empty/whitespace-only input returns ""
# @TEST_EDGE whitespace_variants — multiple spaces, newlines, tabs all collapse to single space
# @TEST_EDGE truncation_boundary — text shorter than, equal to, and longer than max_length
# @TEST_EDGE clean_combined — clean_text correctly chains normalize + truncate
# @TEST_EDGE zero_max_length — max_length=0 forces truncation on any non-empty text
from src.plugins.translate._text_cleaner import clean_text, normalize_whitespace, truncate_text
from src.plugins.translate.[EXT:internal:_text_cleaner] import clean_text, normalize_whitespace, truncate_text
# region TestNormalizeWhitespace [TYPE Class]

26
backend/src/plugins/translate/__tests__/test_token_budget.py
View File

@@ -1,18 +1,18 @@
# #region TestTokenBudget [C:3] [TYPE Module] [SEMANTICS test, token, budget, estimation, batch, translate]
# @BRIEF Verify estimate_token_budget contracts — safe batch sizing, auto-reduction, warning generation.
# @RELATION BINDS_TO -> [estimate_token_budget:Module]
# @TEST_EDGE: empty_rows — empty source rows returns batch_size_adjusted=1
# @TEST_EDGE: small_rows — short text fits in a single batch at requested size
# @TEST_EDGE: large_rows — long text causes batch size reduction
# @TEST_EDGE: multi_language — more target languages increases output estimate
# @TEST_EDGE: context_columns — context columns increase input token estimate
# @TEST_EDGE: dictionary_entries — glossary entries increase input token estimate
# @TEST_EDGE: auto_calc — no batch_size specified, auto-calculates max safe
# @TEST_EDGE: exact_fit — exactly fits context window, batch_adjusted == requested
# @TEST_EDGE: conservative_min — even with huge rows, batch_size_adjusted >= 1
# @TEST_INVARIANT: batch_size_adjusted >= 1 always
# @TEST_INVARIANT: max_output_needed between MIN_MAX_TOKENS(4096) and max_output_tokens(8192)
# @TEST_INVARIANT: warning is None when batch fits, str when reduced
# @RELATION BINDS_TO -> [estimate_token_budget]
# @TEST_EDGE empty_rows — empty source rows returns batch_size_adjusted=1
# @TEST_EDGE small_rows — short text fits in a single batch at requested size
# @TEST_EDGE large_rows — long text causes batch size reduction
# @TEST_EDGE multi_language — more target languages increases output estimate
# @TEST_EDGE context_columns — context columns increase input token estimate
# @TEST_EDGE dictionary_entries — glossary entries increase input token estimate
# @TEST_EDGE auto_calc — no batch_size specified, auto-calculates max safe
# @TEST_EDGE exact_fit — exactly fits context window, batch_adjusted == requested
# @TEST_EDGE conservative_min — even with huge rows, batch_size_adjusted >= 1
# @TEST_INVARIANT batch_size_adjusted >= 1 always
# @TEST_INVARIANT max_output_needed between MIN_MAX_TOKENS(4096) and max_output_tokens(8192)
# @TEST_INVARIANT warning is None when batch fits, str when reduced
from src.plugins.translate._token_budget import DEFAULT_CONTEXT_WINDOW, DEFAULT_MAX_OUTPUT_TOKENS, estimate_token_budget

2
backend/src/plugins/translate/_batch_proc.py
View File

@@ -27,7 +27,7 @@ from ._batch_insert import insert_batch_to_target
from ._lang_detect import batch_detect, get_detector
from ._llm_call import LLMTranslationService
from ._token_budget import estimate_token_budget
from ._utils import _check_translation_cache, _compute_key_hash, _compute_source_hash
from .[EXT:internal:_utils] import _check_translation_cache, _compute_key_hash, _compute_source_hash
from .dictionary import DictionaryManager

2
backend/src/plugins/translate/_batch_sizer.py
View File

@@ -28,7 +28,7 @@ from ._token_budget import (
REASONING_OVERHEAD,
estimate_token_budget,
)
from ._utils import estimate_row_tokens
from .[EXT:internal:_utils] import estimate_row_tokens
# #region AdaptiveBatchSizer [C:3] [TYPE Class]

4
backend/src/plugins/translate/_llm_call.py
View File

@@ -8,7 +8,7 @@
# @RELATION DEPENDS_ON -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [TranslationRecord], [TranslationLanguage]
# @RELATION DEPENDS_ON -> [ContextAwarePromptBuilder]
# @RELATION DEPENDS_ON -> [_llm_http], [_llm_parse]
# @RELATION DEPENDS_ON -> [EXT:method:_llm_http], [_llm_parse]
# @PRE DB session is available. LLM provider is configured on the job.
# @POST TranslationRecord rows created for LLM-processed rows (success/fail/skip).
# @SIDE_EFFECT HTTP calls to LLM provider API; DB writes.
@@ -31,7 +31,7 @@ from ...services.llm_provider import LLMProviderService
from ._llm_http import call_openai_compatible
from ._llm_parse import parse_llm_response
from ._token_budget import estimate_token_budget
from ._utils import _enforce_dictionary
from .[EXT:internal:_utils] import _enforce_dictionary
from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE
from .prompt_builder import ContextAwarePromptBuilder

4
backend/src/plugins/translate/_token_budget.py
View File

@@ -1,8 +1,8 @@
# #region estimate_token_budget [C:3] [TYPE Module] [SEMANTICS translate, token, budget, estimation, llm]
# @BRIEF Calculate safe batch_size and max_tokens for LLM translation calls based on actual content length and model context window limits.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationExecutor:Module]
# @RELATION DEPENDS_ON -> [TranslationExecutor:Module]
# @RELATION DEPENDS_ON -> [TranslationExecutor]
# @RELATION DEPENDS_ON -> [TranslationExecutor]
# @RATIONALE Added comment clarifying PROVIDER_DEFAULTS is a fallback — primary source should be LLMProvider API.
# DeepSeek v4 Flash supports up to 64K context window; output is limited by max_tokens.

6
backend/src/plugins/translate/_utils.py
View File

@@ -1,7 +1,7 @@
# #region TranslationUtils [C:3] [TYPE Module] [SEMANTICS translate, utils, hash, dictionary, cache]
# @BRIEF Shared utility functions for the translation plugin — dictionary enforcement,
# source hashing, cache lookup. Extracted from executor.py to break circular imports.
# @LAYER: Domain
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationRecord]
# @RELATION DEPENDS_ON -> [TranslationLanguage]
# @RATIONALE Extracted from TranslationExecutor to avoid circular imports when sub-services
@@ -22,9 +22,9 @@ from ...models.translate import TranslationLanguage, TranslationRecord
# #region _normalize_term [TYPE Function]
# @BRIEF Normalize a term for case-insensitive unique constraint lookup.
# @RATIONALE: NFC normalization is applied before lowercasing to ensure consistent
# @RATIONALE NFC normalization is applied before lowercasing to ensure consistent
# comparison of Unicode characters (e.g. precomposed vs decomposed forms).
# @REJECTED: Lowercasing without NFC normalization — would cause duplicate entries
# @REJECTED Lowercasing without NFC normalization — would cause duplicate entries
# for semantically identical Unicode strings in different normalization forms.
def _normalize_term(term: str) -> str:
"""Normalize a term by NFC, lowercasing, and removing extra whitespace."""

8
backend/src/plugins/translate/dictionary.py
View File

@@ -1,10 +1,10 @@
# #region DictionaryManagerModule [C:4] [TYPE Module] [SEMANTICS sqlalchemy, translate, dictionary, batch, term]
# @BRIEF Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationJob:Class]
# @RELATION DEPENDS_ON -> [DictionaryEntry:Class]
# @RELATION DEPENDS_ON -> [TerminologyDictionary:Class]
# @RELATION DEPENDS_ON -> [TranslationJobDictionary:Class]
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION DEPENDS_ON -> [DictionaryEntry]
# @RELATION DEPENDS_ON -> [TerminologyDictionary]
# @RELATION DEPENDS_ON -> [TranslationJobDictionary]
# @RATIONALE C4 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion.
# @REJECTED "Keep both" as conflict option — UniqueConstraint prohibits variants; only overwrite/keep existing.
# @REJECTED Monolithic DictionaryManager class — violated INV_7. Decomposed into DictionaryCRUD, DictionaryEntryCRUD, DictionaryImportExport, DictionaryBatchFilter, DictionaryCorrectionService.

2
backend/src/plugins/translate/dictionary_correction.py
View File

@@ -10,7 +10,7 @@ from sqlalchemy.orm import Session
from ...core.logger import belief_scope, logger
from ...models.translate import DictionaryEntry, TerminologyDictionary
from ._utils import _normalize_term
from .[EXT:internal:_utils] import _normalize_term
class DictionaryCorrectionService:

2
backend/src/plugins/translate/dictionary_entries.py
View File

@@ -8,7 +8,7 @@ from sqlalchemy.orm import Session
from ...core.logger import belief_scope, logger
from ...models.translate import DictionaryEntry, TerminologyDictionary
from ._utils import _normalize_term
from .[EXT:internal:_utils] import _normalize_term
from .dictionary_validation import _validate_bcp47

2
backend/src/plugins/translate/dictionary_import_export.py
View File

@@ -11,7 +11,7 @@ from sqlalchemy.orm import Session
from ...core.logger import belief_scope, logger
from ...models.translate import DictionaryEntry, TerminologyDictionary
from ._utils import _detect_delimiter, _normalize_term
from .[EXT:internal:_utils] import _detect_delimiter, _normalize_term
class DictionaryImportExport:

46
backend/src/plugins/translate/events.py
View File

@@ -1,16 +1,16 @@
# #region TranslationEventLog [C:5] [TYPE Module] [SEMANTICS sqlalchemy, translate, event, log, audit]
# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement.
# @LAYER: Domain
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationEvent]
# @RELATION DEPENDS_ON -> [MetricSnapshot]
# @PRE: Database session is open and valid.
# @POST: Events are persisted immutably; terminal events enforce exactly-one invariant per run.
# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion.
# @DATA_CONTRACT: Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]
# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id.
# @RATIONALE: Immutable event log with nullable run_id allows job-level events (no run context) and run-level events.
# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# @REJECTED: Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit.
# @PRE Database session is open and valid.
# @POST Events are persisted immutably; terminal events enforce exactly-one invariant per run.
# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion.
# @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]
# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id.
# @RATIONALE Immutable event log with nullable run_id allows job-level events (no run context) and run-level events.
# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.
# @REJECTED Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit.
from datetime import UTC, datetime, timedelta
from typing import Any
import uuid
@@ -39,10 +39,10 @@ DEFAULT_RETENTION_DAYS = 90
# #region TranslationEventLog [C:5] [TYPE Class]
# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement.
# @PRE: Database session is available.
# @POST: Events are written immutably; terminal events enforced per run.
# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events.
# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id.
# @PRE Database session is available.
# @POST Events are written immutably; terminal events enforced per run.
# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events.
# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id.
class TranslationEventLog:
def __init__(self, db: Session):
@@ -50,9 +50,9 @@ class TranslationEventLog:
# region log_event [TYPE Function]
# @PURPOSE: Write an immutable event. Enforces terminal event invariant for non-null run_id.
# @PRE: event_type must be a known type. If run_id is not None, enforce terminal invariant.
# @POST: TranslationEvent row is created.
# @SIDE_EFFECT: DB write.
# @PRE event_type must be a known type. If run_id is not None, enforce terminal invariant.
# @POST TranslationEvent row is created.
# @SIDE_EFFECT DB write.
def log_event(
self,
job_id: str,
@@ -123,8 +123,8 @@ class TranslationEventLog:
# region query_events [TYPE Function]
# @PURPOSE: Query events with optional filters.
# @PRE: None.
# @POST: Returns list of TranslationEvent dicts matching filters.
# @PRE None.
# @POST Returns list of TranslationEvent dicts matching filters.
def query_events(
self,
job_id: str | None = None,
@@ -166,9 +166,9 @@ class TranslationEventLog:
# region prune_expired [TYPE Function]
# @PURPOSE: Delete events older than retention_days. Persists MetricSnapshot before pruning.
# @PRE: None.
# @POST: Expired events are deleted; MetricSnapshot is created before deletion.
# @SIDE_EFFECT: Creates MetricSnapshot row; deletes TranslationEvent rows.
# @PRE None.
# @POST Expired events are deleted; MetricSnapshot is created before deletion.
# @SIDE_EFFECT Creates MetricSnapshot row; deletes TranslationEvent rows.
def prune_expired(
self,
retention_days: int = DEFAULT_RETENTION_DAYS,
@@ -247,8 +247,8 @@ class TranslationEventLog:
# region get_run_event_summary [TYPE Function]
# @PURPOSE: Get a summary of events for a run, including invariant check.
# @PRE: run_id is not None.
# @POST: Returns dict with event list and invariant validity.
# @PRE run_id is not None.
# @POST Returns dict with event list and invariant validity.
def get_run_event_summary(self, run_id: str) -> dict[str, Any]:
with belief_scope("TranslationEventLog.get_run_event_summary"):
events = self.query_events(run_id=run_id)

4
backend/src/plugins/translate/executor.py
View File

@@ -13,7 +13,7 @@
# @INVARIANT Batch processing is independent — one batch failure does not affect others.
# @RATIONALE Extracted from monolithic executor.py (1974 lines) into thin orchestrator
# to comply with INV_7. Sub-services in _run_service, _batch_proc, _llm_call, _batch_sizer.
# Module-level helpers moved to _utils.py.
# Module-level helpers moved to [EXT:internal:_utils].py.
# @REJECTED Keeping monolithic executor.py at 1974 lines — violates INV_7 by +1574 lines.
# Single monolithic LLM call — would lose all progress on any failure.
@@ -29,7 +29,7 @@ from ...models.translate import TranslationJob, TranslationRun, TranslationRunLa
from ...services.llm_provider import LLMProviderService
from ._run_service import RunExecutionService
from ._token_budget import estimate_token_budget
from ._utils import _check_translation_cache, _compute_source_hash, _enforce_dictionary, estimate_row_tokens
from .[EXT:internal:_utils] import _check_translation_cache, _compute_source_hash, _enforce_dictionary, estimate_row_tokens
__all__ = [
"TranslationExecutor", "estimate_row_tokens",

8
backend/src/plugins/translate/metrics.py
View File

@@ -1,10 +1,10 @@
# #region TranslationMetrics [C:3] [TYPE Module] [SEMANTICS sqlalchemy, translate, metrics, job, statistics]
# @BRIEF Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [TranslationSchedule]
# @RELATION DEPENDS_ON -> [TranslationSchedule]
# @RELATION DEPENDS_ON -> [TranslationSchedule]
# @RELATION DEPENDS_ON -> [TranslationSchedule]
# @PRE Database session is open.
# @POST Metrics are aggregated and returned; no side effects.
# @RATIONALE Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture.

16
backend/src/plugins/translate/orchestrator.py
View File

@@ -61,9 +61,9 @@ class TranslationOrchestrator:
# region start_run [TYPE Function]
# @PURPOSE: Start a new translation run for a job.
# @PRE: job_id exists. For manual runs, an accepted preview session must exist.
# @POST: TranslationRun is created in PENDING status with hash fields and config snapshot.
# @SIDE_EFFECT: DB writes.
# @PRE job_id exists. For manual runs, an accepted preview session must exist.
# @POST TranslationRun is created in PENDING status with hash fields and config snapshot.
# @SIDE_EFFECT DB writes.
def start_run(
self,
job_id: str,
@@ -82,9 +82,9 @@ class TranslationOrchestrator:
# region execute_run [TYPE Function]
# @PURPOSE: Execute a translation run: dispatch executor, generate SQL, submit to Superset.
# @PRE: run is in PENDING status.
# @POST: Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT: LLM calls, DB writes, Superset API calls.
# @PRE run is in PENDING status.
# @POST Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
def execute_run(
self,
run: TranslationRun,
@@ -119,7 +119,7 @@ class TranslationOrchestrator:
# region _generate_and_insert_sql [TYPE Function] [SEMANTICS backward-compat wrapper]
# @PURPOSE: Backward-compatible delegating wrapper for SQL generation and insert.
# @SIDE_EFFECT: Delegates to SQLInsertService. May call Superset API.
# @SIDE_EFFECT Delegates to SQLInsertService. May call Superset API.
def _generate_and_insert_sql(
self,
job: Any,
@@ -133,7 +133,7 @@ class TranslationOrchestrator:
# region _update_language_stats [TYPE Function] [SEMANTICS backward-compat wrapper]
# @PURPOSE: Backward-compatible delegating wrapper for language stats update.
# @SIDE_EFFECT: Delegates to TranslationResultAggregator.update_language_stats.
# @SIDE_EFFECT Delegates to TranslationResultAggregator.update_language_stats.
def _update_language_stats(
self,
run_id: str,

4
backend/src/plugins/translate/orchestrator_aggregator.py
View File

@@ -7,7 +7,7 @@
# @RELATION DEPENDS_ON -> [TranslationRecord]
# @RELATION DEPENDS_ON -> [TranslationRunLanguageStats]
# @RELATION DEPENDS_ON -> [TranslationEventLog]
# @RELATION DEPENDS_ON -> [orchestrator_lang_stats]
# @RELATION DEPENDS_ON -> [EXT:frontend:orchestrator_lang_stats]
# @RELATION DEPENDS_ON -> [orchestrator_query]
# @RATIONALE Language stats aggregation extracted to orchestrator_lang_stats; query methods to orchestrator_query.
@@ -119,7 +119,7 @@ class TranslationResultAggregator:
# region update_language_stats [TYPE Function]
# @PURPOSE: Aggregate TranslationLanguage entries and update TranslationRunLanguageStats.
# @SIDE_EFFECT: DB writes on language_stats objects.
# @SIDE_EFFECT DB writes on language_stats objects.
def update_language_stats(
self,
run_id: str,

6
backend/src/plugins/translate/orchestrator_exec.py
View File

@@ -52,9 +52,9 @@ class TranslationExecutionEngine:
# region execute_run [TYPE Function]
# @PURPOSE: Execute a translation run: dispatch executor, handle outcomes.
# @PRE: run is in PENDING status.
# @POST: Run executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT: LLM calls, DB writes, Superset API calls.
# @PRE run is in PENDING status.
# @POST Run executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
def execute_run(
self,
run: TranslationRun,

6
backend/src/plugins/translate/orchestrator_planner.py
View File

@@ -37,9 +37,9 @@ class TranslationPlanner:
# region plan_run [TYPE Function]
# @PURPOSE: Validate, compute hashes, and create a new TranslationRun.
# @PRE: job_id exists. For manual runs, an accepted preview session exists.
# @POST: TranslationRun is created in PENDING status with hash fields and config snapshot.
# @SIDE_EFFECT: DB writes; event recorded.
# @PRE job_id exists. For manual runs, an accepted preview session exists.
# @POST TranslationRun is created in PENDING status with hash fields and config snapshot.
# @SIDE_EFFECT DB writes; event recorded.
def plan_run(
self,
job_id: str,

6
backend/src/plugins/translate/orchestrator_retry.py
View File

@@ -44,7 +44,7 @@ class TranslationRunRetryManager:
# region retry_failed_batches [TYPE Function]
# @PURPOSE: Retry failed batches in a run.
# @SIDE_EFFECT: Re-executes batch translations; DB writes.
# @SIDE_EFFECT Re-executes batch translations; DB writes.
def retry_failed_batches(self, run_id: str) -> TranslationRun:
with belief_scope("TranslationRunRetryManager.retry_failed_batches"):
run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()
@@ -121,14 +121,14 @@ class TranslationRunRetryManager:
# region retry_insert [TYPE Function]
# @PURPOSE: Retry the SQL insert phase for a completed run.
# @SIDE_EFFECT: Superset API call; DB writes.
# @SIDE_EFFECT Superset API call; DB writes.
def retry_insert(self, run_id: str) -> TranslationRun:
return _retry_insert(self.db, self.config_manager, self.event_log, self.current_user, run_id)
# endregion retry_insert
# region cancel_run [TYPE Function]
# @PURPOSE: Cancel a running translation.
# @SIDE_EFFECT: DB writes; event log.
# @SIDE_EFFECT DB writes; event log.
def cancel_run(self, run_id: str) -> TranslationRun:
return _cancel_run(self.db, self.event_log, self.current_user, run_id)
# endregion cancel_run

6
backend/src/plugins/translate/orchestrator_run_completion.py
View File

@@ -15,7 +15,7 @@ from ...models.translate import TranslationJob, TranslationRun, TranslationRunLa
# region handle_executor_failure [TYPE Function]
# @PURPOSE: Handle executor failure — rollback, mark run as FAILED, log event.
# @SIDE_EFFECT: DB writes; event log.
# @SIDE_EFFECT DB writes; event log.
def handle_executor_failure(
db,
event_log,
@@ -47,7 +47,7 @@ def handle_executor_failure(
# region complete_cancelled [TYPE Function]
# @PURPOSE: Finalize a cancelled run — update language stats and log.
# @SIDE_EFFECT: DB writes; event log.
# @SIDE_EFFECT DB writes; event log.
def complete_cancelled(
db,
event_log,
@@ -69,7 +69,7 @@ def complete_cancelled(
# region complete_success [TYPE Function]
# @PURPOSE: Finalize a successful run — update stats, optionally insert SQL, commit.
# @SIDE_EFFECT: DB writes; event log; Superset API call if skip_insert is False.
# @SIDE_EFFECT DB writes; event log; Superset API call if skip_insert is False.
def complete_success(
db,
event_log,

6
backend/src/plugins/translate/orchestrator_runner.py
View File

@@ -38,9 +38,9 @@ class TranslationStageRunner:
# region execute_run [TYPE Function]
# @PURPOSE: Execute a translation run: dispatch executor, generate SQL, submit to Superset.
# @PRE: run is in PENDING status.
# @POST: Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT: LLM calls, DB writes, Superset API calls.
# @PRE run is in PENDING status.
# @POST Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
def execute_run(
self,
run: TranslationRun,

2
backend/src/plugins/translate/orchestrator_sql_rows.py
View File

@@ -50,7 +50,7 @@ def build_context_keys(job: TranslationJob, effective_target: str | None) -> lis
# #region build_rows [C:2] [TYPE Function] [SEMANTICS sql, rows, build]
# @BRIEF Build row data for SQL INSERT with per-language expansion.
# @SIDE_EFFECT: Reads translation record language entries.
# @SIDE_EFFECT Reads translation record language entries.
def build_rows(
records: list[TranslationRecord],
job: TranslationJob,

2
backend/src/plugins/translate/plugin.py
View File

@@ -12,7 +12,7 @@ from ...core.plugin_base import PluginBase
# #region TranslatePlugin [TYPE Class]
# @BRIEF Plugin for translating SQL queries and dashboard definitions across database dialects.
# @RELATION IMPLEMENTS -> backend.src.core.plugin_base.PluginBase
# @RELATION IMPLEMENTS -> [EXT:path:backend.src.core.plugin_base.PluginBase]
class TranslatePlugin(PluginBase):
@property
def id(self) -> str:

2
backend/src/plugins/translate/preview.py
View File

@@ -65,7 +65,7 @@ class TranslationPreview:
# region preview_rows [TYPE Function]
# @PURPOSE: Fetch sample rows, send to LLM, create preview session with per-language records.
# @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates DB rows.
# @SIDE_EFFECT Fetches data from Superset; calls LLM; creates DB rows.
def preview_rows(
self,
job_id: str,

4
backend/src/plugins/translate/preview_executor.py
View File

@@ -37,7 +37,7 @@ class PreviewExecutor:
# region fetch_sample_rows [TYPE Function]
# @PURPOSE: Fetch sample rows from Superset dataset for preview.
# @SIDE_EFFECT: Calls Superset chart data endpoint.
# @SIDE_EFFECT Calls Superset chart data endpoint.
def fetch_sample_rows(
self,
job: TranslationJob,
@@ -84,7 +84,7 @@ class PreviewExecutor:
# region call_llm [TYPE Function]
# @PURPOSE: Call the configured LLM provider with a prompt.
# @SIDE_EFFECT: Makes HTTP call to LLM provider.
# @SIDE_EFFECT Makes HTTP call to LLM provider.
def call_llm(self, job: TranslationJob, prompt: str, max_tokens: int = 8192) -> str:
with belief_scope("PreviewExecutor.call_llm"):
if not job.provider_id:

4
backend/src/plugins/translate/preview_prompt_builder.py
View File

@@ -29,8 +29,8 @@ class PreviewPromptBuilder:
# region build_prompt_from_rows [TYPE Function]
# @PURPOSE: Build the complete LLM prompt from source rows, dictionary, and job config.
# @PRE: job has valid configuration. source_rows is non-empty.
# @POST: Returns prompt string and token budget metadata.
# @PRE job has valid configuration. source_rows is non-empty.
# @POST Returns prompt string and token budget metadata.
def build_prompt_from_rows(
self,
job: TranslationJob,

2
backend/src/plugins/translate/preview_session_ops.py
View File

@@ -22,7 +22,7 @@ from .preview_session_serializer import (
# #region accept_preview_session [C:2] [TYPE Function] [SEMANTICS preview, session, accept]
# @BRIEF Mark a preview session as accepted, which gates full execution.
# @SIDE_EFFECT: DB writes on session status.
# @SIDE_EFFECT DB writes on session status.
def accept_preview_session(db: Session, job_id: str, _current_user: str | None = None) -> dict[str, Any]:
"""Mark a preview session as accepted and return the session data with records."""
session = (

2
backend/src/plugins/translate/prompt_builder.py
View File

@@ -1,7 +1,7 @@
# #region ContextAwarePromptBuilder [C:2] [TYPE Module] [SEMANTICS translate, prompt, context, dictionary]
# @BRIEF Pure-function prompt builder that enhances dictionary entries with context annotations.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [DictionaryEntry:Class]
# @RELATION DEPENDS_ON -> [DictionaryEntry]
# @RATIONALE Pure functions only — no I/O, no DB access. Separated from executor for testability.
# @REJECTED Embedding context inline in the executor would make it untestable without mocking DB.

48
backend/src/plugins/translate/scheduler.py
View File

@@ -1,16 +1,16 @@
# #region TranslationScheduler [C:4] [TYPE Module] [SEMANTICS sqlalchemy, translate, schedule, cron, job]
# @BRIEF Manage TranslationSchedule rows and register them with core SchedulerService.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [SchedulerService:Class]
# @RELATION DEPENDS_ON -> [TranslationOrchestrator:Class]
# @RELATION DEPENDS_ON -> [TranslationEventLog:Class]
# @PRE: Database session and SchedulerService are available.
# @POST: TranslationSchedule CRUD persisted; APScheduler jobs registered/updated/removed.
# @SIDE_EFFECT: Registers APScheduler jobs; runs translations on trigger; creates events.
# @RATIONALE: Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance.
# @REJECTED: Separate scheduler instance would create resource contention.
# @REJECTED: Polling-based approach — event-driven APScheduler is more precise.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationSchedule]
# @RELATION DEPENDS_ON -> [SchedulerService]
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
# @RELATION DEPENDS_ON -> [TranslationEventLog]
# @PRE Database session and SchedulerService are available.
# @POST TranslationSchedule CRUD persisted; APScheduler jobs registered/updated/removed.
# @SIDE_EFFECT Registers APScheduler jobs; runs translations on trigger; creates events.
# @RATIONALE Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance.
# @REJECTED Separate scheduler instance would create resource contention.
# @REJECTED Polling-based approach — event-driven APScheduler is more precise.
from datetime import UTC, datetime, timedelta
import uuid
@@ -37,8 +37,8 @@ class TranslationScheduler:
# region create_schedule [TYPE Function]
# @PURPOSE: Create a new schedule for a job.
# @PRE: job_id exists. cron_expression is valid.
# @POST: TranslationSchedule row created.
# @PRE job_id exists. cron_expression is valid.
# @POST TranslationSchedule row created.
def create_schedule(
self,
job_id: str,
@@ -83,8 +83,8 @@ class TranslationScheduler:
# region update_schedule [TYPE Function]
# @PURPOSE: Update an existing schedule.
# @PRE: job_id has an existing schedule.
# @POST: Schedule updated.
# @PRE job_id has an existing schedule.
# @POST Schedule updated.
def update_schedule(
self,
job_id: str,
@@ -130,8 +130,8 @@ class TranslationScheduler:
# region delete_schedule [TYPE Function]
# @PURPOSE: Delete a schedule for a job.
# @PRE: job_id has an existing schedule.
# @POST: Schedule deleted.
# @PRE job_id has an existing schedule.
# @POST Schedule deleted.
def delete_schedule(self, job_id: str) -> None:
with belief_scope("TranslationScheduler.delete_schedule"):
schedule = self.db.query(TranslationSchedule).filter(
@@ -156,8 +156,8 @@ class TranslationScheduler:
# region enable_disable_schedule [TYPE Function]
# @PURPOSE: Enable or disable a schedule.
# @PRE: job_id has an existing schedule.
# @POST: Schedule is_active updated.
# @PRE job_id has an existing schedule.
# @POST Schedule is_active updated.
def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule:
with belief_scope("TranslationScheduler.set_schedule_active"):
schedule = self.db.query(TranslationSchedule).filter(
@@ -181,8 +181,8 @@ class TranslationScheduler:
# region get_schedule [TYPE Function]
# @PURPOSE: Get schedule for a job.
# @PRE: job_id exists.
# @POST: Returns TranslationSchedule or raises ValueError.
# @PRE job_id exists.
# @POST Returns TranslationSchedule or raises ValueError.
def get_schedule(self, job_id: str) -> TranslationSchedule:
with belief_scope("TranslationScheduler.get_schedule"):
schedule = self.db.query(TranslationSchedule).filter(
@@ -195,7 +195,7 @@ class TranslationScheduler:
# region list_active_schedules [TYPE Function]
# @PURPOSE: List all active schedules.
# @POST: Returns list of active TranslationSchedule rows.
# @POST Returns list of active TranslationSchedule rows.
@staticmethod
def list_active_schedules(db: Session) -> list[TranslationSchedule]:
return (
@@ -207,8 +207,8 @@ class TranslationScheduler:
# region get_next_executions [TYPE Function]
# @PURPOSE: Compute next N execution times from cron expression.
# @PRE: cron_expression is valid.
# @POST: Returns list of ISO datetime strings.
# @PRE cron_expression is valid.
# @POST Returns list of ISO datetime strings.
@staticmethod
def get_next_executions(cron_expression: str, timezone_str: str = "UTC", n: int = 3) -> list[str]:
from zoneinfo import ZoneInfo

10
backend/src/plugins/translate/service.py
View File

@@ -9,7 +9,7 @@
# @SIDE_EFFECT Queries Superset for column metadata and database dialect at save time.
# @RATIONALE Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity.
# @REJECTED Monolithic service.py — violated INV_7. Decomposed into service_datasource, service_inline_correction, service_bulk_replace, service_utils.
# @REJECTED Monolithic service.py — violated INV_7. Decomposed into service_datasource, service_inline_correction, service_bulk_replace, service[EXT:internal:_utils].
from datetime import UTC, datetime
from typing import Any
@@ -23,7 +23,7 @@ from ...models.translate import TranslationJob, TranslationJobDictionary
from ...schemas.translate import TranslateJobCreate, TranslateJobUpdate
from .dictionary import _validate_bcp47
from .service_datasource import fetch_datasource_metadata
from .service_utils import _extract_dialect, job_to_response
from .service[EXT:internal:_utils] import _extract_dialect, job_to_response
# #region TranslateJobService [TYPE Class]
@@ -64,7 +64,7 @@ class TranslateJobService:
# region create_job [TYPE Function]
# @PURPOSE: Create a new translation job with column validation.
# @SIDE_EFFECT: Validates columns via SupersetClient; caches database_dialect.
# @SIDE_EFFECT Validates columns via SupersetClient; caches database_dialect.
def create_job(self, payload: TranslateJobCreate) -> TranslationJob:
logger.info(f"[TranslateJobService] Creating job '{payload.name}'")
if payload.source_datasource_id and not payload.translation_column:
@@ -132,7 +132,7 @@ class TranslateJobService:
# region update_job [TYPE Function]
# @PURPOSE: Update an existing translation job.
# @SIDE_EFFECT: Re-detects database_dialect if source_datasource_id changed.
# @SIDE_EFFECT Re-detects database_dialect if source_datasource_id changed.
def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob:
logger.info(f"[TranslateJobService] Updating job '{job_id}'")
job = self.get_job(job_id)
@@ -272,5 +272,5 @@ from .service_datasource import ( # noqa: E402, F401
get_dialect_from_database,
)
from .service_inline_correction import InlineCorrectionService # noqa: E402, F401
from .service_utils import _extract_dialect, job_to_response # noqa: E402, F401
from .service[EXT:internal:_utils] import _extract_dialect, job_to_response # noqa: E402, F401
# #endregion TranslateJobService

2
backend/src/plugins/translate/service_inline_correction.py
View File

@@ -14,7 +14,7 @@ from sqlalchemy.orm import Session
from ...core.logger import belief_scope, logger
from ...models.translate import DictionaryEntry, TranslationLanguage, TranslationRecord
from ._utils import _normalize_term
from .[EXT:internal:_utils] import _normalize_term
class InlineCorrectionService:

4
backend/src/plugins/translate/service_target_schema.py
View File

@@ -3,8 +3,8 @@
# сравнение с ожидаемыми (из build_columns), возврат diff.
# @LAYER Service
# @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor]
# @RELATION DEPENDS_ON -> [schemas.translate.TargetSchemaValidationRequest]
# @RELATION DEPENDS_ON -> [schemas.translate.TargetSchemaValidationResponse]
# @RELATION DEPENDS_ON -> [EXT:method:schemas.translate.TargetSchemaValidationRequest]
# @RELATION DEPENDS_ON -> [EXT:method:schemas.translate.TargetSchemaValidationResponse]
# @PRE Superset окружение доступно, target_database_id валиден.
# @POST Возвращает актуальные, ожидаемые, отсутствующие и лишние колонки.
# @SIDE_EFFECT Выполняет SQL-запрос через Superset SQL Lab.

32
backend/src/plugins/translate/sql_generator.py
View File

@@ -1,14 +1,14 @@
# #region SQLGenerator [C:3] [TYPE Module] [SEMANTICS clickhouse, translate, sql, insert, generate]
# @BRIEF Dialect-aware safe SQL generation for INSERT/UPSERT operations.
# @LAYER: Domain
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RELATION DEPENDS_ON -> [TranslationRun]
# @PRE: Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS.
# @POST: Returns safe SQL strings for the target dialect.
# @SIDE_EFFECT: None — pure code generation.
# @RATIONALE: Dialect-aware SQL uses ON CONFLICT for PostgreSQL; plain INSERT for ClickHouse with documented limitations.
# @REJECTED: UPDATE statements — source is append-only; UPSERT covers overwrite case.
# @REJECTED: ORM-based insert bypasses Superset's SQL Lab audit trail.
# @PRE Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS.
# @POST Returns safe SQL strings for the target dialect.
# @SIDE_EFFECT None — pure code generation.
# @RATIONALE Dialect-aware SQL uses ON CONFLICT for PostgreSQL; plain INSERT for ClickHouse with documented limitations.
# @REJECTED UPDATE statements — source is append-only; UPSERT covers overwrite case.
# @REJECTED ORM-based insert bypasses Superset's SQL Lab audit trail.
from datetime import UTC, datetime
from typing import Any
@@ -60,8 +60,8 @@ def _normalize_timestamp_value(value: Any) -> str | None:
# #region _quote_identifier [C:4] [TYPE Function]
# @BRIEF Quote an identifier per dialect rules. PostgreSQL uses double quotes; ClickHouse uses backticks.
# @PRE: identifier is a non-empty string.
# @POST: Returns safely quoted identifier.
# @PRE identifier is a non-empty string.
# @POST Returns safely quoted identifier.
def _quote_identifier(identifier: str, dialect: str) -> str:
"""Quote a SQL identifier per dialect rules."""
if not identifier:
@@ -212,15 +212,15 @@ def generate_upsert_sql(
# #region SQLGenerator [C:3] [TYPE Class]
# @BRIEF Generate safe, dialect-appropriate SQL INSERT/UPSERT statements.
# @PRE: Job has target_schema, target_table, key columns configured.
# @POST: Returns generated SQL string for the target dialect.
# @PRE Job has target_schema, target_table, key columns configured.
# @POST Returns generated SQL string for the target dialect.
class SQLGenerator:
# region SQLGenerator.generate [TYPE Function]
# @PURPOSE: Generate SQL for a set of rows, detecting dialect from the job configuration.
# @PRE: dialect is a supported database dialect. columns list is non-empty. rows is non-empty.
# @POST: Returns tuple of (sql_string, statement_count).
# @SIDE_EFFECT: None — pure SQL generation.
# @PRE dialect is a supported database dialect. columns list is non-empty. rows is non-empty.
# @POST Returns tuple of (sql_string, statement_count).
# @SIDE_EFFECT None — pure SQL generation.
@staticmethod
def generate(
dialect: str,
@@ -334,8 +334,8 @@ class SQLGenerator:
# region SQLGenerator.generate_batch [TYPE Function]
# @PURPOSE: Generate separate INSERT statements for each row (batch-safe version).
# @PRE: Same as generate().
# @POST: Returns list of (sql_string, row_index) tuples.
# @PRE Same as generate().
# @POST Returns list of (sql_string, row_index) tuples.
@staticmethod
def generate_batch(
dialect: str,