busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(translate): add tests for ConnectionService, DbExecutor, orchestrator direct DB dispatch

Browse Source 
- 9 new enhancement test files: test_connection_service.py,
  test_db_executor.py, test_orchestrator_direct_db.py, test_batch_insert.py,
  test_lang_stats.py, test_response_field_coverage.py, test_retry.py,
  test_run_service.py, test_sql_insert_service.py
- 5 new integration tests: test_superset_sqllab_e2e.py,
  test_translate_clickhouse.py, test_translate_corrections.py,
  test_translate_schedules.py, test_translate_status_fk.py
- Updated existing tests for insert_method/connection_id fields
This commit is contained in:
busya
2026-06-11 19:12:45 +03:00
parent 55604498ae
commit 50d6b9226e
23 changed files with 3441 additions and 726 deletions
Download Patch File Download Diff File Expand all files Collapse all files

232
backend/src/plugins/translate/__tests__/test_batch_insert.py Normal file
View File

@@ -0,0 +1,232 @@
# #region TestBatchInsert [C:3] [TYPE Module] [SEMANTICS test,translate,batch,insert]
# @BRIEF Tests for _batch_insert.py — column building, row building, context keys.
# @RELATION BINDS_TO -> [_batch_insert_module]
import uuid
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from src.models.translate import TranslationJob, TranslationLanguage, TranslationRecord
from src.plugins.translate._batch_insert import (
_build_target_columns,
_build_context_keys,
_build_insert_rows,
_fetch_batch_records,
_generate_insert_sql,
)
# #region make_job [C:1] [TYPE Function]
def make_job(**overrides) -> TranslationJob:
job = MagicMock(spec=TranslationJob)
job.target_key_cols = ["id"]
job.target_column = "translated_text"
job.translation_column = "source_text"
job.target_language_column = "lang_code"
job.target_source_column = "source_text"
job.target_source_language_column = "src_lang"
job.target_schema = "public"
job.target_table = "translations"
job.target_languages = ["ru"]
job.context_columns = []
job.target_dialect = "clickhouse"
job.database_dialect = None
job.upsert_strategy = "INSERT"
job.environment_id = "env-1"
job.source_dialect = "postgresql"
for k, v in overrides.items():
setattr(job, k, v)
return job
def make_record(**overrides) -> TranslationRecord:
rec = MagicMock(spec=TranslationRecord)
rec.id = str(uuid.uuid4())
rec.source_sql = "Hello"
rec.target_sql = "Привет"
rec.source_data = {"id": 1, "extra": "data"}
rec.status = "SUCCESS"
rec.languages = []
for k, v in overrides.items():
setattr(rec, k, v)
return rec
def make_lang(code: str = "ru", value: str = "Привет", detected: str = "en", status: str = "completed") -> MagicMock:
lang = MagicMock(spec=TranslationLanguage)
lang.language_code = code
lang.translated_value = value
lang.final_value = value
lang.source_language_detected = detected
lang.status = status
return lang
# #endregion make_job
# #region TestBuildColumns [C:2] [TYPE Class]
class TestBuildColumns:
"""Verify _build_target_columns assembles correct column list."""
def test_basic_columns(self):
job = make_job()
cols = _build_target_columns(job, "translated_text")
assert "translated_text" in cols
assert "lang_code" in cols
assert "source_text" in cols
assert "src_lang" in cols
assert "context" in cols
assert "is_original" in cols
def test_no_key_cols(self):
job = make_job(target_key_cols=[])
cols = _build_target_columns(job, "translated_text")
assert "id" not in cols
def test_dedup_same_column(self):
job = make_job(target_key_cols=["source_text"])
cols = _build_target_columns(job, "translated_text")
# source_text appears in key_cols and in target_source_column — should be deduped
assert cols.count("source_text") == 1
def test_no_effective_target(self):
job = make_job(target_column=None, translation_column=None)
cols = _build_target_columns(job, None)
assert all(c in cols for c in ["context", "is_original"])
# #endregion TestBuildColumns
# #region TestBuildContextKeys [C:2] [TYPE Class]
class TestBuildContextKeys:
"""Verify _build_context_keys builds correct context list."""
def test_empty_context(self):
job = make_job(context_columns=[])
keys = _build_context_keys(job, "translated_text")
assert "source_text" in keys # translation_column != effective_target
def test_with_context_columns(self):
job = make_job(context_columns=["category", "author"])
keys = _build_context_keys(job, "translated_text")
assert "category" in keys
assert "author" in keys
def test_no_extra_when_same(self):
job = make_job(translation_column="translated_text")
keys = _build_context_keys(job, "translated_text")
assert keys == []
# #endregion TestBuildContextKeys
# #region TestBuildInsertRows [C:2] [TYPE Class]
class TestBuildInsertRows:
"""Verify _build_insert_rows produces correct row dicts."""
def test_single_record_no_languages(self):
job = make_job(target_key_cols=["id"])
rec = make_record()
rows = _build_insert_rows([rec], job, "translated_text", "ru", [])
# Should produce 2 rows: original + fallback
assert len(rows) == 2
assert rows[0]["is_original"] == 1
assert rows[1]["is_original"] == 0
assert rows[1]["translated_text"] == rec.target_sql
def test_single_record_with_languages(self):
job = make_job(target_key_cols=["id"])
rec = make_record(languages=[make_lang("ru", "Привет мир")])
rows = _build_insert_rows([rec], job, "translated_text", "ru", [])
# 2 rows: original + translated
assert len(rows) == 2
assert rows[1]["translated_text"] == "Привет мир"
assert rows[1]["lang_code"] == "ru"
def test_skip_matching_language(self):
"""Skip language code that matches detected source language."""
job = make_job(target_key_cols=["id"])
rec = make_record(languages=[make_lang("en", "Hello", detected="en")])
rows = _build_insert_rows([rec], job, "translated_text", "en", [])
# Only the original row — language is skipped because en == detected
assert len(rows) == 1
def test_source_column_values(self):
job = make_job(target_key_cols=["id"])
rec = make_record()
rows = _build_insert_rows([rec], job, "translated_text", "ru", [])
assert rows[0]["source_text"] == "Hello"
assert rows[0]["src_lang"] == "und" # no languages on record
def test_context_data(self):
job = make_job(target_key_cols=["id"], context_columns=["extra"])
rec = make_record()
rows = _build_insert_rows([rec], job, "translated_text", "ru", ["extra"])
import json
context = json.loads(rows[0]["context"])
assert context.get("extra") == "data"
# #endregion TestBuildInsertRows
# #region TestFetchBatchRecords [C:2] [TYPE Class]
class TestFetchBatchRecords:
"""Verify _fetch_batch_records queries correctly."""
def test_fetch_returns_records(self):
db = MagicMock()
db.query.return_value.options.return_value.filter.return_value.all.return_value = [make_record()]
result = _fetch_batch_records(db, "batch-1")
assert len(result) == 1
def test_fetch_empty(self):
db = MagicMock()
db.query.return_value.options.return_value.filter.return_value.all.return_value = []
result = _fetch_batch_records(db, "batch-1")
assert result == []
# #endregion TestFetchBatchRecords
# #region TestGenerateInsertSQL [C:2] [TYPE Class]
class TestGenerateInsertSQL:
"""Verify _generate_insert_sql produces SQL."""
def test_generates_sql(self):
job = make_job(target_schema="public", target_table="test_table", upsert_strategy="INSERT")
sql, count = _generate_insert_sql(
dialect="clickhouse",
job=job,
columns=["id", "translated_text"],
rows_for_sql=[{"id": 1, "translated_text": "hello"}],
)
assert sql is not None
assert "INSERT INTO" in sql
assert count == 1
def test_generates_upsert(self):
job = make_job(target_schema="public", target_table="test_table", upsert_strategy="MERGE", target_key_cols=["id"])
sql, count = _generate_insert_sql(
dialect="clickhouse",
job=job,
columns=["id", "translated_text"],
rows_for_sql=[{"id": 1, "translated_text": "hello"}],
)
assert sql is not None
assert count == 1
def test_unknown_dialect_falls_back(self):
job = make_job(target_schema="public", target_table="test")
sql, count = _generate_insert_sql(
dialect="unknown_dialect_xyz",
job=job,
columns=["col1"],
rows_for_sql=[{"col1": "val"}],
)
# Falls back to default quoting
assert sql is not None
assert count == 1
# #endregion TestGenerateInsertSQL
# #endregion TestBatchInsert

168
backend/src/plugins/translate/__tests__/test_lang_stats.py Normal file
View File

@@ -0,0 +1,168 @@
# #region TestLangStats [C:2] [TYPE Module] [SEMANTICS test,translate,language,stats]
# @BRIEF Tests for orchestrator_lang_stats.update_language_stats — aggregation of per-language statistics.
# @RELATION BINDS_TO -> [update_language_stats]
import uuid
from unittest.mock import MagicMock
from src.models.translate import TranslationRecord, TranslationLanguage, TranslationRunLanguageStats
from src.plugins.translate.orchestrator_lang_stats import update_language_stats
# #region make_lang_entry [C:1] [TYPE Function]
def make_lang(code: str, status: str = "translated", value: str = "hello") -> TranslationLanguage:
le = MagicMock(spec=TranslationLanguage)
le.language_code = code
le.status = status
le.translated_value = value
return le
def make_record(run_id: str) -> TranslationRecord:
rec = MagicMock(spec=TranslationRecord)
rec.id = str(uuid.uuid4())
rec.run_id = run_id
return rec
# #endregion make_lang_entry
# #region TestLangStatsAggregation [C:2] [TYPE Class]
class TestLangStatsAggregation:
"""Verify update_language_stats aggregates correctly."""
def test_single_language_all_translated(self):
db = MagicMock()
run_id = "run-1"
ru_stat = MagicMock(spec=TranslationRunLanguageStats)
records = [make_record(run_id)]
db.query.return_value.filter.return_value.all.side_effect = [
records,
[make_lang("ru", "translated", "Привет")],
]
update_language_stats(db, None, run_id, {"ru": ru_stat})
assert ru_stat.total_rows == 1
assert ru_stat.translated_rows == 1
assert ru_stat.failed_rows == 0
assert ru_stat.skipped_rows == 0
def test_mixed_statuses(self):
db = MagicMock()
run_id = "run-2"
en_stat = MagicMock(spec=TranslationRunLanguageStats)
records = [make_record(run_id)]
langs = [
make_lang("en", "translated", "Hello"),
make_lang("en", "failed", ""),
make_lang("en", "skipped", ""),
]
db.query.return_value.filter.return_value.all.side_effect = [
records,
langs,
]
update_language_stats(db, None, run_id, {"en": en_stat})
assert en_stat.total_rows == 3
assert en_stat.translated_rows == 1
assert en_stat.failed_rows == 1
assert en_stat.skipped_rows == 1
def test_multiple_languages(self):
db = MagicMock()
run_id = "run-3"
ru_stat = MagicMock(spec=TranslationRunLanguageStats)
en_stat = MagicMock(spec=TranslationRunLanguageStats)
records = [make_record(run_id), make_record(run_id)]
langs = [
make_lang("ru", "translated", "Привет"),
make_lang("en", "translated", "Hello"),
]
db.query.return_value.filter.return_value.all.side_effect = [
records,
langs,
]
update_language_stats(db, None, run_id, {"ru": ru_stat, "en": en_stat})
assert ru_stat.total_rows == 1
assert ru_stat.translated_rows == 1
assert en_stat.total_rows == 1
assert en_stat.translated_rows == 1
def test_empty_language_stats_map(self):
db = MagicMock()
run_id = "run-4"
records = [make_record(run_id)]
db.query.return_value.filter.return_value.all.side_effect = [
records,
[make_lang("ru", "translated")],
]
# Should not raise even with empty map
update_language_stats(db, None, run_id, {})
# No assertions needed — just verifying no exception
def test_no_records_returns_early(self):
db = MagicMock()
run_id = "run-5"
db.query.return_value.filter.return_value.all.return_value = []
update_language_stats(db, None, run_id, {"ru": MagicMock()})
# Should return early without querying languages
def test_approved_and_edited_count_as_translated(self):
db = MagicMock()
run_id = "run-6"
de_stat = MagicMock(spec=TranslationRunLanguageStats)
records = [make_record(run_id)]
langs = [
make_lang("de", "approved", "Hallo"),
make_lang("de", "edited", "Guten Tag"),
make_lang("de", "translated", "Tschüss"),
]
db.query.return_value.filter.return_value.all.side_effect = [
records,
langs,
]
update_language_stats(db, None, run_id, {"de": de_stat})
assert de_stat.translated_rows == 3
def test_token_and_cost_estimation(self):
db = MagicMock()
run_id = "run-7"
en_stat = MagicMock(spec=TranslationRunLanguageStats)
records = [make_record(run_id)]
langs = [make_lang("en", "translated", "Hello World")]
db.query.return_value.filter.return_value.all.side_effect = [
records,
langs,
]
update_language_stats(db, None, run_id, {"en": en_stat})
assert en_stat.token_count > 0
assert en_stat.estimated_cost >= 0
def test_empty_translated_value_no_crash(self):
db = MagicMock()
run_id = "run-8"
ru_stat = MagicMock(spec=TranslationRunLanguageStats)
records = [make_record(run_id)]
langs = [make_lang("ru", "translated", None)]
db.query.return_value.filter.return_value.all.side_effect = [
records,
langs,
]
update_language_stats(db, None, run_id, {"ru": ru_stat})
assert ru_stat.total_rows == 1
# #endregion TestLangStatsAggregation

189
backend/src/plugins/translate/__tests__/test_preview.py
View File

@@ -225,195 +225,6 @@ class TestTranslationPreview:
assert len(result["records"]) == 1
# endregion test_preview_with_dictionary
# region test_preview_row_approve [TYPE Function]
# @PURPOSE: Verify approving a preview row changes its status.
def test_preview_row_approve(self):
"""Approve action should set record status to APPROVED."""
db = MagicMock()
config_manager = MagicMock()
# Mock active session
session = MagicMock(spec=TranslationPreviewSession)
session.id = "session-1"
session.job_id = "job-123"
session.status = "ACTIVE"
session.created_at = datetime.now(UTC)
# Mock record
record = MagicMock(spec=TranslationPreviewRecord)
record.id = "record-1"
record.session_id = "session-1"
record.status = "PENDING"
record.target_sql = "test translation"
record.feedback = None
# Setup: db.query().filter().order_by().first() -> session (session lookup)
db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session
# Setup: db.query().filter().first() -> record (record lookup)
db.query.return_value.filter.return_value.first.return_value = record
preview_service = TranslationPreview(db, config_manager, "test-user")
result = preview_service.update_preview_row(
job_id="job-123",
row_id="record-1",
action="approve",
)
assert record.status == "APPROVED"
assert result["status"] == "APPROVED"
assert db.commit.called
# endregion test_preview_row_approve
# region test_preview_row_reject [TYPE Function]
# @PURPOSE: Verify rejecting a preview row changes its status.
def test_preview_row_reject(self):
"""Reject action should set record status to REJECTED."""
db = MagicMock()
config_manager = MagicMock()
session = MagicMock(spec=TranslationPreviewSession)
session.id = "session-1"
session.job_id = "job-123"
session.status = "ACTIVE"
record = MagicMock(spec=TranslationPreviewRecord)
record.id = "record-1"
record.session_id = "session-1"
record.status = "PENDING"
record.target_sql = "test"
record.feedback = None
db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session
db.query.return_value.filter.return_value.first.return_value = record
preview_service = TranslationPreview(db, config_manager, "test-user")
result = preview_service.update_preview_row(
job_id="job-123",
row_id="record-1",
action="reject",
)
assert record.status == "REJECTED"
assert result["status"] == "REJECTED"
# endregion test_preview_row_reject
# region test_preview_row_edit [TYPE Function]
# @PURPOSE: Verify editing a preview row updates its translation and status.
def test_preview_row_edit(self):
"""Edit action should update translation and set status to APPROVED."""
db = MagicMock()
config_manager = MagicMock()
session = MagicMock(spec=TranslationPreviewSession)
session.id = "session-1"
session.job_id = "job-123"
session.status = "ACTIVE"
record = MagicMock(spec=TranslationPreviewRecord)
record.id = "record-1"
record.session_id = "session-1"
record.status = "PENDING"
record.target_sql = "old translation"
record.feedback = None
db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session
db.query.return_value.filter.return_value.first.return_value = record
preview_service = TranslationPreview(db, config_manager, "test-user")
result = preview_service.update_preview_row(
job_id="job-123",
row_id="record-1",
action="edit",
translation="edited translation",
)
assert record.target_sql == "edited translation"
assert record.status == "APPROVED"
assert result["status"] == "APPROVED"
assert result["target_sql"] == "edited translation"
# endregion test_preview_row_edit
# region test_preview_row_invalid_action [TYPE Function]
# @PURPOSE: Verify invalid action raises ValueError.
def test_preview_row_invalid_action(self):
"""Invalid action should raise ValueError."""
db = MagicMock()
config_manager = MagicMock()
session = MagicMock(spec=TranslationPreviewSession)
session.id = "session-1"
session.job_id = "job-123"
session.status = "ACTIVE"
record = MagicMock(spec=TranslationPreviewRecord)
record.id = "record-1"
record.session_id = "session-1"
record.status = "PENDING"
db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session
db.query.return_value.filter.return_value.first.return_value = record
preview_service = TranslationPreview(db, config_manager, "test-user")
with pytest.raises(ValueError, match="Invalid action"):
preview_service.update_preview_row(
job_id="job-123",
row_id="record-1",
action="invalid_action",
)
# endregion test_preview_row_invalid_action
# region test_preview_accept_session [TYPE Function]
# @PURPOSE: Verify accepting a preview session gates execution.
def test_preview_accept_session(self):
"""Accept should set session status to APPLIED and return records."""
db = MagicMock()
config_manager = MagicMock()
session = MagicMock(spec=TranslationPreviewSession)
session.id = "session-1"
session.job_id = "job-123"
session.status = "ACTIVE"
session.created_by = "test-user"
session.created_at = datetime.now(UTC)
session.expires_at = datetime.now(UTC) + timedelta(hours=24)
record = MagicMock(spec=TranslationPreviewRecord)
record.id = "record-1"
record.session_id = "session-1"
record.source_sql = "Hello"
record.target_sql = "Привет"
record.status = "APPROVED"
record.feedback = None
# Session lookup
db.query.return_value.filter.return_value.order_by.return_value.first.return_value = session
# Records lookup
db.query.return_value.filter.return_value.all.return_value = [record]
preview_service = TranslationPreview(db, config_manager, "test-user")
result = preview_service.accept_preview_session(job_id="job-123")
assert session.status == "APPLIED"
assert result["status"] == "APPLIED"
assert len(result["records"]) == 1
assert result["records"][0]["status"] == "APPROVED"
assert db.commit.called
# endregion test_preview_accept_session
# region test_preview_no_active_session [TYPE Function]
# @PURPOSE: Verify accept raises error when no active session.
def test_preview_no_active_session(self):
"""Accept without active session should raise ValueError."""
db = MagicMock()
config_manager = MagicMock()
db.query.return_value.filter.return_value.order_by.return_value.first.return_value = None
preview_service = TranslationPreview(db, config_manager, "test-user")
with pytest.raises(ValueError, match="No active preview session"):
preview_service.accept_preview_session(job_id="job-123")
# endregion test_preview_no_active_session
# region test_cost_estimation [TYPE Function]
# @PURPOSE: Verify token and cost estimation methods.
def test_cost_estimation(self):

181
backend/src/plugins/translate/__tests__/test_response_field_coverage.py Normal file
View File

@@ -0,0 +1,181 @@
# #region ResponseFieldCoverageTests [C:3] [TYPE Module] [SEMANTICS test, translate, field-coverage, serialization]
# @BRIEF Tests that verify full field coverage across the API response chain.
# @LAYER Test
# @RELATION BINDS_TO -> [job_to_response]
# @RELATION BINDS_TO -> [TranslateJobResponse]
# @RELATION BINDS_TO -> [duplicate_job]
# @RELATION BINDS_TO -> [get_run_status]
# @RELATION BINDS_TO -> [get_run_history]
# @INVARIANT Every field in TranslateJobResponse must be mapped in job_to_response.
# @INVARIANT Every data field in TranslationJob must be copied in duplicate_job.
# @INVARIANT Run dict serializers must include insert_method + connection_snapshot.
# @TEST_CONTRACT job_to_response maps ALL TranslateJobResponse fields
# @TEST_CONTRACT duplicate_job copies ALL TranslationJob data fields
# @TEST_CONTRACT get_run_status/get_run_history include insert_method
# @RATIONALE Regression guard: when new columns are added to models/schemas,
# the response serializers and duplication logic must be updated too.
# These tests fail if a field is silently defaulted.
from unittest.mock import MagicMock
import pytest
from src.schemas.translate import TranslateJobResponse
from src.plugins.translate.service_utils import job_to_response
from src.plugins.translate.orchestrator_query import get_run_history
from src.models.translate import TranslationJob, TranslationRun
# #region TestJobToResponse [C:2] [TYPE Function] [SEMANTICS test, field-coverage, serialization]
# @TEST_INVARIANT: job_to_response-maps-all-response-fields -> VERIFIED_BY: test_job_to_response_covers_all_fields
class TestJobToResponse:
"""Verify job_to_response() maps EVERY field from TranslateJobResponse."""
# Fields set externally (not from job ORM)
EXTERNAL_FIELDS = {"dictionary_ids"}
def test_job_to_response_covers_all_fields(self):
"""Every field in TranslateJobResponse must be passed to the constructor."""
import inspect
from src.plugins.translate import service_utils
# Get source of job_to_response
src = inspect.getsource(service_utils.job_to_response)
response_fields = set(TranslateJobResponse.model_fields.keys())
needed = response_fields - self.EXTERNAL_FIELDS
missing = []
for field in sorted(needed):
# Check the field name appears in the function body as a kwarg
if f"{field}=" not in src:
missing.append(field)
assert not missing, (
f"Fields present in TranslateJobResponse but NOT mapped in job_to_response(): {missing}. "
f"Add them to service_utils.py job_to_response()."
)
# #endregion TestJobToResponse
# #region TestDuplicateJob [C:2] [TYPE Function] [SEMANTICS test, field-coverage, duplication]
# @TEST_INVARIANT: duplicate_job-copies-all-fields -> VERIFIED_BY: test_duplicate_job_copies_all_fields
class TestDuplicateJob:
"""Verify duplicate_job() copies all data-bearing TranslationJob columns."""
# Fields intentionally NOT copied
SKIP_FIELDS = {
"id", # new UUID
"status", # set to "DRAFT" explicitly
"created_by", # set to current user
"created_at", # auto-set by ORM
"updated_at", # auto-set by ORM
"name", # set to f"{source.name} (Copy)"
}
def test_duplicate_job_copies_all_fields(self):
"""Every data-bearing ORM column should be referenced in duplicate_job()."""
import inspect
from src.plugins.translate import service
src = inspect.getsource(service.TranslateJobService.duplicate_job)
orm_cols = {c.name for c in TranslationJob.__table__.columns}
data_cols = orm_cols - self.SKIP_FIELDS
missing = []
for col in sorted(data_cols):
# Check the column name is assigned in duplicate_job body
if f"{col}=" not in src and f"{col} =" not in src:
missing.append(col)
assert not missing, (
f"ORM columns present but NOT copied in duplicate_job(): {missing}. "
f"Add them to service.py duplicate_job()."
)
# #endregion TestDuplicateJob
# #region TestRunSerializers [C:2] [TYPE Function] [SEMANTICS test, field-coverage, run, serialization]
# @TEST_INVARIANT: get_run_status-includes-insert_method -> VERIFIED_BY: test_get_run_status_includes_insert_method
# @TEST_INVARIANT: get_run_history-includes-insert_method -> VERIFIED_BY: test_get_run_history_includes_insert_method
class TestRunSerializers:
"""Verify run dict serializers include insert_method and connection_snapshot."""
FIELDS = {"insert_method", "connection_snapshot"}
def test_get_run_status_includes_insert_method(self):
"""get_run_status() dict must include insert_method and connection_snapshot."""
import inspect
from src.plugins.translate import orchestrator_aggregator
src = inspect.getsource(orchestrator_aggregator.TranslationResultAggregator.get_run_status)
for field in self.FIELDS:
# The returned dict literal should contain the field
if f'"{field}"' not in src and f"'{field}'" not in src:
# Also check run. field access
pass
assert f'"{field}"' in src or f"'{field}'" in src or f"run.{field}" in src, (
f"Field '{field}' missing from get_run_status() dict. "
f"Add it to orchestrator_aggregator.py get_run_status()."
)
def test_get_run_history_includes_insert_method(self):
"""get_run_history() dict must include insert_method and connection_snapshot."""
import inspect
from src.plugins.translate import orchestrator_query
src = inspect.getsource(orchestrator_query.get_run_history)
for field in self.FIELDS:
assert f'"{field}"' in src or f"'{field}'" in src or f"r.{field}" in src, (
f"Field '{field}' missing from get_run_history() dict. "
f"Add it to orchestrator_query.py get_run_history()."
)
# #endregion TestRunSerializers
# #region TestRunModelColumns [C:1] [TYPE Function] [SEMANTICS test, field-coverage, orm]
# @TEST_EDGE: new column on TranslationRun without serialization -> test_catches
class TestRunModelColumns:
"""Verify that new columns on TranslationRun are not forgotten."""
KNOWN_RUN_SERIALIZERS = {
"orchestrator_query.get_run_history": [
"id", "job_id", "status", "trigger_type",
"started_at", "completed_at", "error_message",
"total_records", "successful_records", "failed_records", "skipped_records",
"cache_hits", "insert_status", "insert_method", "connection_snapshot",
"superset_execution_id", "created_by", "created_at",
],
"orchestrator_aggregator.get_run_status": [
"id", "job_id", "status", "trigger_type",
"started_at", "completed_at", "error_message",
"total_records", "successful_records", "failed_records", "skipped_records",
"cache_hits", "insert_status", "insert_method", "connection_snapshot",
"config_snapshot",
"superset_execution_id", "batch_count", "language_stats",
"event_invariants", "created_by", "created_at",
],
}
# ORM columns that are NOT expected in any run serializer
SKIP_RUN_COLUMNS = {
"key_hash", "config_hash", "dict_snapshot_hash",
"id", "job_id",
"records_inserted_at", "insert_batches_json",
"superset_execution_log",
}
def test_all_run_columns_serialized(self):
"""Every TranslationRun column should appear in at least one serializer."""
orm_cols = {c.name for c in TranslationRun.__table__.columns}
expected_columns = orm_cols - self.SKIP_RUN_COLUMNS
all_serialized = set()
for fields in self.KNOWN_RUN_SERIALIZERS.values():
all_serialized.update(fields)
missing = expected_columns - all_serialized
assert not missing, (
f"TranslationRun columns not serialized in any run dict: {missing}. "
f"Add them to get_run_status() and/or get_run_history()."
)
# #endregion TestRunModelColumns
# #endregion ResponseFieldCoverageTests

132
backend/src/plugins/translate/__tests__/test_retry.py Normal file
View File

@@ -0,0 +1,132 @@
# #region TestRunRetry [C:3] [TYPE Module] [SEMANTICS test,translate,retry,integration]
# @BRIEF Tests for TranslationRunRetryManager — retry_failed_batches, cancel_run.
# @RELATION BINDS_TO -> [TranslationRunRetryManager]
import uuid
from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch
import pytest
from src.models.translate import TranslationBatch, TranslationJob, TranslationRun, TranslationRecord
from src.plugins.translate.orchestrator_retry import TranslationRunRetryManager
from src.plugins.translate.executor import TranslationExecutor
# #region TestRetryFailedBatches [C:3] [TYPE Class]
class TestRetryFailedBatches:
"""Verify retry_failed_batches handles runs, batches, records correctly."""
def _make_manager(self, db=None):
if db is None:
db = MagicMock()
config = MagicMock()
event_log = MagicMock()
return TranslationRunRetryManager(db, config, event_log, "test_user")
def _make_run(self, status="FAILED") -> TranslationRun:
run = MagicMock(spec=TranslationRun)
run.id = str(uuid.uuid4())
run.job_id = str(uuid.uuid4())
run.status = status
run.successful_records = 0
run.failed_records = 10
run.skipped_records = 0
return run
def _make_job(self) -> TranslationJob:
job = MagicMock(spec=TranslationJob)
job.id = str(uuid.uuid4())
return job
def _make_batch(self, run_id: str, status="FAILED", idx=0) -> TranslationBatch:
batch = MagicMock(spec=TranslationBatch)
batch.id = str(uuid.uuid4())
batch.run_id = run_id
batch.batch_index = idx
batch.status = status
return batch
async def test_retry_run_not_found(self):
db = MagicMock()
db.query.return_value.filter.return_value.first.return_value = None
mgr = self._make_manager(db)
with pytest.raises(ValueError, match="not found"):
await mgr.retry_failed_batches("non-existent")
async def test_retry_job_not_found(self):
db = MagicMock()
run = self._make_run()
def query_side_effect(model):
q = MagicMock()
if model == TranslationRun:
q.filter.return_value.first.return_value = run
elif model == TranslationJob:
q.filter.return_value.first.return_value = None
else:
q.filter.return_value.filter.return_value.all.return_value = []
return q
db.query.side_effect = query_side_effect
mgr = self._make_manager(db)
with pytest.raises(ValueError, match="not found"):
await mgr.retry_failed_batches(run.id)
async def test_retry_no_failed_batches(self):
db = MagicMock()
run = self._make_run()
job = self._make_job()
def query_side_effect(model):
q = MagicMock()
q.filter.return_value.filter.return_value.all.return_value = []
q.filter.return_value.all.return_value = []
if model == TranslationRun:
q.filter.return_value.first.return_value = run
elif model == TranslationJob:
q.filter.return_value.first.return_value = job
return q
db.query.side_effect = query_side_effect
mgr = self._make_manager(db)
with pytest.raises(ValueError, match="No failed batches"):
await mgr.retry_failed_batches(run.id)
async def test_retry_job_not_found(self):
db = MagicMock()
run = self._make_run()
def query_side_effect(model):
q = MagicMock()
if model == TranslationRun:
q.filter.return_value.first.return_value = run
else:
q.filter.return_value.first.return_value = None
return q
db.query.side_effect = query_side_effect
mgr = self._make_manager(db)
with pytest.raises(ValueError, match="not found"):
await mgr.retry_failed_batches(run.id)
async def test_cancel_run(self):
db = MagicMock()
mgr = self._make_manager(db)
run = self._make_run("RUNNING")
with patch('src.plugins.translate.orchestrator_retry._cancel_run') as mock_cancel:
mock_cancel.return_value = run
result = mgr.cancel_run(run.id)
assert result.status == "RUNNING"
mock_cancel.assert_called_once()
async def test_retry_insert(self):
db = MagicMock()
mgr = self._make_manager(db)
run = self._make_run("COMPLETED")
with patch('src.plugins.translate.orchestrator_retry._retry_insert', new_callable=AsyncMock) as mock_retry_insert:
mock_retry_insert.return_value = run
result = await mgr.retry_insert(run.id)
assert result.status == "COMPLETED"
mock_retry_insert.assert_called_once()
# #endregion TestRetryFailedBatches

295
backend/src/plugins/translate/__tests__/test_run_service.py Normal file
View File

@@ -0,0 +1,295 @@
# #region TestRunExecutionService [C:3] [TYPE Module] [SEMANTICS test,translate,run,orchestration]
# @BRIEF Tests for RunExecutionService — filter_new_keys, load_preview_edits, key_hash, lang_stats.
# @RELATION BINDS_TO -> [RunExecutionService]
import uuid
from unittest.mock import MagicMock, patch
import pytest
from src.models.translate import (
TranslationJob,
TranslationRun,
TranslationRecord,
TranslationRunLanguageStats,
TranslationPreviewSession,
TranslationPreviewRecord,
TranslationPreviewLanguage,
TranslationLanguage,
)
from src.plugins.translate._run_service import RunExecutionService
# #region make_service [C:1] [TYPE Function]
def make_service(db=None) -> RunExecutionService:
if db is None:
db = MagicMock()
return RunExecutionService(db, MagicMock(), "test_user")
def make_job(**overrides) -> TranslationJob:
job = MagicMock(spec=TranslationJob)
job.id = "job-1"
job.target_key_cols = ["id"]
job.source_key_cols = []
for k, v in overrides.items():
setattr(job, k, v)
return job
def make_run(**overrides) -> TranslationRun:
run = MagicMock(spec=TranslationRun)
run.id = str(uuid.uuid4())
run.job_id = "job-1"
run.status = "RUNNING"
for k, v in overrides.items():
setattr(run, k, v)
return run
def make_record(**overrides) -> TranslationRecord:
rec = MagicMock(spec=TranslationRecord)
rec.id = str(uuid.uuid4())
rec.run_id = "run-1"
rec.status = "SUCCESS"
rec.source_data = {"id": 1, "name": "Alice"}
rec.source_sql = "SELECT 1"
rec.target_sql = "translated"
for k, v in overrides.items():
setattr(rec, k, v)
return rec
# #endregion make_service
# #region TestFilterNewKeys [C:3] [TYPE Class]
class TestFilterNewKeys:
"""Verify _filter_new_keys correctly deduplicates by key columns."""
def test_no_prev_run_returns_all(self):
db = MagicMock()
db.query.return_value.filter.return_value.filter.return_value.order_by.return_value.first.return_value = None
svc = make_service(db)
job = make_job(target_key_cols=["id"])
rows = [{"source_data": {"id": 1}}, {"source_data": {"id": 2}}]
result = svc._filter_new_keys(job, "run-1", rows)
assert len(result) == 2
def test_filters_duplicates(self):
db = MagicMock()
prev_run = make_run(status="COMPLETED", id="prev-run-1")
def make_run_query():
q = MagicMock()
# Code calls: .filter(a, b, c).order_by(...).first()
q.filter.return_value.order_by.return_value.first.return_value = prev_run
return q
def make_rec_query():
q = MagicMock()
rec = make_record(run_id="prev-run-1", source_data={"id": 1})
# Code calls: .filter(a, b).all()
q.filter.return_value.all.return_value = [rec]
return q
db.query.side_effect = lambda model: make_rec_query() if model == TranslationRecord else make_run_query()
svc = make_service(db)
job = make_job(target_key_cols=["id"])
rows = [
{"source_data": {"id": 1}},
{"source_data": {"id": 2}},
{"source_data": {"id": 3}},
]
result = svc._filter_new_keys(job, "run-1", rows)
assert len(result) == 2
assert result[0]["source_data"]["id"] == 2
def test_all_duplicates_returns_empty(self):
db = MagicMock()
prev_run = make_run(status="COMPLETED", id="prev-run-2")
def make_run_query():
q = MagicMock()
q.filter.return_value.order_by.return_value.first.return_value = prev_run
return q
def make_rec_query():
q = MagicMock()
rec1 = make_record(run_id="prev-run-2", source_data={"id": 1})
rec2 = make_record(run_id="prev-run-2", source_data={"id": 2})
q.filter.return_value.all.return_value = [rec1, rec2]
return q
db.query.side_effect = lambda model: make_rec_query() if model == TranslationRecord else make_run_query()
svc = make_service(db)
job = make_job(target_key_cols=["id"])
rows = [
{"source_data": {"id": 1}},
{"source_data": {"id": 2}},
]
result = svc._filter_new_keys(job, "run-1", rows)
assert len(result) == 0
def test_no_key_cols_returns_all(self):
db = MagicMock()
svc = make_service(db)
job = make_job(target_key_cols=[], source_key_cols=[])
rows = [{"source_data": {"id": 1}}]
result = svc._filter_new_keys(job, "run-1", rows)
assert len(result) == 1
def test_missing_source_data_handles_gracefully(self):
db = MagicMock()
db.query.return_value.filter.return_value.filter.return_value.order_by.return_value.first.return_value = None
svc = make_service(db)
job = make_job(target_key_cols=["id"])
rows = [{"source_data": None}, {}]
result = svc._filter_new_keys(job, "run-1", rows)
assert len(result) == 2
# #endregion TestFilterNewKeys
# #region TestLoadPreviewEdits [C:3] [TYPE Class]
class TestLoadPreviewEdits:
"""Verify _load_preview_edits loads approved/edited preview translations."""
def test_no_session_returns_empty_cache(self):
db = MagicMock()
db.query.return_value.filter.return_value.filter.return_value.order_by.return_value.first.return_value = None
svc = make_service(db)
svc._load_preview_edits("job-1")
assert svc._preview_edits_cache == {}
def test_loads_edited_languages(self):
db = MagicMock()
def query_side_effect(model):
q = MagicMock()
if model == TranslationPreviewSession:
session = MagicMock(spec=TranslationPreviewSession)
session.id = "preview-1"
# Code calls: .filter(a, b).order_by(...).first()
q.filter.return_value.order_by.return_value.first.return_value = session
elif model == TranslationPreviewRecord:
lang_entry = MagicMock()
lang_entry.language_code = "ru"
lang_entry.status = "edited"
lang_entry.user_edit = "исправленный"
lang_entry.final_value = "исправленный"
record = MagicMock()
record.source_data = {"id": 1}
record.languages = [lang_entry]
# Code calls: .filter(a).all()
q.filter.return_value.all.return_value = [record]
return q
db.query.side_effect = query_side_effect
svc = make_service(db)
svc._load_preview_edits("job-1")
assert svc._preview_edits_cache is not None
assert len(svc._preview_edits_cache) == 1
def test_approved_language_loaded(self):
db = MagicMock()
def query_side_effect(model):
q = MagicMock()
if model == TranslationPreviewSession:
session = MagicMock(spec=TranslationPreviewSession)
session.id = "preview-1"
q.filter.return_value.order_by.return_value.first.return_value = session
elif model == TranslationPreviewRecord:
lang_entry = MagicMock()
lang_entry.language_code = "de"
lang_entry.status = "approved"
lang_entry.user_edit = "Guten Tag"
lang_entry.final_value = "Guten Tag"
record = MagicMock()
record.source_data = {"id": 2}
record.languages = [lang_entry]
q.filter.return_value.all.return_value = [record]
return q
db.query.side_effect = query_side_effect
svc = make_service(db)
svc._load_preview_edits("job-1")
assert len(svc._preview_edits_cache) == 1
# #endregion TestLoadPreviewEdits
# #region TestKeyHash [C:3] [TYPE Class]
class TestKeyHash:
"""Verify _compute_key_hash produces consistent hashes."""
def test_hash_consistent(self):
svc = make_service()
h1 = svc._compute_key_hash({"id": 1, "name": "Alice"})
h2 = svc._compute_key_hash({"name": "Alice", "id": 1})
assert h1 == h2 # same data, sorted keys
def test_hash_different_data(self):
svc = make_service()
h1 = svc._compute_key_hash({"id": 1})
h2 = svc._compute_key_hash({"id": 2})
assert h1 != h2
def test_hash_empty_dict(self):
svc = make_service()
h = svc._compute_key_hash({})
assert isinstance(h, str)
assert len(h) == 16
# #endregion TestKeyHash
# #region TestUpdateLangStatsIncremental [C:3] [TYPE Class]
class TestUpdateLangStatsIncremental:
"""Verify _update_language_stats_incremental aggregates correctly."""
def test_updates_stats(self):
db = MagicMock()
record = make_record()
def query_side_effect(model):
q = MagicMock()
if model == TranslationRecord:
# Code calls: .filter(run_id == ...).all()
q.filter.return_value.all.return_value = [record]
elif model == TranslationLanguage:
lang = MagicMock()
lang.language_code = "ru"
lang.status = "translated"
lang.translated_value = "Привет"
# Code calls: .filter(record_id.in_(...)).all()
q.filter.return_value.all.return_value = [lang]
return q
db.query.side_effect = query_side_effect
svc = make_service(db)
ru_stat = MagicMock(spec=TranslationRunLanguageStats)
svc._update_language_stats_incremental("run-1", {"ru": ru_stat})
assert ru_stat.total_rows == 1
assert ru_stat.translated_rows == 1
def test_no_records_returns_early(self):
db = MagicMock()
db.query.return_value.filter.return_value.all.return_value = []
svc = make_service(db)
svc._update_language_stats_incremental("run-1", {"ru": MagicMock()})
# Should not query language entries when there are no records
# #endregion TestUpdateLangStatsIncremental
# #endregion TestRunExecutionService

221
backend/src/plugins/translate/__tests__/test_sql_insert_service.py Normal file
View File

@@ -0,0 +1,221 @@
# #region TestSQLInsertService [C:3] [TYPE Module] [SEMANTICS test,translate,sql,insert]
# @BRIEF Tests for SQLInsertService — SQL generation, Superset/direct DB execution.
# @RELATION BINDS_TO -> [SQLInsertService]
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
from src.models.translate import TranslationJob, TranslationRun, TranslationRecord
from src.plugins.translate.orchestrator_sql import SQLInsertService
# #region TestSQLInsertService [C:3] [TYPE Class]
@pytest.mark.asyncio
class TestSQLInsertService:
"""Verify SQLInsertService handles generation and execution."""
def _make_service(self, db=None):
if db is None:
db = MagicMock()
config = MagicMock()
event_log = MagicMock()
return SQLInsertService(db, config, event_log)
def _make_job(self, **kwargs) -> TranslationJob:
job = MagicMock(spec=TranslationJob)
job.id = "job-1"
job.target_column = "translated_text"
job.translation_column = "source_text"
job.target_languages = ["ru"]
job.target_schema = "public"
job.target_table = "translations"
job.target_key_cols = []
job.upsert_strategy = "INSERT"
job.insert_method = "sqllab"
job.connection_id = None
job.environment_id = "env-1"
job.source_dialect = "postgresql"
job.database_dialect = None
job.target_dialect = None
for k, v in kwargs.items():
setattr(job, k, v)
return job
def _make_run(self) -> TranslationRun:
run = MagicMock(spec=TranslationRun)
run.id = "run-1"
return run
async def test_no_records_returns_skipped(self):
db = MagicMock()
db.query.return_value.options.return_value.filter.return_value.all.return_value = []
service = self._make_service(db)
job = self._make_job()
run = self._make_run()
result = await service.generate_and_insert_sql(job, run)
assert result["status"] == "skipped"
assert result["reason"] == "no_records"
async def test_sql_generation_failure_returns_failed(self):
db = MagicMock()
record = MagicMock(spec=TranslationRecord)
record.target_sql = "test"
db.query.return_value.options.return_value.filter.return_value.all.return_value = [record]
service = self._make_service(db)
job = self._make_job()
run = self._make_run()
with patch('src.plugins.translate.orchestrator_sql.SQLGenerator.generate',
side_effect=ValueError("Invalid SQL")):
with patch('src.plugins.translate.orchestrator_sql.build_columns', return_value=["col1"]):
with patch('src.plugins.translate.orchestrator_sql.build_rows', return_value=[{"col1": "val1"}]):
with patch('src.plugins.translate.orchestrator_sql.build_context_keys', return_value=[]):
result = await service.generate_and_insert_sql(job, run)
assert result["status"] == "failed"
assert "Invalid SQL" in result["error_message"]
async def test_direct_db_connection_not_found(self):
db = MagicMock()
record = MagicMock(spec=TranslationRecord)
record.target_sql = "test"
db.query.return_value.options.return_value.filter.return_value.all.return_value = [record]
service = self._make_service(db)
job = self._make_job(insert_method="direct_db", connection_id="conn-1")
run = self._make_run()
with patch('src.plugins.translate.orchestrator_sql.SQLGenerator.generate',
return_value=("INSERT INTO ...", 1)):
with patch('src.plugins.translate.orchestrator_sql.build_columns', return_value=["col1"]):
with patch('src.plugins.translate.orchestrator_sql.build_rows', return_value=[{"col1": "val1"}]):
with patch('src.plugins.translate.orchestrator_sql.build_context_keys', return_value=[]):
with patch('src.plugins.translate.orchestrator_sql.ConnectionService') as mock_conn:
mock_conn.return_value.get_connection.return_value = None
result = await service.generate_and_insert_sql(job, run)
assert result["status"] == "failed"
assert "not found" in result["error_message"]
async def test_resolve_dialect_fallback_to_default(self):
service = self._make_service()
job = self._make_job(database_dialect=None, target_dialect=None)
with patch('src.plugins.translate.orchestrator_sql.SupersetSqlLabExecutor') as mock_exec:
mock_exec.return_value.resolve_database_id = AsyncMock()
mock_exec.return_value.get_database_backend = MagicMock(return_value=None)
dialect = await service._resolve_dialect(job)
assert dialect == "postgresql"
async def test_superset_sqllab_exception_returns_failed(self):
db = MagicMock()
service = self._make_service(db)
job = self._make_job()
run = self._make_run()
with patch('src.plugins.translate.orchestrator_sql.SupersetSqlLabExecutor') as mock_exec:
instance = mock_exec.return_value
instance.resolve_database_id = AsyncMock()
instance.execute_and_poll = AsyncMock(side_effect=Exception("Superset API error"))
result = await service._execute_superset_sqllab(job, run, "INSERT INTO ...")
assert result["status"] == "failed"
assert "Superset API error" in result["error_message"]
async def test_direct_db_success(self):
db = MagicMock()
service = self._make_service(db)
job = self._make_job(insert_method="direct_db", connection_id="conn-1")
run = self._make_run()
with patch('src.plugins.translate.orchestrator_sql.ConnectionService') as mock_conn_cls:
mock_conn = MagicMock()
mock_conn.name = "ch_test"
mock_conn.dialect = "clickhouse"
mock_conn.host = "localhost"
mock_conn.port = 8123
mock_conn.database = "test_db"
mock_conn_cls.return_value.get_connection.return_value = mock_conn
with patch('src.plugins.translate.orchestrator_sql.DbExecutor') as mock_exec_cls:
mock_result = MagicMock()
mock_result.success = True
mock_result.rows_affected = 10
mock_result.execution_time_ms = 50
mock_exec_cls.return_value.execute_sql.return_value = mock_result
result = await service._execute_direct_db(job, run, "INSERT INTO ...")
assert result["status"] == "success"
assert result["rows_affected"] == 10
assert result["connection_name"] == "ch_test"
assert run.connection_snapshot is not None
assert run.insert_method == "direct_db"
assert db.commit.called
async def test_direct_db_execution_failure(self):
db = MagicMock()
service = self._make_service(db)
job = self._make_job(insert_method="direct_db", connection_id="conn-1")
run = self._make_run()
with patch('src.plugins.translate.orchestrator_sql.ConnectionService') as mock_conn_cls:
mock_conn = MagicMock()
mock_conn.name = "ch_test"
mock_conn.dialect = "clickhouse"
mock_conn.host = "localhost"
mock_conn.port = 8123
mock_conn.database = "test_db"
mock_conn_cls.return_value.get_connection.return_value = mock_conn
with patch('src.plugins.translate.orchestrator_sql.DbExecutor') as mock_exec_cls:
mock_result = MagicMock()
mock_result.success = False
mock_result.error = "DB timeout"
mock_result.execution_time_ms = 5000
mock_exec_cls.return_value.execute_sql.return_value = mock_result
result = await service._execute_direct_db(job, run, "INSERT INTO ...")
assert result["status"] == "failed"
assert "DB timeout" in result["error_message"]
assert result["execution_time_ms"] == 5000
async def test_resolve_dialect_direct_db_connection(self):
service = self._make_service()
job = self._make_job(insert_method="direct_db", connection_id="conn-1")
with patch('src.plugins.translate.orchestrator_sql.ConnectionService') as mock_conn_cls:
mock_conn = MagicMock()
mock_conn.dialect = "clickhouse"
mock_conn_cls.return_value.get_connection.return_value = mock_conn
dialect = await service._resolve_dialect(job)
assert dialect == "clickhouse"
async def test_empty_columns_fallback(self):
db = MagicMock()
record = MagicMock()
record.target_sql = "test"
db.query.return_value.options.return_value.filter.return_value.all.return_value = [record]
service = self._make_service(db)
job = self._make_job(target_column=None, translation_column="source_text")
run = self._make_run()
with patch('src.plugins.translate.orchestrator_sql.SQLGenerator.generate',
return_value=("INSERT INTO ... VALUES ('test')", 1)) as mock_gen:
with patch('src.plugins.translate.orchestrator_sql.build_columns', return_value=[]):
with patch('src.plugins.translate.orchestrator_sql.build_rows', return_value=[]):
with patch('src.plugins.translate.orchestrator_sql.build_context_keys', return_value=[]):
with patch.object(service, '_resolve_dialect', AsyncMock(return_value="postgresql")):
with patch.object(service, '_execute_superset_sqllab', AsyncMock(return_value={"status": "success"})):
result = await service.generate_and_insert_sql(job, run)
assert result["status"] == "success"
mock_gen.assert_called_once()
# #endregion TestSQLInsertService