busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
f22b0cf41c2cdbbdbd0f92ba57bc90e542a974f9
ss-tools/backend/tests/test_translate_scheduler_execution.py
busya de023c7a31 fix(backend+frontend): migration deadlock, async pattern, timeout safety, fire-and-forget translations 
Backend:
- MigrationPlugin.execute — remove AsyncJobRunner.run() deadlock on post-migration
  ID sync (2 deadlocks total: plugins/migration.py + api/routes/migration.py
  trigger_sync_now). Replace blocking runner.run with direct await.
- MigrationPlugin.execute — fix temp file leak (dry_run=True prevented cleanup).
- MigrationPlugin.execute — IdMappingService(SessionLocal()) now closed in finally.
- MigrationPlugin.execute — wire IdMappingService into MigrationEngine constructor
  so cross-filter patching actually works instead of silently skipping.
- MigrationPlugin.execute — SupersetClient.aclose() in finally to prevent
  httpx connection pool leak.
- TaskManager — _async_tasks dict leak (add_done_callback cleanup).
- JobLifecycle — CancelledError handler (tasks stuck in RUNNING after cancel).
- JobLifecycle — persist_task BEFORE _broadcast_task_status (crash consistency).
- JobLifecycle — wait_for_resolution/wait_for_input now have 3600s timeout.
- AsyncJobRunner.run — 300s timeout on future.result() (APScheduler thread safety).

Translate scheduler:
- execute_scheduled_translation — replace blocking runner.run(orch.execute_run(run))
  with fire-and-forget TranslationOrchestrator.execute_background(). APScheduler
  thread is freed in ms instead of blocking for the full translation duration.
  Translation runs of 10k+ rows (200+ LLM batches, hours) no longer hit the
  300s runner timeout.
- TranslationOrchestrator.execute_background — new static method: opens own DB
  session, dispatches asyncio.create_task, handles errors + notification.
- Scheduler last_run_at updated at dispatch time (not after completion).

Frontend:
- MigrationModel.stepReady[3] now requires dryRunResult != null (was always
  true, allowing UI to reach step 3 without dry-run).
- WizardModel.goToStep gate for step 3 uses stepReady[3].
- +page.svelte  — dryRunResult no longer self-clears via reactive loop.
- Progress bar step 3 indicator gate fixed for new readiness logic.

Tests:
- 2 new regression tests for migration sync (deadlock-free, completes cleanly).
- test_migration_plugin.py — _make_mock_superset_client/_make_mock_mapping_service
  helpers for proper async mock behavior (aclose, sync_environment AsyncMock).
- 10 scheduled-translation tests updated for fire-and-forget pattern.
- MigrationModel.test.ts — step 3 invariant + goToStep block test.
- All affected tests: 104 backend + 79 frontend = 183 passed.
2026-07-07 21:04:23 +03:00

518 lines
18 KiB
Python
Raw Blame History

# #region TestTranslateSchedulerExecution [C:3] [TYPE Module] [SEMANTICS test, translate, scheduler, execution, trigger-type, baseline]
# @BRIEF Verify execute_scheduled_translation contracts — trigger_type dispatch, baseline expiry fallback.
# @RELATION BINDS_TO -> [execute_scheduled_translation]
# @TEST_EDGE: new_key_only_mode — trigger_type="new_key_only" when baseline not expired
# @TEST_EDGE: baseline_expired_fallback — trigger_type="scheduled" when baseline > 90 days old
# @TEST_EDGE: full_mode_default — trigger_type="scheduled" for execution_mode="full"
# @TEST_EDGE: inactive_schedule — function returns early without executing
# @TEST_EDGE: execution_error_handled — run marked FAILED, schedule tracking updated
# @TEST_INVARIANT: trigger_type_dispatch -> VERIFIED_BY: [test_new_key_only_mode, test_baseline_expired_fallback, test_full_mode_default]
from pathlib import Path
import sys
import asyncio
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
from datetime import UTC, datetime, timedelta
from unittest.mock import MagicMock, patch
def _make_coro_runner():
"""Mock runner whose run() executes the coroutine synchronously via asyncio.run."""
r = MagicMock()
r.run.side_effect = lambda coro: asyncio.run(coro)
return r
# -- Helpers -----------------------------------------------------------------
def _make_most_recent_run(created_at_delta_days=60):
"""Build a MagicMock TranslationRun with given age."""
run = MagicMock()
run.id = "prev-run-1"
run.job_id = "job-1"
run.status = "COMPLETED"
run.insert_status = "succeeded"
run.created_at = datetime.now(UTC) - timedelta(days=created_at_delta_days)
return run
def _make_mock_run(run_id="run-1", job_id="job-1"):
"""Build a mock TranslationRun returned by start_run."""
run = MagicMock()
run.id = run_id
run.job_id = job_id
run.status = "PENDING"
run.trigger_type = None
run.total_records = 0
run.successful_records = 0
run.failed_records = 0
run.skipped_records = 0
run.insert_status = None
run.superset_execution_id = None
run.superset_execution_log = None
run.error_message = None
return run
# -- Tests: trigger_type dispatch and execution modes -----------------------
# #region test_new_key_only_mode [C:1] [TYPE Function]
def test_new_key_only_mode():
"""execution_mode="new_key_only" with 60-day-old prev run → trigger_type="new_key_only"."""
from src.plugins.translate.scheduler import execute_scheduled_translation
mock_db = MagicMock()
mock_session_maker = MagicMock(return_value=mock_db)
mock_config = MagicMock()
schedule = MagicMock()
schedule.id = "sched-1"
schedule.job_id = "job-1"
schedule.is_active = True
schedule.execution_mode = "new_key_only"
most_recent = _make_most_recent_run(created_at_delta_days=60) # 60 days < 90
q1 = MagicMock()
q1.filter.return_value.first.return_value = schedule
q2 = MagicMock()
q2.filter.return_value.order_by.return_value.first.return_value = None # no concurrent
q3 = MagicMock()
q3.filter.return_value.order_by.return_value.first.return_value = most_recent
mock_db.query.side_effect = [q1, q2, q3]
mock_run = _make_mock_run()
with patch(
"src.plugins.translate.orchestrator.TranslationOrchestrator"
) as mock_orch_cls, \
patch("src.dependencies.get_async_job_runner", return_value=_make_coro_runner()):
mock_orch = MagicMock()
mock_orch.start_run.return_value = mock_run
mock_orch_cls.return_value = mock_orch
execute_scheduled_translation(
schedule_id="sched-1",
job_id="job-1",
db_session_maker=mock_session_maker,
config_manager=mock_config,
execution_mode="new_key_only",
)
assert mock_run.trigger_type == "new_key_only", (
f"Expected trigger_type='new_key_only', got '{mock_run.trigger_type}'"
)
mock_orch.start_run.assert_called_once_with(job_id="job-1", is_scheduled=True)
# execute_background is called (fire-and-forget) instead of execute_run via runner.run
mock_orch_cls.execute_background.assert_called()
mock_db.close.assert_called_once()
# #endregion test_new_key_only_mode
# #region test_baseline_expired_fallback [C:1] [TYPE Function]
def test_baseline_expired_fallback():
"""execution_mode="new_key_only" with >90-day-old prev run → trigger_type="scheduled" baseline fallback."""
from src.plugins.translate.scheduler import execute_scheduled_translation
mock_db = MagicMock()
mock_session_maker = MagicMock(return_value=mock_db)
mock_config = MagicMock()
schedule = MagicMock()
schedule.id = "sched-1"
schedule.job_id = "job-1"
schedule.is_active = True
schedule.execution_mode = "new_key_only"
most_recent = _make_most_recent_run(created_at_delta_days=95) # 95 days > 90
q1 = MagicMock()
q1.filter.return_value.first.return_value = schedule
q2 = MagicMock()
q2.filter.return_value.order_by.return_value.first.return_value = None
q3 = MagicMock()
q3.filter.return_value.order_by.return_value.first.return_value = most_recent
mock_db.query.side_effect = [q1, q2, q3]
mock_run = _make_mock_run()
with patch(
"src.plugins.translate.orchestrator.TranslationOrchestrator"
) as mock_orch_cls, \
patch("src.dependencies.get_async_job_runner", return_value=_make_coro_runner()):
mock_orch = MagicMock()
mock_orch.start_run.return_value = mock_run
mock_orch_cls.return_value = mock_orch
with patch(
"src.plugins.translate.scheduler.TranslationEventLog"
) as mock_event_log_cls:
mock_event_log = MagicMock()
mock_event_log_cls.return_value = mock_event_log
execute_scheduled_translation(
schedule_id="sched-1",
job_id="job-1",
db_session_maker=mock_session_maker,
config_manager=mock_config,
execution_mode="new_key_only",
)
assert mock_run.trigger_type == "scheduled", (
f"Expected trigger_type='scheduled' (baseline expired), got '{mock_run.trigger_type}'"
)
mock_event_log.log_event.assert_called_once()
event_call = mock_event_log.log_event.call_args
assert event_call.kwargs["job_id"] == "job-1"
assert event_call.kwargs["event_type"] == "RUN_STARTED"
assert event_call.kwargs["payload"]["reason"] == "baseline_expired"
assert event_call.kwargs["payload"]["age_days"] == 95
mock_db.close.assert_called_once()
# #endregion test_baseline_expired_fallback
# #region test_full_mode_default [C:1] [TYPE Function]
def test_full_mode_default():
"""execution_mode="full" → trigger_type="scheduled" regardless of baseline age."""
from src.plugins.translate.scheduler import execute_scheduled_translation
mock_db = MagicMock()
mock_session_maker = MagicMock(return_value=mock_db)
mock_config = MagicMock()
schedule = MagicMock()
schedule.id = "sched-1"
schedule.job_id = "job-1"
schedule.is_active = True
schedule.execution_mode = "full"
most_recent = _make_most_recent_run(created_at_delta_days=30)
q1 = MagicMock()
q1.filter.return_value.first.return_value = schedule
q2 = MagicMock()
q2.filter.return_value.order_by.return_value.first.return_value = None
q3 = MagicMock()
q3.filter.return_value.order_by.return_value.first.return_value = most_recent
mock_db.query.side_effect = [q1, q2, q3]
mock_run = _make_mock_run()
with patch(
"src.plugins.translate.orchestrator.TranslationOrchestrator"
) as mock_orch_cls, \
patch("src.dependencies.get_async_job_runner", return_value=_make_coro_runner()):
mock_orch = MagicMock()
mock_orch.start_run.return_value = mock_run
mock_orch_cls.return_value = mock_orch
execute_scheduled_translation(
schedule_id="sched-1",
job_id="job-1",
db_session_maker=mock_session_maker,
config_manager=mock_config,
execution_mode="full",
)
assert mock_run.trigger_type == "scheduled", (
f"Expected trigger_type='scheduled' for full mode, got '{mock_run.trigger_type}'"
)
mock_orch_cls.execute_background.assert_called()
mock_db.close.assert_called_once()
# #endregion test_full_mode_default
# #region test_inactive_schedule_skips [C:1] [TYPE Function]
def test_inactive_schedule_skips():
"""Inactive schedule → function returns early without executing anything."""
from src.plugins.translate.scheduler import execute_scheduled_translation
mock_db = MagicMock()
mock_session_maker = MagicMock(return_value=mock_db)
mock_config = MagicMock()
schedule = MagicMock()
schedule.id = "sched-1"
schedule.job_id = "job-1"
schedule.is_active = False
q1 = MagicMock()
q1.filter.return_value.first.return_value = schedule
mock_db.query.return_value = q1
with patch(
"src.plugins.translate.orchestrator.TranslationOrchestrator"
) as mock_orch_cls, \
patch("src.dependencies.get_async_job_runner", return_value=_make_coro_runner()):
execute_scheduled_translation(
schedule_id="sched-1",
job_id="job-1",
db_session_maker=mock_session_maker,
config_manager=mock_config,
execution_mode="full",
)
mock_orch_cls.assert_not_called()
mock_db.close.assert_called_once()
# #endregion test_inactive_schedule_skips
# #region test_schedule_not_found_skips [C:1] [TYPE Function]
def test_schedule_not_found_skips():
"""Schedule not found → function returns early."""
from src.plugins.translate.scheduler import execute_scheduled_translation
mock_db = MagicMock()
mock_session_maker = MagicMock(return_value=mock_db)
mock_config = MagicMock()
q1 = MagicMock()
q1.filter.return_value.first.return_value = None
mock_db.query.return_value = q1
with patch(
"src.plugins.translate.orchestrator.TranslationOrchestrator"
) as mock_orch_cls, \
patch("src.dependencies.get_async_job_runner", return_value=_make_coro_runner()):
execute_scheduled_translation(
schedule_id="sched-1",
job_id="job-1",
db_session_maker=mock_session_maker,
config_manager=mock_config,
execution_mode="full",
)
mock_orch_cls.assert_not_called()
mock_db.close.assert_called_once()
# #endregion test_schedule_not_found_skips
# #region test_baseline_expired_full_mode [C:1] [TYPE Function]
def test_baseline_expired_full_mode():
"""execution_mode="full" with expired baseline → trigger_type stays 'scheduled'."""
from src.plugins.translate.scheduler import execute_scheduled_translation
mock_db = MagicMock()
mock_session_maker = MagicMock(return_value=mock_db)
mock_config = MagicMock()
schedule = MagicMock()
schedule.id = "sched-1"
schedule.job_id = "job-1"
schedule.is_active = True
schedule.execution_mode = "full"
most_recent = _make_most_recent_run(created_at_delta_days=100) # expired
q1 = MagicMock()
q1.filter.return_value.first.return_value = schedule
q2 = MagicMock()
q2.filter.return_value.order_by.return_value.first.return_value = None
q3 = MagicMock()
q3.filter.return_value.order_by.return_value.first.return_value = most_recent
mock_db.query.side_effect = [q1, q2, q3]
mock_run = _make_mock_run()
with patch(
"src.plugins.translate.orchestrator.TranslationOrchestrator"
) as mock_orch_cls, \
patch("src.dependencies.get_async_job_runner", return_value=_make_coro_runner()):
mock_orch = MagicMock()
mock_orch.start_run.return_value = mock_run
mock_orch_cls.return_value = mock_orch
with patch(
"src.plugins.translate.scheduler.TranslationEventLog"
) as mock_event_log_cls:
mock_event_log = MagicMock()
mock_event_log_cls.return_value = mock_event_log
execute_scheduled_translation(
schedule_id="sched-1",
job_id="job-1",
db_session_maker=mock_session_maker,
config_manager=mock_config,
execution_mode="full",
)
assert mock_run.trigger_type == "scheduled"
mock_event_log.log_event.assert_called_once()
event_call = mock_event_log.log_event.call_args
assert event_call.kwargs["payload"]["reason"] == "baseline_expired"
mock_db.close.assert_called_once()
# #endregion test_baseline_expired_full_mode
# #region test_execution_error_handled [C:1] [TYPE Function]
def test_execution_error_handled():
"""execute_background is called even when execute_run would raise — scheduler dispatches safely."""
from src.plugins.translate.scheduler import execute_scheduled_translation
mock_db = MagicMock()
mock_session_maker = MagicMock(return_value=mock_db)
mock_config = MagicMock()
schedule = MagicMock()
schedule.id = "sched-1"
schedule.job_id = "job-1"
schedule.is_active = True
schedule.execution_mode = "full"
q1 = MagicMock()
q1.filter.return_value.first.return_value = schedule
q2 = MagicMock()
q2.filter.return_value.order_by.return_value.first.return_value = None
q3 = MagicMock()
q3.filter.return_value.order_by.return_value.first.return_value = None
mock_db.query.side_effect = [q1, q2, q3]
mock_run = _make_mock_run()
with patch(
"src.plugins.translate.orchestrator.TranslationOrchestrator"
) as mock_orch_cls, \
patch("src.dependencies.get_async_job_runner", return_value=_make_coro_runner()):
mock_orch = MagicMock()
mock_orch.start_run.return_value = mock_run
mock_orch_cls.return_value = mock_orch
execute_scheduled_translation(
schedule_id="sched-1",
job_id="job-1",
db_session_maker=mock_session_maker,
config_manager=mock_config,
execution_mode="full",
)
# Scheduler dispatches via execute_background — does NOT catch execution errors itself
mock_orch_cls.execute_background.assert_called_once_with(
run_id=mock_run.id,
db_session_maker=mock_session_maker,
config_manager=mock_config,
current_user="scheduler",
)
# Schedule tracking is updated at dispatch time
assert schedule.last_run_at is not None
mock_db.close.assert_called_once()
# #endregion test_execution_error_handled
# #region test_execution_run_status_failed_path [C:1] [TYPE Function]
def test_execution_run_status_failed_path():
"""Scheduler dispatches regardless of run.status — status is handled by background task."""
from src.plugins.translate.scheduler import execute_scheduled_translation
mock_db = MagicMock()
mock_session_maker = MagicMock(return_value=mock_db)
mock_config = MagicMock()
schedule = MagicMock()
schedule.id = "sched-fail"
schedule.job_id = "job-fail"
schedule.is_active = True
schedule.execution_mode = "full"
most_recent = _make_most_recent_run(created_at_delta_days=30)
q1 = MagicMock()
q1.filter.return_value.first.return_value = schedule
q2 = MagicMock()
q2.filter.return_value.order_by.return_value.first.return_value = None
q3 = MagicMock()
q3.filter.return_value.order_by.return_value.first.return_value = most_recent
mock_db.query.side_effect = [q1, q2, q3]
mock_run = _make_mock_run(run_id="run-fail", job_id="job-fail")
mock_run.status = "FAILED" # simulate run already failed
with patch(
"src.plugins.translate.orchestrator.TranslationOrchestrator"
) as mock_orch_cls, \
patch("src.dependencies.get_async_job_runner", return_value=_make_coro_runner()):
mock_orch = MagicMock()
mock_orch.start_run.return_value = mock_run
mock_orch_cls.return_value = mock_orch
execute_scheduled_translation(
schedule_id="sched-fail",
job_id="job-fail",
db_session_maker=mock_session_maker,
config_manager=mock_config,
execution_mode="full",
)
# Scheduler dispatches background task regardless of run.status
mock_orch_cls.execute_background.assert_called()
mock_db.close.assert_called_once()
# #endregion test_execution_run_status_failed_path
# #region test_execution_notification_error [C:1] [TYPE Function]
def test_execution_notification_error():
"""Scheduler dispatches safely — notification moves to background task."""
from src.plugins.translate.scheduler import execute_scheduled_translation
mock_db = MagicMock()
mock_session_maker = MagicMock(return_value=mock_db)
mock_config = MagicMock()
schedule = MagicMock()
schedule.id = "sched-notif"
schedule.job_id = "job-notif"
schedule.is_active = True
schedule.execution_mode = "full"
q1 = MagicMock()
q1.filter.return_value.first.return_value = schedule
q2 = MagicMock()
q2.filter.return_value.order_by.return_value.first.return_value = None
q3 = MagicMock()
q3.filter.return_value.order_by.return_value.first.return_value = None
mock_db.query.side_effect = [q1, q2, q3]
mock_run = _make_mock_run(run_id="run-notif", job_id="job-notif")
with patch(
"src.plugins.translate.orchestrator.TranslationOrchestrator"
) as mock_orch_cls, \
patch("src.dependencies.get_async_job_runner", return_value=_make_coro_runner()):
mock_orch = MagicMock()
mock_orch.start_run.return_value = mock_run
mock_orch_cls.return_value = mock_orch
execute_scheduled_translation(
schedule_id="sched-notif",
job_id="job-notif",
db_session_maker=mock_session_maker,
config_manager=mock_config,
execution_mode="full",
)
# Scheduler dispatches — notification is handled by execute_background
mock_orch_cls.execute_background.assert_called()
# Schedule tracking is updated at dispatch
assert schedule.last_run_at is not None
mock_db.close.assert_called_once()
# #endregion test_execution_notification_error
# #endregion TestTranslateSchedulerExecution
Reference in New Issue View Git Blame Copy Permalink