busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
f22b0cf41c2cdbbdbd0f92ba57bc90e542a974f9
ss-tools/backend/tests/api/test_migration.py
busya de023c7a31 fix(backend+frontend): migration deadlock, async pattern, timeout safety, fire-and-forget translations 
Backend:
- MigrationPlugin.execute — remove AsyncJobRunner.run() deadlock on post-migration
  ID sync (2 deadlocks total: plugins/migration.py + api/routes/migration.py
  trigger_sync_now). Replace blocking runner.run with direct await.
- MigrationPlugin.execute — fix temp file leak (dry_run=True prevented cleanup).
- MigrationPlugin.execute — IdMappingService(SessionLocal()) now closed in finally.
- MigrationPlugin.execute — wire IdMappingService into MigrationEngine constructor
  so cross-filter patching actually works instead of silently skipping.
- MigrationPlugin.execute — SupersetClient.aclose() in finally to prevent
  httpx connection pool leak.
- TaskManager — _async_tasks dict leak (add_done_callback cleanup).
- JobLifecycle — CancelledError handler (tasks stuck in RUNNING after cancel).
- JobLifecycle — persist_task BEFORE _broadcast_task_status (crash consistency).
- JobLifecycle — wait_for_resolution/wait_for_input now have 3600s timeout.
- AsyncJobRunner.run — 300s timeout on future.result() (APScheduler thread safety).

Translate scheduler:
- execute_scheduled_translation — replace blocking runner.run(orch.execute_run(run))
  with fire-and-forget TranslationOrchestrator.execute_background(). APScheduler
  thread is freed in ms instead of blocking for the full translation duration.
  Translation runs of 10k+ rows (200+ LLM batches, hours) no longer hit the
  300s runner timeout.
- TranslationOrchestrator.execute_background — new static method: opens own DB
  session, dispatches asyncio.create_task, handles errors + notification.
- Scheduler last_run_at updated at dispatch time (not after completion).

Frontend:
- MigrationModel.stepReady[3] now requires dryRunResult != null (was always
  true, allowing UI to reach step 3 without dry-run).
- WizardModel.goToStep gate for step 3 uses stepReady[3].
- +page.svelte  — dryRunResult no longer self-clears via reactive loop.
- Progress bar step 3 indicator gate fixed for new readiness logic.

Tests:
- 2 new regression tests for migration sync (deadlock-free, completes cleanly).
- test_migration_plugin.py — _make_mock_superset_client/_make_mock_mapping_service
  helpers for proper async mock behavior (aclose, sync_environment AsyncMock).
- 10 scheduled-translation tests updated for fire-and-forget pattern.
- MigrationModel.test.ts — step 3 invariant + goToStep block test.
- All affected tests: 104 backend + 79 frontend = 183 passed.
2026-07-07 21:04:23 +03:00

410 lines
15 KiB
Python
Raw Blame History

# #region Test.Api.Migration [C:3] [TYPE Module] [SEMANTICS test,migration,routes]
# @BRIEF Tests for migration API routes — dashboards, execute, dry-run, settings, mappings, sync.
# @RELATION BINDS_TO -> [MigrationApi]
import os
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:")
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:")
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-tests")
os.environ.setdefault("DEV_MODE", "true")
import sys
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
_src = str(Path(__file__).resolve().parent.parent.parent / "src")
if _src not in sys.path:
sys.path.insert(0, _src)
def _make_env(id_val="env-1", name="Test Env", url="https://superset.example.com"):
env = MagicMock()
env.id = id_val
env.name = name
env.url = url
return env
def _make_client(overrides: dict | None = None) -> TestClient:
from src.api.routes.migration import router
from src.core.database import get_db
from src.dependencies import get_config_manager, get_current_user, get_task_manager
from src.schemas.auth import User, RoleSchema
app = FastAPI()
app.include_router(router)
mock_user = User(
id="admin-1", username="admin", email="admin@x.com",
auth_source="LOCAL",
created_at=__import__("datetime").datetime.now(),
roles=[RoleSchema(id="r1", name="Admin", description="", permissions=[])],
)
app.dependency_overrides[get_db] = lambda: MagicMock()
app.dependency_overrides[get_current_user] = lambda: mock_user
app.dependency_overrides[get_config_manager] = lambda: MagicMock()
app.dependency_overrides[get_task_manager] = lambda: MagicMock()
if overrides:
for dep, fn in overrides.items():
app.dependency_overrides[dep] = fn
return TestClient(app)
# ── get_dashboards ──
class TestGetDashboards:
"""GET /api/environments/{env_id}/dashboards"""
def test_success(self):
mock_config = MagicMock()
mock_config.get_environments.return_value = [_make_env()]
dashboards = [{"id": 1, "title": "Test Dash", "last_modified": "2024-01-01T00:00:00", "status": "published"}]
mock_client = MagicMock()
mock_client.get_dashboards_summary = AsyncMock(return_value=dashboards)
from src.dependencies import get_config_manager
with patch("src.api.routes.migration.AsyncSupersetClient", return_value=mock_client):
client = _make_client({get_config_manager: lambda: mock_config})
resp = client.get("/api/environments/env-1/dashboards")
assert resp.status_code == 200
data = resp.json()
assert len(data) == 1
assert data[0]["id"] == 1
def test_environment_not_found(self):
mock_config = MagicMock()
mock_config.get_environments.return_value = []
from src.dependencies import get_config_manager
client = _make_client({get_config_manager: lambda: mock_config})
resp = client.get("/api/environments/bad-env/dashboards")
assert resp.status_code == 404
# ── execute_migration ──
class TestExecuteMigration:
"""POST /migration/execute"""
SELECTION_PAYLOAD = {
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1, 2, 3],
"replace_db_config": True,
"fix_cross_filters": False,
}
def test_success(self):
mock_config = MagicMock()
e1 = _make_env("env-1")
e2 = _make_env("env-2")
mock_config.get_environments.return_value = [e1, e2]
mock_tm = MagicMock()
mock_tm.create_task = AsyncMock(return_value=MagicMock(id="task-123"))
from src.dependencies import get_config_manager, get_task_manager
client = _make_client({
get_config_manager: lambda: mock_config,
get_task_manager: lambda: mock_tm,
})
resp = client.post("/api/migration/execute", json=self.SELECTION_PAYLOAD)
assert resp.status_code == 200
data = resp.json()
assert data["task_id"] == "task-123"
def test_invalid_source_env(self):
mock_config = MagicMock()
mock_config.get_environments.return_value = [_make_env("env-1")]
from src.dependencies import get_config_manager
client = _make_client({get_config_manager: lambda: mock_config})
resp = client.post("/api/migration/execute", json=self.SELECTION_PAYLOAD)
assert resp.status_code == 400
def test_invalid_target_env(self):
mock_config = MagicMock()
mock_config.get_environments.return_value = [_make_env("env-2")]
from src.dependencies import get_config_manager
client = _make_client({get_config_manager: lambda: mock_config})
resp = client.post("/api/migration/execute", json=self.SELECTION_PAYLOAD)
assert resp.status_code == 400
def test_invalid_target_env(self):
mock_config = MagicMock()
mock_config.get_environments.return_value = [_make_env("env-2")]
from src.dependencies import get_config_manager
client = _make_client({get_config_manager: lambda: mock_config})
resp = client.post("/api/migration/execute", json=self.SELECTION_PAYLOAD)
assert resp.status_code == 400
def test_task_creation_fails(self):
mock_config = MagicMock()
e1 = _make_env("env-1")
e2 = _make_env("env-2")
mock_config.get_environments.return_value = [e1, e2]
mock_tm = MagicMock()
mock_tm.create_task = AsyncMock(side_effect=RuntimeError("task failed"))
from src.dependencies import get_config_manager, get_task_manager
client = _make_client({
get_config_manager: lambda: mock_config,
get_task_manager: lambda: mock_tm,
})
resp = client.post("/api/migration/execute", json=self.SELECTION_PAYLOAD)
assert resp.status_code == 500
# ── dry_run_migration ──
class TestDryRunMigration:
"""POST /migration/dry-run"""
SELECTION_PAYLOAD = {
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1, 2],
"replace_db_config": False,
"fix_cross_filters": False,
}
def test_success(self):
mock_config = MagicMock()
e1 = _make_env("env-1")
e2 = _make_env("env-2")
mock_config.get_environments.return_value = [e1, e2]
mock_dry_run = MagicMock()
mock_dry_run.run = AsyncMock(return_value={"summary": "ok", "risks": []})
from src.dependencies import get_config_manager
with patch("src.api.routes.migration.MigrationDryRunService", return_value=mock_dry_run):
client = _make_client({get_config_manager: lambda: mock_config})
resp = client.post("/api/migration/dry-run", json=self.SELECTION_PAYLOAD)
assert resp.status_code == 200
assert resp.json()["summary"] == "ok"
def test_identical_environments(self):
mock_config = MagicMock()
e1 = _make_env("env-1")
mock_config.get_environments.return_value = [e1]
from src.dependencies import get_config_manager
client = _make_client({get_config_manager: lambda: mock_config})
resp = client.post("/api/migration/dry-run", json={
"source_env_id": "env-1", "target_env_id": "env-1", "selected_ids": [1],
})
assert resp.status_code == 400
def test_invalid_environments(self):
mock_config = MagicMock()
mock_config.get_environments.return_value = []
from src.dependencies import get_config_manager
client = _make_client({get_config_manager: lambda: mock_config})
resp = client.post("/api/migration/dry-run", json=self.SELECTION_PAYLOAD)
assert resp.status_code == 400
def test_no_dashboards_selected(self):
mock_config = MagicMock()
e1 = _make_env("env-1")
e2 = _make_env("env-2")
mock_config.get_environments.return_value = [e1, e2]
from src.dependencies import get_config_manager
client = _make_client({get_config_manager: lambda: mock_config})
resp = client.post("/api/migration/dry-run", json={
"source_env_id": "env-1", "target_env_id": "env-2", "selected_ids": [],
})
assert resp.status_code == 400
def test_value_error_from_service(self):
mock_config = MagicMock()
e1 = _make_env("env-1")
e2 = _make_env("env-2")
mock_config.get_environments.return_value = [e1, e2]
mock_dry_run = MagicMock()
mock_dry_run.run = AsyncMock(side_effect=ValueError("orchestrator error"))
from src.dependencies import get_config_manager
with patch("src.api.routes.migration.MigrationDryRunService", return_value=mock_dry_run):
client = _make_client({get_config_manager: lambda: mock_config})
resp = client.post("/api/migration/dry-run", json=self.SELECTION_PAYLOAD)
assert resp.status_code == 500
# ── get_migration_settings ──
class TestGetMigrationSettings:
"""GET /migration/settings"""
def test_success(self):
mock_config = MagicMock()
mock_cfg = MagicMock()
mock_cfg.settings.migration_sync_cron = "0 2 * * *"
mock_config.get_config.return_value = mock_cfg
from src.dependencies import get_config_manager
client = _make_client({get_config_manager: lambda: mock_config})
resp = client.get("/api/migration/settings")
assert resp.status_code == 200
assert resp.json()["cron"] == "0 2 * * *"
# ── update_migration_settings ──
class TestUpdateMigrationSettings:
"""PUT /migration/settings"""
def test_success(self):
mock_config = MagicMock()
mock_cfg = MagicMock()
mock_config.get_config.return_value = mock_cfg
from src.dependencies import get_config_manager
client = _make_client({get_config_manager: lambda: mock_config})
resp = client.put("/api/migration/settings", json={"cron": "0 3 * * *"})
assert resp.status_code == 200
data = resp.json()
assert data["cron"] == "0 3 * * *"
assert data["status"] == "updated"
mock_config.save_config.assert_called_once()
def test_missing_cron_field(self):
mock_config = MagicMock()
from src.dependencies import get_config_manager
client = _make_client({get_config_manager: lambda: mock_config})
resp = client.put("/api/migration/settings", json={})
assert resp.status_code == 400
# ── get_resource_mappings ──
class TestGetResourceMappings:
"""GET /migration/mappings-data"""
def test_success(self):
mock_db = MagicMock()
mock_query = MagicMock()
mock_db.query.return_value = mock_query
mock_query.filter.return_value = mock_query
mock_query.count.return_value = 2
def _mock_resource_type(value: str):
m = MagicMock()
m.value = value
return m
mock_query.order_by.return_value.offset.return_value.limit.return_value.all.return_value = [
MagicMock(id=1, environment_id="env-1", resource_type=_mock_resource_type("database"), uuid="u1",
remote_integer_id=100, resource_name="DB", last_synced_at=None),
MagicMock(id=2, environment_id="env-1", resource_type=_mock_resource_type("database"), uuid="u2",
remote_integer_id=101, resource_name="DB2", last_synced_at=None),
]
from src.core.database import get_db
client = _make_client({get_db: lambda: mock_db})
resp = client.get("/api/migration/mappings-data")
assert resp.status_code == 200
data = resp.json()
assert data["total"] == 2
assert len(data["items"]) == 2
def test_with_filters(self):
mock_db = MagicMock()
mock_query = MagicMock()
mock_db.query.return_value = mock_query
mock_query.filter.return_value = mock_query
mock_query.count.return_value = 1
mock_query.order_by.return_value.offset.return_value.limit.return_value.all.return_value = []
from src.core.database import get_db
client = _make_client({get_db: lambda: mock_db})
resp = client.get("/api/migration/mappings-data?env_id=env-1&resource_type=database&search=test&skip=0&limit=10")
assert resp.status_code == 200
# ── trigger_sync_now ──
class TestTriggerSyncNow:
"""POST /migration/sync-now"""
def test_success(self):
mock_config = MagicMock()
mock_cfg = MagicMock()
e1 = _make_env("env-1", "Env 1", "https://s1.example.com")
e2 = _make_env("env-2", "Env 2", "https://s2.example.com")
mock_cfg.environments = [e1, e2]
mock_config.get_config.return_value = mock_cfg
mock_db = MagicMock()
mock_db.query.return_value.filter_by.return_value.first.side_effect = [None, None]
mock_sync = MagicMock()
mock_sync.sync_environment = AsyncMock()
from src.core.database import get_db
from src.dependencies import get_config_manager
with patch("src.api.routes.migration.IdMappingService", return_value=mock_sync), \
patch("src.api.routes.migration.AsyncSupersetClient"):
client = _make_client({
get_config_manager: lambda: mock_config,
get_db: lambda: mock_db,
})
resp = client.post("/api/migration/sync-now")
assert resp.status_code == 200
data = resp.json()
assert data["status"] == "completed"
assert data["synced_count"] == 2
def test_no_environments(self):
mock_config = MagicMock()
mock_cfg = MagicMock()
mock_cfg.environments = []
mock_config.get_config.return_value = mock_cfg
from src.dependencies import get_config_manager
client = _make_client({get_config_manager: lambda: mock_config})
resp = client.post("/api/migration/sync-now")
assert resp.status_code == 400
def test_sync_with_failures(self):
mock_config = MagicMock()
mock_cfg = MagicMock()
e1 = _make_env("env-1")
e2 = _make_env("env-2")
mock_cfg.environments = [e1, e2]
mock_config.get_config.return_value = mock_cfg
mock_db = MagicMock()
mock_db.query.return_value.filter_by.return_value.first.return_value = None
mock_sync = MagicMock()
mock_sync.sync_environment = AsyncMock()
from src.core.database import get_db
from src.dependencies import get_config_manager
# Make AsyncSupersetClient raise on 2nd call to simulate sync failure
with patch("src.api.routes.migration.IdMappingService", return_value=mock_sync), \
patch("src.api.routes.migration.AsyncSupersetClient", side_effect=[MagicMock(), RuntimeError("Connection failed")]):
client = _make_client({
get_config_manager: lambda: mock_config,
get_db: lambda: mock_db,
})
resp = client.post("/api/migration/sync-now")
assert resp.status_code == 200
data = resp.json()
assert data["synced_count"] == 1
assert data["failed_count"] == 1
# #endregion Test.Api.Migration
Reference in New Issue View Git Blame Copy Permalink