busya
de023c7a31
Backend: - MigrationPlugin.execute — remove AsyncJobRunner.run() deadlock on post-migration ID sync (2 deadlocks total: plugins/migration.py + api/routes/migration.py trigger_sync_now). Replace blocking runner.run with direct await. - MigrationPlugin.execute — fix temp file leak (dry_run=True prevented cleanup). - MigrationPlugin.execute — IdMappingService(SessionLocal()) now closed in finally. - MigrationPlugin.execute — wire IdMappingService into MigrationEngine constructor so cross-filter patching actually works instead of silently skipping. - MigrationPlugin.execute — SupersetClient.aclose() in finally to prevent httpx connection pool leak. - TaskManager — _async_tasks dict leak (add_done_callback cleanup). - JobLifecycle — CancelledError handler (tasks stuck in RUNNING after cancel). - JobLifecycle — persist_task BEFORE _broadcast_task_status (crash consistency). - JobLifecycle — wait_for_resolution/wait_for_input now have 3600s timeout. - AsyncJobRunner.run — 300s timeout on future.result() (APScheduler thread safety). Translate scheduler: - execute_scheduled_translation — replace blocking runner.run(orch.execute_run(run)) with fire-and-forget TranslationOrchestrator.execute_background(). APScheduler thread is freed in ms instead of blocking for the full translation duration. Translation runs of 10k+ rows (200+ LLM batches, hours) no longer hit the 300s runner timeout. - TranslationOrchestrator.execute_background — new static method: opens own DB session, dispatches asyncio.create_task, handles errors + notification. - Scheduler last_run_at updated at dispatch time (not after completion). Frontend: - MigrationModel.stepReady[3] now requires dryRunResult != null (was always true, allowing UI to reach step 3 without dry-run). - WizardModel.goToStep gate for step 3 uses stepReady[3]. - +page.svelte — dryRunResult no longer self-clears via reactive loop. - Progress bar step 3 indicator gate fixed for new readiness logic. Tests: - 2 new regression tests for migration sync (deadlock-free, completes cleanly). - test_migration_plugin.py — _make_mock_superset_client/_make_mock_mapping_service helpers for proper async mock behavior (aclose, sync_environment AsyncMock). - 10 scheduled-translation tests updated for fire-and-forget pattern. - MigrationModel.test.ts — step 3 invariant + goToStep block test. - All affected tests: 104 backend + 79 frontend = 183 passed.
854 lines
36 KiB
Python
854 lines
36 KiB
Python
# #region Test.MigrationPlugin [C:3] [TYPE Module] [SEMANTICS test,migration,plugin]
|
|
# @BRIEF Unit tests for MigrationPlugin — properties, get_schema, execute, and edge cases.
|
|
# @RELATION BINDS_TO -> [MigrationPlugin]
|
|
# @TEST_EDGE: missing_env_field -> Raises ValueError
|
|
# @TEST_EDGE: invalid_regex_pattern -> Regex compilation succeeds filter
|
|
# @TEST_EDGE: target_api_timeout -> Partial success with failed_dashboards
|
|
# @TEST_EDGE: no_selection_criteria -> Returns NO_SELECTION
|
|
# @TEST_EDGE: zero_dashboards_match -> Returns NO_MATCHES
|
|
# @TEST_EDGE: uninitialized_client -> Raises ValueError on falsy client
|
|
# @TEST_EDGE: non_dict_db_mapping -> Falls back to empty dict
|
|
# @TEST_EDGE: transform_retry_after_resolution -> Retries after wait_for_resolution
|
|
# @TEST_EDGE: password_error_yaml_pattern -> Extracts db_name from yaml path
|
|
# @TEST_EDGE: id_mapping_sync_failure -> Non-fatal, continues
|
|
|
|
import pytest
|
|
from unittest.mock import AsyncMock, MagicMock, patch, PropertyMock
|
|
|
|
from src.plugins.migration import MigrationPlugin
|
|
|
|
|
|
# ── Helpers ──
|
|
|
|
|
|
def _make_env(id_val="env-1", name="Source Env"):
|
|
env = MagicMock()
|
|
env.id = id_val
|
|
env.name = name
|
|
env.url = "https://superset.example.com"
|
|
env.username = "admin"
|
|
env.password = "secret"
|
|
return env
|
|
|
|
|
|
def _make_task_context():
|
|
ctx = MagicMock()
|
|
ctx.logger = MagicMock()
|
|
ctx.logger.with_source.return_value = MagicMock()
|
|
return ctx
|
|
|
|
|
|
def _make_dashboard(dash_id=1, title="Test Dash"):
|
|
return {"id": dash_id, "slug": f"slug-{dash_id}", "dashboard_title": title}
|
|
|
|
|
|
def _make_mock_superset_client():
|
|
"""Create a MagicMock that can be used as a SupersetClient (aclose is awaitable)."""
|
|
client = MagicMock()
|
|
client.aclose = AsyncMock()
|
|
return client
|
|
|
|
|
|
def _make_mock_mapping_service():
|
|
"""Create a MagicMock that can be used as IdMappingService (sync_environment is awaitable)."""
|
|
svc = MagicMock()
|
|
svc.sync_environment = AsyncMock()
|
|
return svc
|
|
|
|
|
|
class TestMigrationPluginProperties:
|
|
"""Verify static property values."""
|
|
|
|
def test_id(self):
|
|
plugin = MigrationPlugin()
|
|
assert plugin.id == "superset-migration"
|
|
|
|
def test_name(self):
|
|
plugin = MigrationPlugin()
|
|
assert plugin.name == "Superset Dashboard Migration"
|
|
|
|
def test_description(self):
|
|
plugin = MigrationPlugin()
|
|
assert plugin.description == "Migrates dashboards between Superset environments."
|
|
|
|
def test_version(self):
|
|
plugin = MigrationPlugin()
|
|
assert plugin.version == "1.0.0"
|
|
|
|
def test_ui_route(self):
|
|
plugin = MigrationPlugin()
|
|
assert plugin.ui_route == "/migration"
|
|
|
|
|
|
class TestMigrationPluginGetSchema:
|
|
"""Verify get_schema — dynamic schema based on environments."""
|
|
|
|
def test_get_schema_with_envs(self):
|
|
plugin = MigrationPlugin()
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [_make_env("e1", "Dev"), _make_env("e2", "Prod")]
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm):
|
|
schema = plugin.get_schema()
|
|
|
|
assert schema["type"] == "object"
|
|
assert schema["properties"]["from_env"]["enum"] == ["Dev", "Prod"]
|
|
assert schema["properties"]["to_env"]["enum"] == ["Dev", "Prod"]
|
|
assert "dashboard_regex" in schema["properties"]
|
|
assert "from_env" in schema["required"]
|
|
assert "to_env" in schema["required"]
|
|
|
|
def test_get_schema_no_envs(self):
|
|
"""Fallback to dev/prod when no environments configured."""
|
|
plugin = MigrationPlugin()
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = []
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm):
|
|
schema = plugin.get_schema()
|
|
|
|
assert schema["properties"]["from_env"]["enum"] == ["dev", "prod"]
|
|
|
|
|
|
class TestMigrationPluginExecute:
|
|
"""Verify MigrationPlugin.execute with various scenarios."""
|
|
|
|
# ── Happy path: migrate by name + regex ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_happy_regex(self):
|
|
"""Happy: regex-based dashboard selection succeeds."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [
|
|
_make_dashboard(1, "Revenue Dashboard"),
|
|
_make_dashboard(2, "Sales Dashboard"),
|
|
]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip_content", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"from_env": "Source",
|
|
"to_env": "Target",
|
|
"dashboard_regex": "Revenue",
|
|
"replace_db_config": False,
|
|
"fix_cross_filters": True,
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert result["selected_dashboards"] == 1
|
|
assert len(result["migrated_dashboards"]) == 1
|
|
assert result["migrated_dashboards"][0]["title"] == "Revenue Dashboard"
|
|
|
|
# ── Happy path: migrate by selected_ids ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_happy_selected_ids(self):
|
|
"""Happy: selected_ids-based dashboard selection."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [
|
|
_make_dashboard(1, "Dash A"),
|
|
_make_dashboard(2, "Dash B"),
|
|
_make_dashboard(3, "Dash C"),
|
|
]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"from_env": "Source",
|
|
"to_env": "Target",
|
|
"dashboard_regex": "Revenue",
|
|
"replace_db_config": False,
|
|
"fix_cross_filters": True,
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert result["selected_dashboards"] == 1
|
|
assert len(result["migrated_dashboards"]) == 1
|
|
assert result["migrated_dashboards"][0]["title"] == "Revenue Dashboard"
|
|
|
|
# ── Happy path: migrate by selected_ids ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_happy_selected_ids(self):
|
|
"""Happy: selected_ids-based dashboard selection."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [
|
|
_make_dashboard(1, "Dash A"),
|
|
_make_dashboard(2, "Dash B"),
|
|
_make_dashboard(3, "Dash C"),
|
|
]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1, 3],
|
|
"replace_db_config": False,
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert result["selected_dashboards"] == 2
|
|
|
|
# ── Edge: environment not found ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_env_not_found(self):
|
|
"""Negative: missing environment raises ValueError."""
|
|
plugin = MigrationPlugin()
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = []
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm):
|
|
with pytest.raises(ValueError, match="Could not resolve source or target"):
|
|
await plugin.execute({
|
|
"from_env": "Missing",
|
|
"to_env": "Target",
|
|
"dashboard_regex": ".*",
|
|
})
|
|
|
|
# ── Edge: no selection criteria ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_no_selection_criteria(self):
|
|
"""Edge: no dashboard_regex or selected_ids returns NO_SELECTION."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, []))
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient', return_value=mock_src_client):
|
|
|
|
result = await plugin.execute({
|
|
"from_env": "Source",
|
|
"to_env": "Target",
|
|
})
|
|
|
|
assert result["status"] == "NO_SELECTION"
|
|
|
|
# ── Edge: zero dashboards match ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_zero_matches(self):
|
|
"""Edge: regex matches zero dashboards returns NO_MATCHES."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Other")]))
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient', return_value=mock_src_client):
|
|
|
|
result = await plugin.execute({
|
|
"from_env": "Source",
|
|
"to_env": "Target",
|
|
"dashboard_regex": "NonExistent",
|
|
})
|
|
|
|
assert result["status"] == "NO_MATCHES"
|
|
|
|
# ── Edge: partial success ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_partial_success(self):
|
|
"""Edge: one dashboard fails, others succeed → PARTIAL_SUCCESS."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [
|
|
_make_dashboard(1, "Working"),
|
|
_make_dashboard(2, "Failing"),
|
|
]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(side_effect=[
|
|
(b"zip_ok", "meta"), RuntimeError("Export failed for dash 2")
|
|
])
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1, 2],
|
|
"replace_db_config": False,
|
|
})
|
|
|
|
assert result["status"] == "PARTIAL_SUCCESS"
|
|
assert len(result["migrated_dashboards"]) == 1
|
|
assert len(result["failed_dashboards"]) == 1
|
|
|
|
# ── Edge: transform_zip fails, no replace_db_config ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_transform_fails_no_replace(self):
|
|
"""Edge: transform_zip fails and replace_db_config is False → dashboard in failed list."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = False
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": False,
|
|
})
|
|
|
|
assert len(result["failed_dashboards"]) == 1
|
|
assert "transform" in result["failed_dashboards"][0]["error"].lower()
|
|
|
|
# ── Edge: password injection flow ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_password_injection(self):
|
|
"""Edge: missing DB password triggers await_input, retries with password."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_task_manager = MagicMock()
|
|
mock_task_manager.get_task.return_value = MagicMock(
|
|
params={"passwords": {"PostgreSQL": "secret123"}}
|
|
)
|
|
mock_task_manager.await_input = MagicMock()
|
|
mock_task_manager.wait_for_input = AsyncMock()
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
# First export succeeds, import fails with password error
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
# First import fails with password error, retry succeeds
|
|
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[
|
|
RuntimeError("Must provide a password for the database 'PostgreSQL'"),
|
|
None,
|
|
])
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.dependencies.get_task_manager', return_value=mock_task_manager), \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": False,
|
|
"_task_id": "task-1",
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert len(result["migrated_dashboards"]) == 1
|
|
# Password params should be deleted from task after retry
|
|
assert "passwords" not in mock_task_manager.get_task("task-1").params
|
|
|
|
# ── Edge: with TaskContext logger ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_with_task_context(self):
|
|
"""Edge: execute uses TaskContext for logging when provided."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
ctx = _make_task_context()
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
}, context=ctx)
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
|
|
# ── Edge: replace_db_config loads mappings from DB ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_replace_db_config(self):
|
|
"""Edge: replace_db_config loads DB mappings from catalog."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
# Mock DB session with Environment and DatabaseMapping
|
|
mock_db = MagicMock()
|
|
mock_db_env = MagicMock()
|
|
mock_db_env.id = 1
|
|
mock_db_env.name = "Source"
|
|
mock_db_env2 = MagicMock()
|
|
mock_db_env2.id = 2
|
|
mock_db_env2.name = "Target"
|
|
|
|
mock_mapping = MagicMock()
|
|
mock_mapping.source_db_uuid = "src-uuid"
|
|
mock_mapping.target_db_uuid = "tgt-uuid"
|
|
|
|
# DB query chain: Environment filter → first returns mock_db_env
|
|
# DatabaseMapping filter → all returns [mock_mapping]
|
|
mock_db.query.return_value.filter.return_value.first.side_effect = [mock_db_env, mock_db_env2]
|
|
mock_db.query.return_value.filter.return_value.all.return_value = [mock_mapping]
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal', return_value=mock_db):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": True,
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert result["mapping_count"] > 0
|
|
|
|
# ── Edge: uninitialized client ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_uninitialized_client_raises(self):
|
|
"""Edge: SupersetClient returns falsy -> ValueError."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC:
|
|
|
|
# Return None (falsy) for the source client
|
|
MockSC.side_effect = [None, _make_mock_superset_client()]
|
|
|
|
with pytest.raises(ValueError, match="Clients not initialized"):
|
|
await plugin.execute({
|
|
"from_env": "Source",
|
|
"to_env": "Target",
|
|
"dashboard_regex": ".*",
|
|
})
|
|
|
|
# ── Edge: non-dict db_mappings ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_non_dict_db_mapping(self):
|
|
"""Edge: db_mappings param is not a dict, falls back to empty dict."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [_make_dashboard(1, "Test Dash")]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"db_mappings": "not-a-dict", # non-dict value
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
|
|
# ── Edge: transform fails with replace_db_config, retries ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_transform_retry_after_resolution(self):
|
|
"""Edge: transform fails -> wait_for_resolution -> retry succeeds."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_task_manager = MagicMock()
|
|
mock_task_manager.wait_for_resolution = AsyncMock()
|
|
|
|
# Use a single client mock that handles both get_dashboards and import_dashboard
|
|
mock_client = _make_mock_superset_client()
|
|
mock_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
# First call returns False (fails), second call returns True (retry succeeds)
|
|
mock_engine.transform_zip.side_effect = [False, True]
|
|
|
|
# Mock DB session for Environments and DatabaseMapping
|
|
mock_db = MagicMock()
|
|
mock_db_env = MagicMock()
|
|
mock_db_env.id = 1
|
|
mock_db_env.name = "Source"
|
|
mock_db_env2 = MagicMock()
|
|
mock_db_env2.id = 2
|
|
mock_db_env2.name = "Target"
|
|
# Extend side_effect to cover initial (2) + retry (2) calls to first()
|
|
mock_db.query.return_value.filter.return_value.first.side_effect = [
|
|
mock_db_env, mock_db_env2, mock_db_env, mock_db_env2
|
|
]
|
|
mock_db.query.return_value.filter.return_value.all.return_value = []
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient', return_value=mock_client), \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.dependencies.get_task_manager', return_value=mock_task_manager), \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal', return_value=mock_db):
|
|
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": True,
|
|
"_task_id": "task-retry-1",
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert mock_task_manager.wait_for_resolution.called
|
|
# transform_zip should have been called twice
|
|
assert mock_engine.transform_zip.call_count == 2
|
|
# import_dashboard should have been called once (after retry)
|
|
assert mock_client.import_dashboard.called
|
|
|
|
# ── Edge: password error with yaml path pattern ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_password_error_yaml_pattern(self):
|
|
"""Edge: password error with 'databases/db_name.yaml' pattern."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_task_manager = MagicMock()
|
|
mock_task_manager.get_task.return_value = MagicMock(
|
|
params={"passwords": {"PostgreSQL": "secret123"}}
|
|
)
|
|
mock_task_manager.await_input = MagicMock()
|
|
mock_task_manager.wait_for_input = AsyncMock()
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[
|
|
RuntimeError("Must provide a password for the database databases/PostgreSQL.yaml"),
|
|
None,
|
|
])
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.dependencies.get_task_manager', return_value=mock_task_manager), \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": False,
|
|
"_task_id": "task-pw-1",
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert len(result["migrated_dashboards"]) == 1
|
|
|
|
# ── Edge: ID mapping sync failure is non-fatal ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_id_mapping_sync_failure(self):
|
|
"""Edge: IdMappingService.sync_environment raises, execution continues."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [_make_dashboard(1, "Dash")]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
mock_db_session = MagicMock()
|
|
mock_sync = MagicMock()
|
|
mock_sync.sync_environment = AsyncMock(side_effect=RuntimeError("Sync failed"))
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=mock_sync), \
|
|
patch('src.plugins.migration.SessionLocal', return_value=mock_db_session):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
# Should not raise — sync failure is caught and logged
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert mock_sync.sync_environment.called, "sync_environment should have been awaited"
|
|
|
|
# ── Regression: post-migration sync does not deadlock (direct await, not runner.run) ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_id_mapping_sync_completes_without_deadlock(self):
|
|
"""Regression: post-migration sync completes via direct await (not runner.run)."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [_make_dashboard(1, "Dash A"), _make_dashboard(2, "Dash B")]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
mock_db_session = MagicMock()
|
|
mock_sync = MagicMock()
|
|
mock_sync.sync_environment = AsyncMock() # completes without error
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=mock_sync), \
|
|
patch('src.plugins.migration.SessionLocal', return_value=mock_db_session):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1, 2],
|
|
})
|
|
|
|
# Sync must complete and all dashboards must be migrated
|
|
assert result["status"] == "SUCCESS"
|
|
assert len(result["migrated_dashboards"]) == 2
|
|
assert result["failed_dashboards"] == []
|
|
# Verify sync was called with correct params
|
|
mock_sync.sync_environment.assert_called_once()
|
|
call_args = mock_sync.sync_environment.call_args
|
|
assert call_args[1]["incremental"] is True
|
|
|
|
# #endregion Test.MigrationPlugin
|