1423 lines
64 KiB
Python
1423 lines
64 KiB
Python
# #region Test.MigrationPlugin [C:3] [TYPE Module] [SEMANTICS test,migration,plugin]
|
|
# @BRIEF Unit tests for MigrationPlugin — properties, get_schema, execute, and edge cases.
|
|
# @RELATION BINDS_TO -> [Plugin.Migration.MigrationPlugin]
|
|
# @TEST_EDGE: missing_env_field -> Raises ValueError
|
|
# @TEST_EDGE: invalid_regex_pattern -> Regex compilation succeeds filter
|
|
# @TEST_EDGE: target_api_timeout -> Partial success with failed_dashboards
|
|
# @TEST_EDGE: no_selection_criteria -> Returns NO_SELECTION
|
|
# @TEST_EDGE: zero_dashboards_match -> Returns NO_MATCHES
|
|
# @TEST_EDGE: uninitialized_client -> Raises ValueError on falsy client
|
|
# @TEST_EDGE: non_dict_db_mapping -> Falls back to empty dict
|
|
# @TEST_EDGE: transform_retry_after_resolution -> Retries after wait_for_resolution
|
|
# @TEST_EDGE: password_error_yaml_pattern -> Extracts db_name from yaml path
|
|
# @TEST_EDGE: id_mapping_sync_failure -> Non-fatal, continues
|
|
|
|
import pytest
|
|
from unittest.mock import AsyncMock, MagicMock, patch, PropertyMock
|
|
|
|
from src.plugins.migration import MigrationPlugin
|
|
|
|
|
|
# ── Helpers ──
|
|
|
|
|
|
def _make_env(id_val="env-1", name="Source Env"):
|
|
env = MagicMock()
|
|
env.id = id_val
|
|
env.name = name
|
|
env.url = "https://superset.example.com"
|
|
env.username = "admin"
|
|
env.password = "secret"
|
|
return env
|
|
|
|
|
|
def _make_task_context():
|
|
ctx = MagicMock()
|
|
ctx.logger = MagicMock()
|
|
ctx.logger.with_source.return_value = MagicMock()
|
|
return ctx
|
|
|
|
|
|
def _make_logging_task_context():
|
|
"""Create a context whose task logger records structured metadata calls."""
|
|
ctx = MagicMock()
|
|
ctx.logger.with_source.side_effect = lambda source: ctx.logger
|
|
ctx.logger.info = MagicMock()
|
|
ctx.logger.warning = MagicMock()
|
|
ctx.logger.error = MagicMock()
|
|
return ctx
|
|
|
|
|
|
def _make_dashboard(dash_id=1, title="Test Dash"):
|
|
return {"id": dash_id, "slug": f"slug-{dash_id}", "dashboard_title": title}
|
|
|
|
|
|
def _make_mock_superset_client():
|
|
"""Create a MagicMock that can be used as a SupersetClient (aclose is awaitable)."""
|
|
client = MagicMock()
|
|
client.aclose = AsyncMock()
|
|
return client
|
|
|
|
|
|
def _make_mock_mapping_service():
|
|
"""Create a MagicMock that can be used as IdMappingService (sync_environment is awaitable)."""
|
|
svc = MagicMock()
|
|
svc.sync_environment = AsyncMock()
|
|
return svc
|
|
|
|
|
|
class TestMigrationPluginProperties:
|
|
"""Verify static property values."""
|
|
|
|
def test_id(self):
|
|
plugin = MigrationPlugin()
|
|
assert plugin.id == "superset-migration"
|
|
|
|
def test_name(self):
|
|
plugin = MigrationPlugin()
|
|
assert plugin.name == "Superset Dashboard Migration"
|
|
|
|
def test_description(self):
|
|
plugin = MigrationPlugin()
|
|
assert plugin.description == "Migrates dashboards between Superset environments."
|
|
|
|
def test_version(self):
|
|
plugin = MigrationPlugin()
|
|
assert plugin.version == "1.0.0"
|
|
|
|
def test_ui_route(self):
|
|
plugin = MigrationPlugin()
|
|
assert plugin.ui_route == "/migration"
|
|
|
|
|
|
class TestMigrationPluginGetSchema:
|
|
"""Verify get_schema — dynamic schema based on environments."""
|
|
|
|
def test_get_schema_with_envs(self):
|
|
plugin = MigrationPlugin()
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [_make_env("e1", "Dev"), _make_env("e2", "Prod")]
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm):
|
|
schema = plugin.get_schema()
|
|
|
|
assert schema["type"] == "object"
|
|
assert schema["properties"]["from_env"]["enum"] == ["Dev", "Prod"]
|
|
assert schema["properties"]["to_env"]["enum"] == ["Dev", "Prod"]
|
|
assert "dashboard_regex" in schema["properties"]
|
|
assert "from_env" in schema["required"]
|
|
assert "to_env" in schema["required"]
|
|
|
|
def test_get_schema_no_envs(self):
|
|
"""Fallback to dev/prod when no environments configured."""
|
|
plugin = MigrationPlugin()
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = []
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm):
|
|
schema = plugin.get_schema()
|
|
|
|
assert schema["properties"]["from_env"]["enum"] == ["dev", "prod"]
|
|
|
|
def test_get_schema_includes_composite_key_fields(self):
|
|
"""Composite-key sync fields are exposed with correct types and defaults."""
|
|
plugin = MigrationPlugin()
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [_make_env("e1", "Dev"), _make_env("e2", "Prod")]
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm):
|
|
schema = plugin.get_schema()
|
|
|
|
props = schema["properties"]
|
|
assert props["sync_dataset_composite_keys"]["type"] == "boolean"
|
|
assert props["sync_dataset_composite_keys"]["default"] is True
|
|
assert props["composite_key_mutation_server"]["enum"] == ["target", "source"]
|
|
assert props["composite_key_mutation_server"]["default"] == "target"
|
|
|
|
|
|
class TestMigrationPluginExecute:
|
|
"""Verify MigrationPlugin.execute with various scenarios."""
|
|
|
|
# ── Happy path: migrate by name + regex ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_happy_regex(self):
|
|
"""Happy: regex-based dashboard selection succeeds."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [
|
|
_make_dashboard(1, "Revenue Dashboard"),
|
|
_make_dashboard(2, "Sales Dashboard"),
|
|
]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip_content", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"from_env": "Source",
|
|
"to_env": "Target",
|
|
"dashboard_regex": "Revenue",
|
|
"replace_db_config": False,
|
|
"fix_cross_filters": True,
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert result["selected_dashboards"] == 1
|
|
assert len(result["migrated_dashboards"]) == 1
|
|
assert result["migrated_dashboards"][0]["title"] == "Revenue Dashboard"
|
|
|
|
# ── Happy path: migrate by selected_ids ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_happy_selected_ids(self):
|
|
"""Happy: selected_ids-based dashboard selection."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [
|
|
_make_dashboard(1, "Dash A"),
|
|
_make_dashboard(2, "Dash B"),
|
|
_make_dashboard(3, "Dash C"),
|
|
]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"from_env": "Source",
|
|
"to_env": "Target",
|
|
"dashboard_regex": "Revenue",
|
|
"replace_db_config": False,
|
|
"fix_cross_filters": True,
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert result["selected_dashboards"] == 1
|
|
assert len(result["migrated_dashboards"]) == 1
|
|
assert result["migrated_dashboards"][0]["title"] == "Revenue Dashboard"
|
|
|
|
# ── Happy path: migrate by selected_ids ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_happy_selected_ids(self):
|
|
"""Happy: selected_ids-based dashboard selection."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [
|
|
_make_dashboard(1, "Dash A"),
|
|
_make_dashboard(2, "Dash B"),
|
|
_make_dashboard(3, "Dash C"),
|
|
]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1, 3],
|
|
"replace_db_config": False,
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert result["selected_dashboards"] == 2
|
|
|
|
# ── Edge: environment not found ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_env_not_found(self):
|
|
"""Negative: missing environment raises ValueError."""
|
|
plugin = MigrationPlugin()
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = []
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm):
|
|
with pytest.raises(ValueError, match="Could not resolve source or target"):
|
|
await plugin.execute({
|
|
"from_env": "Missing",
|
|
"to_env": "Target",
|
|
"dashboard_regex": ".*",
|
|
})
|
|
|
|
# ── Edge: no selection criteria ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_no_selection_criteria(self):
|
|
"""Edge: no dashboard_regex or selected_ids returns NO_SELECTION."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, []))
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient', return_value=mock_src_client):
|
|
|
|
result = await plugin.execute({
|
|
"from_env": "Source",
|
|
"to_env": "Target",
|
|
})
|
|
|
|
assert result["status"] == "NO_SELECTION"
|
|
|
|
# ── Edge: zero dashboards match ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_zero_matches(self):
|
|
"""Edge: regex matches zero dashboards returns NO_MATCHES."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Other")]))
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient', return_value=mock_src_client):
|
|
|
|
result = await plugin.execute({
|
|
"from_env": "Source",
|
|
"to_env": "Target",
|
|
"dashboard_regex": "NonExistent",
|
|
})
|
|
|
|
assert result["status"] == "NO_MATCHES"
|
|
|
|
# ── Edge: partial success ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_partial_success(self):
|
|
"""Edge: one dashboard fails, others succeed → PARTIAL_SUCCESS."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [
|
|
_make_dashboard(1, "Working"),
|
|
_make_dashboard(2, "Failing"),
|
|
]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(side_effect=[
|
|
(b"zip_ok", "meta"), RuntimeError("Export failed for dash 2")
|
|
])
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1, 2],
|
|
"replace_db_config": False,
|
|
})
|
|
|
|
assert result["status"] == "PARTIAL_SUCCESS"
|
|
assert len(result["migrated_dashboards"]) == 1
|
|
assert len(result["failed_dashboards"]) == 1
|
|
|
|
# ── Edge: transform_zip fails, no replace_db_config ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_transform_fails_no_replace(self):
|
|
"""Edge: transform_zip fails and replace_db_config is False → dashboard in failed list."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = False
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": False,
|
|
})
|
|
|
|
assert len(result["failed_dashboards"]) == 1
|
|
assert "transform" in result["failed_dashboards"][0]["error"].lower()
|
|
|
|
# ── Edge: password injection flow ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_password_injection(self):
|
|
"""Edge: missing DB password triggers await_input, retries with password."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_task_manager = MagicMock()
|
|
mock_task_manager.get_task.return_value = MagicMock(
|
|
params={"passwords": {"PostgreSQL": "secret123"}}
|
|
)
|
|
mock_task_manager.await_input = AsyncMock()
|
|
mock_task_manager.wait_for_input = AsyncMock()
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
# First export succeeds, import fails with password error
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
# First import fails with password error, retry succeeds
|
|
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[
|
|
RuntimeError("Must provide a password for the database 'PostgreSQL'"),
|
|
None,
|
|
])
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.dependencies.get_task_manager', return_value=mock_task_manager), \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": False,
|
|
"_task_id": "task-1",
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert len(result["migrated_dashboards"]) == 1
|
|
# Password params should be deleted from task after retry
|
|
assert "passwords" not in mock_task_manager.get_task("task-1").params
|
|
|
|
# ── Edge: with TaskContext logger ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_with_task_context(self):
|
|
"""Edge: execute uses TaskContext for logging when provided."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
ctx = _make_task_context()
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
}, context=ctx)
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
|
|
# ── Edge: replace_db_config loads mappings from DB ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_replace_db_config(self):
|
|
"""Edge: replace_db_config loads DB mappings from catalog."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
# Mock DB session with Environment and DatabaseMapping
|
|
mock_db = MagicMock()
|
|
mock_db_env = MagicMock()
|
|
mock_db_env.id = 1
|
|
mock_db_env.name = "Source"
|
|
mock_db_env2 = MagicMock()
|
|
mock_db_env2.id = 2
|
|
mock_db_env2.name = "Target"
|
|
|
|
mock_mapping = MagicMock()
|
|
mock_mapping.source_db_uuid = "src-uuid"
|
|
mock_mapping.target_db_uuid = "tgt-uuid"
|
|
|
|
# DB query chain: Environment filter → first returns mock_db_env
|
|
# DatabaseMapping filter → all returns [mock_mapping]
|
|
mock_db.query.return_value.filter.return_value.first.side_effect = [mock_db_env, mock_db_env2]
|
|
mock_db.query.return_value.filter.return_value.all.return_value = [mock_mapping]
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal', return_value=mock_db):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": True,
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert result["mapping_count"] > 0
|
|
|
|
# ── Edge: uninitialized client ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_uninitialized_client_raises(self):
|
|
"""Edge: SupersetClient returns falsy -> ValueError."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC:
|
|
|
|
# Return None (falsy) for the source client
|
|
MockSC.side_effect = [None, _make_mock_superset_client()]
|
|
|
|
with pytest.raises(ValueError, match="Clients not initialized"):
|
|
await plugin.execute({
|
|
"from_env": "Source",
|
|
"to_env": "Target",
|
|
"dashboard_regex": ".*",
|
|
})
|
|
|
|
# ── Edge: non-dict db_mappings ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_non_dict_db_mapping(self):
|
|
"""Edge: db_mappings param is not a dict, falls back to empty dict."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [_make_dashboard(1, "Test Dash")]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"db_mappings": "not-a-dict", # non-dict value
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
|
|
# ── Edge: transform fails with replace_db_config, retries ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_transform_retry_after_resolution(self):
|
|
"""Edge: transform fails -> wait_for_resolution -> retry succeeds."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_task_manager = MagicMock()
|
|
mock_task_manager.wait_for_resolution = AsyncMock()
|
|
|
|
# Use a single client mock that handles both get_dashboards and import_dashboard
|
|
mock_client = _make_mock_superset_client()
|
|
mock_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
# First call returns False (fails), second call returns True (retry succeeds)
|
|
mock_engine.transform_zip.side_effect = [False, True]
|
|
|
|
# Mock DB session for Environments and DatabaseMapping
|
|
mock_db = MagicMock()
|
|
mock_db_env = MagicMock()
|
|
mock_db_env.id = 1
|
|
mock_db_env.name = "Source"
|
|
mock_db_env2 = MagicMock()
|
|
mock_db_env2.id = 2
|
|
mock_db_env2.name = "Target"
|
|
# Extend side_effect to cover initial (2) + retry (2) calls to first()
|
|
mock_db.query.return_value.filter.return_value.first.side_effect = [
|
|
mock_db_env, mock_db_env2, mock_db_env, mock_db_env2
|
|
]
|
|
mock_db.query.return_value.filter.return_value.all.return_value = []
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient', return_value=mock_client), \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.dependencies.get_task_manager', return_value=mock_task_manager), \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal', return_value=mock_db):
|
|
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": True,
|
|
"_task_id": "task-retry-1",
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert mock_task_manager.wait_for_resolution.called
|
|
# transform_zip should have been called twice
|
|
assert mock_engine.transform_zip.call_count == 2
|
|
# import_dashboard should have been called once (after retry)
|
|
assert mock_client.import_dashboard.called
|
|
|
|
# ── Edge: password error with yaml path pattern ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_password_error_yaml_pattern(self):
|
|
"""Edge: password error with 'databases/db_name.yaml' pattern."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_task_manager = MagicMock()
|
|
mock_task_manager.get_task.return_value = MagicMock(
|
|
params={"passwords": {"PostgreSQL": "secret123"}}
|
|
)
|
|
mock_task_manager.await_input = AsyncMock()
|
|
mock_task_manager.wait_for_input = AsyncMock()
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[
|
|
RuntimeError("Must provide a password for the database databases/PostgreSQL.yaml"),
|
|
None,
|
|
])
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.dependencies.get_task_manager', return_value=mock_task_manager), \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": False,
|
|
"_task_id": "task-pw-1",
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert len(result["migrated_dashboards"]) == 1
|
|
|
|
# ── Edge: ID mapping sync failure is non-fatal ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_id_mapping_sync_failure(self):
|
|
"""Edge: IdMappingService.sync_environment raises, execution continues."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [_make_dashboard(1, "Dash")]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
mock_db_session = MagicMock()
|
|
mock_sync = MagicMock()
|
|
mock_sync.sync_environment = AsyncMock(side_effect=RuntimeError("Sync failed"))
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=mock_sync), \
|
|
patch('src.plugins.migration.SessionLocal', return_value=mock_db_session):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
# Should not raise — sync failure is caught and logged
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert mock_sync.sync_environment.called, "sync_environment should have been awaited"
|
|
|
|
# ── Regression: post-migration sync does not deadlock (direct await, not runner.run) ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_id_mapping_sync_completes_without_deadlock(self):
|
|
"""Regression: post-migration sync completes via direct await (not runner.run)."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [_make_dashboard(1, "Dash A"), _make_dashboard(2, "Dash B")]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock()
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
mock_db_session = MagicMock()
|
|
mock_sync = MagicMock()
|
|
mock_sync.sync_environment = AsyncMock() # completes without error
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=mock_sync), \
|
|
patch('src.plugins.migration.SessionLocal', return_value=mock_db_session):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1, 2],
|
|
})
|
|
|
|
# Sync must complete and all dashboards must be migrated
|
|
assert result["status"] == "SUCCESS"
|
|
assert len(result["migrated_dashboards"]) == 2
|
|
assert result["failed_dashboards"] == []
|
|
# Verify sync was called with correct params
|
|
mock_sync.sync_environment.assert_called_once()
|
|
call_args = mock_sync.sync_environment.call_args
|
|
assert call_args[1]["incremental"] is True
|
|
|
|
# ── Isolation: one import failure does not abort batch ──
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_import_failure_continues_batch(self):
|
|
"""One dashboard import raises SupersetAPIError; second still migrates."""
|
|
from src.core.utils.network import SupersetAPIError
|
|
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
dashboards = [
|
|
_make_dashboard(1, "Working"),
|
|
_make_dashboard(2, "Broken Import"),
|
|
]
|
|
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, dashboards))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock(
|
|
side_effect=[
|
|
None,
|
|
SupersetAPIError(
|
|
"API error during upload: Dataset boom",
|
|
status_code=422,
|
|
response_body='{"errors":[{"message":"Dataset boom","error_type":"GENERIC_COMMAND_ERROR","extra":{"issue_codes":[{"code":1010}]}}]}',
|
|
errors=[{
|
|
"message": "Dataset boom",
|
|
"error_type": "GENERIC_COMMAND_ERROR",
|
|
"extra": {"issue_codes": [{"code": 1010}]},
|
|
}],
|
|
),
|
|
]
|
|
)
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1, 2],
|
|
})
|
|
|
|
assert result["status"] == "PARTIAL_SUCCESS"
|
|
assert len(result["migrated_dashboards"]) == 1
|
|
assert result["migrated_dashboards"][0]["title"] == "Working"
|
|
assert len(result["failed_dashboards"]) == 1
|
|
failed = result["failed_dashboards"][0]
|
|
assert failed["title"] == "Broken Import"
|
|
assert "Dataset boom" in failed["error"]
|
|
assert failed.get("error_type") == "GENERIC_COMMAND_ERROR"
|
|
assert 1010 in (failed.get("issue_codes") or [])
|
|
|
|
class TestMigrationPluginCompositeKeyFallback:
|
|
"""Composite-key sync fallback on non-password import failures."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_composite_key_target_fallback(self):
|
|
"""Target fallback syncs datasets on the target and retries import once."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
contracts = [{"uuid": "ds-1", "database_uuid": "tgt-db", "catalog": None, "schema": "public", "table_name": "users"}]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[
|
|
RuntimeError("Generic import boom"),
|
|
None,
|
|
])
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
mock_engine.read_dataset_contracts_from_zip.return_value = contracts
|
|
|
|
mock_sync = AsyncMock(return_value={
|
|
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
|
|
})
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": False,
|
|
"sync_dataset_composite_keys": True,
|
|
"composite_key_mutation_server": "target",
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert len(result["migrated_dashboards"]) == 1
|
|
mock_sync.assert_awaited_once_with(mock_tgt_client, contracts)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_composite_key_source_fallback(self, tmp_path):
|
|
"""Source fallback aligns source datasets with target live keys, re-exports and retries."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
tmp_source = tmp_path / "source.zip"
|
|
tmp_transformed = tmp_path / "transformed.zip"
|
|
tmp_source.write_bytes(b"zip")
|
|
|
|
source_contracts = [{"uuid": "ds-1", "database_uuid": "src-db", "catalog": None, "schema": "public", "table_name": "users"}]
|
|
live_target = {"ds-1": {"uuid": "ds-1", "database_uuid": "tgt-db", "catalog": None, "schema": "public", "table_name": "users"}}
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(side_effect=[(b"zip", "meta"), (b"zip2", "meta")])
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[RuntimeError("boom"), None])
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
mock_engine.read_dataset_contracts_from_zip.return_value = source_contracts
|
|
|
|
mock_sync = AsyncMock(return_value={
|
|
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
|
|
})
|
|
mock_read_live = AsyncMock(return_value=live_target)
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
|
|
patch('src.plugins.migration.read_live_dataset_contracts', new=mock_read_live), \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(
|
|
side_effect=[str(tmp_source), str(tmp_transformed)]
|
|
)
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": False,
|
|
"db_mappings": {"src-db": "tgt-db"},
|
|
"sync_dataset_composite_keys": True,
|
|
"composite_key_mutation_server": "source",
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert len(result["migrated_dashboards"]) == 1
|
|
expected_sync_contract = {
|
|
"uuid": "ds-1",
|
|
"catalog": None,
|
|
"schema": "public",
|
|
"table_name": "users",
|
|
"database_uuid": "src-db",
|
|
}
|
|
mock_sync.assert_awaited_once_with(mock_src_client, [expected_sync_contract])
|
|
assert mock_src_client.export_dashboard.await_count == 2
|
|
assert mock_engine.transform_zip.call_count == 2
|
|
assert mock_engine.transform_zip.call_args.args[:2] == (
|
|
str(tmp_source), str(tmp_transformed),
|
|
)
|
|
mock_tgt_client.import_dashboard.assert_awaited_with(
|
|
file_name=str(tmp_transformed), dash_id=1, dash_slug="slug-1"
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_composite_key_source_no_inverse_db_uuid(self, tmp_path):
|
|
"""When the target db uuid has no inverse mapping, the contract syncs without database_uuid."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
tmp_source = tmp_path / "source.zip"
|
|
tmp_transformed = tmp_path / "transformed.zip"
|
|
tmp_source.write_bytes(b"zip")
|
|
|
|
source_contracts = [{"uuid": "ds-1", "database_uuid": "src-db", "catalog": None, "schema": "public", "table_name": "users"}]
|
|
live_target = {"ds-1": {"uuid": "ds-1", "database_uuid": "tgt-db", "catalog": None, "schema": "public", "table_name": "users"}}
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(side_effect=[(b"zip", "meta"), (b"zip2", "meta")])
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[RuntimeError("boom"), None])
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
mock_engine.read_dataset_contracts_from_zip.return_value = source_contracts
|
|
|
|
mock_sync = AsyncMock(return_value={
|
|
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0,
|
|
"errors": [{"uuid": "ds-1", "error": "cannot_derive_source_db_uuid"}],
|
|
})
|
|
mock_read_live = AsyncMock(return_value=live_target)
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
|
|
patch('src.plugins.migration.read_live_dataset_contracts', new=mock_read_live), \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(
|
|
side_effect=[str(tmp_source), str(tmp_transformed)]
|
|
)
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": False,
|
|
"db_mappings": {},
|
|
"sync_dataset_composite_keys": True,
|
|
"composite_key_mutation_server": "source",
|
|
})
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
assert len(result["migrated_dashboards"]) == 1
|
|
sync_contract = mock_sync.await_args.args[1][0]
|
|
assert "database_uuid" not in sync_contract
|
|
assert sync_contract["schema"] == "public"
|
|
assert sync_contract["table_name"] == "users"
|
|
assert any(
|
|
e.get("error") == "cannot_derive_source_db_uuid"
|
|
for e in mock_sync.return_value["errors"]
|
|
)
|
|
assert mock_src_client.export_dashboard.await_count == 2
|
|
assert mock_engine.transform_zip.call_count == 2
|
|
assert mock_engine.transform_zip.call_args.args[:2] == (
|
|
str(tmp_source), str(tmp_transformed),
|
|
)
|
|
mock_tgt_client.import_dashboard.assert_awaited_with(
|
|
file_name=str(tmp_transformed), dash_id=1, dash_slug="slug-1"
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_composite_key_source_skips_sync_when_no_contracts(self, tmp_path):
|
|
"""When the source archive yields no dataset contracts, sync and live reads are skipped entirely."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
tmp_source = tmp_path / "source.zip"
|
|
tmp_transformed = tmp_path / "transformed.zip"
|
|
tmp_source.write_bytes(b"zip")
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(side_effect=[(b"zip", "meta"), (b"zip2", "meta")])
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[RuntimeError("boom"), None])
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
mock_engine.read_dataset_contracts_from_zip.return_value = []
|
|
|
|
mock_sync = AsyncMock(return_value={
|
|
"changed": 0, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
|
|
})
|
|
mock_read_live = AsyncMock(return_value={})
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
|
|
patch('src.plugins.migration.read_live_dataset_contracts', new=mock_read_live), \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(
|
|
side_effect=[str(tmp_source), str(tmp_transformed)]
|
|
)
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": False,
|
|
"db_mappings": {"src-db": "tgt-db"},
|
|
"sync_dataset_composite_keys": True,
|
|
"composite_key_mutation_server": "source",
|
|
})
|
|
|
|
assert result["status"] == "PARTIAL_SUCCESS"
|
|
assert len(result["failed_dashboards"]) == 1
|
|
mock_sync.assert_not_awaited()
|
|
mock_read_live.assert_not_awaited()
|
|
assert mock_src_client.export_dashboard.await_count == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_composite_key_fallback_failure_preserves_entry(self):
|
|
"""When the retry also fails, the original entry is preserved with the sync report attached."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[
|
|
RuntimeError("Generic import boom"),
|
|
RuntimeError("Still broken"),
|
|
])
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
mock_engine.read_dataset_contracts_from_zip.return_value = [
|
|
{"uuid": "ds-1", "database_uuid": "tgt-db", "catalog": None, "schema": "public", "table_name": "users"}
|
|
]
|
|
|
|
report = {"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": []}
|
|
mock_sync = AsyncMock(return_value=report)
|
|
context = _make_logging_task_context()
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": False,
|
|
"sync_dataset_composite_keys": True,
|
|
"composite_key_mutation_server": "target",
|
|
}, context=context)
|
|
|
|
assert result["status"] == "PARTIAL_SUCCESS"
|
|
entry = result["failed_dashboards"][0]
|
|
assert "Generic import boom" in entry["error"]
|
|
assert entry["composite_key_sync_report"] == report
|
|
assert entry["composite_key_mutation_server"] == "target"
|
|
assert entry["composite_key_retried"] is True
|
|
|
|
recovery_calls = [
|
|
call for call in context.logger.error.call_args_list
|
|
if call.args and "Recovery retry import failed" in call.args[0]
|
|
]
|
|
assert recovery_calls
|
|
recovery_metadata = recovery_calls[0].kwargs["extra"]
|
|
assert recovery_metadata["dashboard_id"] == 1
|
|
assert recovery_metadata["recovery_phase"] == "failed"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_composite_key_fallback_logs_full_recovery_timeline(self):
|
|
"""Successful fallback emits task metadata for each user-visible recovery phase."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[RuntimeError("Generic import boom"), None])
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
mock_engine.read_dataset_contracts_from_zip.return_value = [
|
|
{"uuid": "ds-1", "database_uuid": "tgt-db", "catalog": None, "schema": "public", "table_name": "users"}
|
|
]
|
|
report = {"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": []}
|
|
context = _make_logging_task_context()
|
|
|
|
with patch("src.plugins.migration.get_config_manager", return_value=mock_cm), \
|
|
patch("src.plugins.migration.SupersetClient") as MockSC, \
|
|
patch("src.plugins.migration.MigrationEngine", return_value=mock_engine), \
|
|
patch("src.plugins.migration.create_temp_file") as mock_ctf, \
|
|
patch("src.plugins.migration.sync_dataset_composite_keys", new=AsyncMock(return_value=report)), \
|
|
patch("src.plugins.migration.IdMappingService", return_value=_make_mock_mapping_service()), \
|
|
patch("src.plugins.migration.SessionLocal"):
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": False,
|
|
"sync_dataset_composite_keys": True,
|
|
"composite_key_mutation_server": "target",
|
|
}, context=context)
|
|
|
|
assert result["status"] == "SUCCESS"
|
|
warning_metadata = context.logger.warning.call_args.kwargs["extra"]
|
|
assert warning_metadata["recovery_phase"] == "started"
|
|
assert warning_metadata["attempt"] == "initial"
|
|
metadata_calls = [
|
|
call.kwargs["extra"]
|
|
for call in context.logger.info.call_args_list
|
|
if "extra" in call.kwargs and "recovery_phase" in call.kwargs["extra"]
|
|
]
|
|
assert [metadata["recovery_phase"] for metadata in metadata_calls] == [
|
|
"sync", "retry_import", "completed"
|
|
]
|
|
assert metadata_calls[1]["sync_changed"] == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_composite_key_never_syncs_password_failure(self):
|
|
"""Password-required failures never trigger the composite-key fallback."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock(side_effect=RuntimeError(
|
|
"Must provide a password for the database PostgreSQL"
|
|
))
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
mock_sync = AsyncMock(return_value={
|
|
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
|
|
})
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": False,
|
|
"sync_dataset_composite_keys": True,
|
|
})
|
|
|
|
mock_sync.assert_not_awaited()
|
|
assert len(result["failed_dashboards"]) == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_composite_key_disabled_skips_fallback(self):
|
|
"""When sync_dataset_composite_keys is False, non-password failures keep the legacy entry."""
|
|
plugin = MigrationPlugin()
|
|
src_env = _make_env("env-1", "Source")
|
|
tgt_env = _make_env("env-2", "Target")
|
|
mock_cm = MagicMock()
|
|
mock_cm.get_environments.return_value = [src_env, tgt_env]
|
|
|
|
mock_src_client = _make_mock_superset_client()
|
|
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
|
|
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
|
|
mock_tgt_client = _make_mock_superset_client()
|
|
mock_tgt_client.import_dashboard = AsyncMock(side_effect=RuntimeError("Generic import boom"))
|
|
|
|
mock_engine = MagicMock()
|
|
mock_engine.transform_zip.return_value = True
|
|
|
|
mock_sync = AsyncMock(return_value={
|
|
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
|
|
})
|
|
|
|
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
|
|
patch('src.plugins.migration.SupersetClient') as MockSC, \
|
|
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
|
|
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
|
|
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
|
|
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
|
|
patch('src.plugins.migration.SessionLocal'):
|
|
|
|
MockSC.side_effect = [mock_src_client, mock_tgt_client]
|
|
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
|
|
|
|
result = await plugin.execute({
|
|
"source_env_id": "env-1",
|
|
"target_env_id": "env-2",
|
|
"selected_ids": [1],
|
|
"replace_db_config": False,
|
|
"sync_dataset_composite_keys": False,
|
|
})
|
|
|
|
mock_sync.assert_not_awaited()
|
|
entry = result["failed_dashboards"][0]
|
|
assert "composite_key_retried" not in entry
|
|
# #endregion Test.MigrationPlugin
|
|
# #region Test.Migration.RecoveryLogs [C:3] [TYPE Module] [SEMANTICS test,migration,recovery,logging]
|
|
# @BRIEF Verify migration recovery emits structured, user-facing phase events.
|
|
# @RELATION BINDS_TO -> [Plugin.Migration.Execute]
|
|
# @TEST_CONTRACT: [InitialImportFailure + RecoveryResult] -> [RecoveryPhaseLogs]
|
|
# @TEST_SCENARIO: recovery_success -> Initial failure is followed by sync, retry, and success events.
|
|
# @TEST_EDGE: external_fail -> Recovery failure remains visible with dashboard context.
|
|
# @TEST_INVARIANT: Per-dashboard export/transform/import failure never aborts the batch -> VERIFIED_BY: [recovery_success, external_fail]
|
|
# #endregion Test.Migration.RecoveryLogs
|