busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

chore: checkpoint working tree onto master

Browse Source 
Carried over from 042-dashboard-scenario-registry:
- dashboard/migration backend changes + tests (dataset_key_sync)
- specs updates; drop generated doxygen artifacts
- research notes, integration artifacts, session log
This commit is contained in:
busya
2026-08-18 12:20:42 +03:00
parent 604b3dc706
commit 977f3d6d75
25 changed files with 3318 additions and 79596 deletions
Download Patch File Download Diff File Expand all files Collapse all files

421
backend/tests/plugins/test_migration_plugin.py
View File

@@ -109,6 +109,21 @@ class TestMigrationPluginGetSchema:
assert schema["properties"]["from_env"]["enum"] == ["dev", "prod"]
def test_get_schema_includes_composite_key_fields(self):
"""Composite-key sync fields are exposed with correct types and defaults."""
plugin = MigrationPlugin()
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [_make_env("e1", "Dev"), _make_env("e2", "Prod")]
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm):
schema = plugin.get_schema()
props = schema["properties"]
assert props["sync_dataset_composite_keys"]["type"] == "boolean"
assert props["sync_dataset_composite_keys"]["default"] is True
assert props["composite_key_mutation_server"]["enum"] == ["target", "source"]
assert props["composite_key_mutation_server"]["default"] == "target"
class TestMigrationPluginExecute:
"""Verify MigrationPlugin.execute with various scenarios."""
@@ -917,4 +932,410 @@ class TestMigrationPluginExecute:
assert failed.get("error_type") == "GENERIC_COMMAND_ERROR"
assert 1010 in (failed.get("issue_codes") or [])
class TestMigrationPluginCompositeKeyFallback:
"""Composite-key sync fallback on non-password import failures."""
@pytest.mark.asyncio
async def test_execute_composite_key_target_fallback(self):
"""Target fallback syncs datasets on the target and retries import once."""
plugin = MigrationPlugin()
src_env = _make_env("env-1", "Source")
tgt_env = _make_env("env-2", "Target")
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [src_env, tgt_env]
contracts = [{"uuid": "ds-1", "database_uuid": "tgt-db", "catalog": None, "schema": "public", "table_name": "users"}]
mock_src_client = _make_mock_superset_client()
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
mock_tgt_client = _make_mock_superset_client()
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[
RuntimeError("Generic import boom"),
None,
])
mock_engine = MagicMock()
mock_engine.transform_zip.return_value = True
mock_engine.read_dataset_contracts_from_zip.return_value = contracts
mock_sync = AsyncMock(return_value={
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
})
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
patch('src.plugins.migration.SupersetClient') as MockSC, \
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
patch('src.plugins.migration.SessionLocal'):
MockSC.side_effect = [mock_src_client, mock_tgt_client]
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
result = await plugin.execute({
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1],
"replace_db_config": False,
"sync_dataset_composite_keys": True,
"composite_key_mutation_server": "target",
})
assert result["status"] == "SUCCESS"
assert len(result["migrated_dashboards"]) == 1
mock_sync.assert_awaited_once_with(mock_tgt_client, contracts)
@pytest.mark.asyncio
async def test_execute_composite_key_source_fallback(self, tmp_path):
"""Source fallback aligns source datasets with target live keys, re-exports and retries."""
plugin = MigrationPlugin()
src_env = _make_env("env-1", "Source")
tgt_env = _make_env("env-2", "Target")
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [src_env, tgt_env]
tmp_source = tmp_path / "source.zip"
tmp_transformed = tmp_path / "transformed.zip"
tmp_source.write_bytes(b"zip")
source_contracts = [{"uuid": "ds-1", "database_uuid": "src-db", "catalog": None, "schema": "public", "table_name": "users"}]
live_target = {"ds-1": {"uuid": "ds-1", "database_uuid": "tgt-db", "catalog": None, "schema": "public", "table_name": "users"}}
mock_src_client = _make_mock_superset_client()
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
mock_src_client.export_dashboard = AsyncMock(side_effect=[(b"zip", "meta"), (b"zip2", "meta")])
mock_tgt_client = _make_mock_superset_client()
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[RuntimeError("boom"), None])
mock_engine = MagicMock()
mock_engine.transform_zip.return_value = True
mock_engine.read_dataset_contracts_from_zip.return_value = source_contracts
mock_sync = AsyncMock(return_value={
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
})
mock_read_live = AsyncMock(return_value=live_target)
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
patch('src.plugins.migration.SupersetClient') as MockSC, \
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
patch('src.plugins.migration.read_live_dataset_contracts', new=mock_read_live), \
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
patch('src.plugins.migration.SessionLocal'):
MockSC.side_effect = [mock_src_client, mock_tgt_client]
mock_ctf.return_value.__enter__ = MagicMock(
side_effect=[str(tmp_source), str(tmp_transformed)]
)
result = await plugin.execute({
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1],
"replace_db_config": False,
"db_mappings": {"src-db": "tgt-db"},
"sync_dataset_composite_keys": True,
"composite_key_mutation_server": "source",
})
assert result["status"] == "SUCCESS"
assert len(result["migrated_dashboards"]) == 1
expected_sync_contract = {
"uuid": "ds-1",
"catalog": None,
"schema": "public",
"table_name": "users",
"database_uuid": "src-db",
}
mock_sync.assert_awaited_once_with(mock_src_client, [expected_sync_contract])
assert mock_src_client.export_dashboard.await_count == 2
assert mock_engine.transform_zip.call_count == 2
assert mock_engine.transform_zip.call_args.args[:2] == (
str(tmp_source), str(tmp_transformed),
)
mock_tgt_client.import_dashboard.assert_awaited_with(
file_name=str(tmp_transformed), dash_id=1, dash_slug="slug-1"
)
@pytest.mark.asyncio
async def test_execute_composite_key_source_no_inverse_db_uuid(self, tmp_path):
"""When the target db uuid has no inverse mapping, the contract syncs without database_uuid."""
plugin = MigrationPlugin()
src_env = _make_env("env-1", "Source")
tgt_env = _make_env("env-2", "Target")
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [src_env, tgt_env]
tmp_source = tmp_path / "source.zip"
tmp_transformed = tmp_path / "transformed.zip"
tmp_source.write_bytes(b"zip")
source_contracts = [{"uuid": "ds-1", "database_uuid": "src-db", "catalog": None, "schema": "public", "table_name": "users"}]
live_target = {"ds-1": {"uuid": "ds-1", "database_uuid": "tgt-db", "catalog": None, "schema": "public", "table_name": "users"}}
mock_src_client = _make_mock_superset_client()
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
mock_src_client.export_dashboard = AsyncMock(side_effect=[(b"zip", "meta"), (b"zip2", "meta")])
mock_tgt_client = _make_mock_superset_client()
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[RuntimeError("boom"), None])
mock_engine = MagicMock()
mock_engine.transform_zip.return_value = True
mock_engine.read_dataset_contracts_from_zip.return_value = source_contracts
mock_sync = AsyncMock(return_value={
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0,
"errors": [{"uuid": "ds-1", "error": "cannot_derive_source_db_uuid"}],
})
mock_read_live = AsyncMock(return_value=live_target)
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
patch('src.plugins.migration.SupersetClient') as MockSC, \
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
patch('src.plugins.migration.read_live_dataset_contracts', new=mock_read_live), \
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
patch('src.plugins.migration.SessionLocal'):
MockSC.side_effect = [mock_src_client, mock_tgt_client]
mock_ctf.return_value.__enter__ = MagicMock(
side_effect=[str(tmp_source), str(tmp_transformed)]
)
result = await plugin.execute({
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1],
"replace_db_config": False,
"db_mappings": {},
"sync_dataset_composite_keys": True,
"composite_key_mutation_server": "source",
})
assert result["status"] == "SUCCESS"
assert len(result["migrated_dashboards"]) == 1
sync_contract = mock_sync.await_args.args[1][0]
assert "database_uuid" not in sync_contract
assert sync_contract["schema"] == "public"
assert sync_contract["table_name"] == "users"
assert any(
e.get("error") == "cannot_derive_source_db_uuid"
for e in mock_sync.return_value["errors"]
)
assert mock_src_client.export_dashboard.await_count == 2
assert mock_engine.transform_zip.call_count == 2
assert mock_engine.transform_zip.call_args.args[:2] == (
str(tmp_source), str(tmp_transformed),
)
mock_tgt_client.import_dashboard.assert_awaited_with(
file_name=str(tmp_transformed), dash_id=1, dash_slug="slug-1"
)
@pytest.mark.asyncio
async def test_execute_composite_key_source_skips_sync_when_no_contracts(self, tmp_path):
"""When the source archive yields no dataset contracts, sync and live reads are skipped entirely."""
plugin = MigrationPlugin()
src_env = _make_env("env-1", "Source")
tgt_env = _make_env("env-2", "Target")
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [src_env, tgt_env]
tmp_source = tmp_path / "source.zip"
tmp_transformed = tmp_path / "transformed.zip"
tmp_source.write_bytes(b"zip")
mock_src_client = _make_mock_superset_client()
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
mock_src_client.export_dashboard = AsyncMock(side_effect=[(b"zip", "meta"), (b"zip2", "meta")])
mock_tgt_client = _make_mock_superset_client()
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[RuntimeError("boom"), None])
mock_engine = MagicMock()
mock_engine.transform_zip.return_value = True
mock_engine.read_dataset_contracts_from_zip.return_value = []
mock_sync = AsyncMock(return_value={
"changed": 0, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
})
mock_read_live = AsyncMock(return_value={})
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
patch('src.plugins.migration.SupersetClient') as MockSC, \
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
patch('src.plugins.migration.read_live_dataset_contracts', new=mock_read_live), \
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
patch('src.plugins.migration.SessionLocal'):
MockSC.side_effect = [mock_src_client, mock_tgt_client]
mock_ctf.return_value.__enter__ = MagicMock(
side_effect=[str(tmp_source), str(tmp_transformed)]
)
result = await plugin.execute({
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1],
"replace_db_config": False,
"db_mappings": {"src-db": "tgt-db"},
"sync_dataset_composite_keys": True,
"composite_key_mutation_server": "source",
})
assert result["status"] == "PARTIAL_SUCCESS"
assert len(result["failed_dashboards"]) == 1
mock_sync.assert_not_awaited()
mock_read_live.assert_not_awaited()
assert mock_src_client.export_dashboard.await_count == 1
@pytest.mark.asyncio
async def test_execute_composite_key_fallback_failure_preserves_entry(self):
"""When the retry also fails, the original entry is preserved with the sync report attached."""
plugin = MigrationPlugin()
src_env = _make_env("env-1", "Source")
tgt_env = _make_env("env-2", "Target")
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [src_env, tgt_env]
mock_src_client = _make_mock_superset_client()
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
mock_tgt_client = _make_mock_superset_client()
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[
RuntimeError("Generic import boom"),
RuntimeError("Still broken"),
])
mock_engine = MagicMock()
mock_engine.transform_zip.return_value = True
mock_engine.read_dataset_contracts_from_zip.return_value = [
{"uuid": "ds-1", "database_uuid": "tgt-db", "catalog": None, "schema": "public", "table_name": "users"}
]
report = {"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": []}
mock_sync = AsyncMock(return_value=report)
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
patch('src.plugins.migration.SupersetClient') as MockSC, \
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
patch('src.plugins.migration.SessionLocal'):
MockSC.side_effect = [mock_src_client, mock_tgt_client]
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
result = await plugin.execute({
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1],
"replace_db_config": False,
"sync_dataset_composite_keys": True,
"composite_key_mutation_server": "target",
})
assert result["status"] == "PARTIAL_SUCCESS"
entry = result["failed_dashboards"][0]
assert "Generic import boom" in entry["error"]
assert entry["composite_key_sync_report"] == report
assert entry["composite_key_mutation_server"] == "target"
assert entry["composite_key_retried"] is True
@pytest.mark.asyncio
async def test_execute_composite_key_never_syncs_password_failure(self):
"""Password-required failures never trigger the composite-key fallback."""
plugin = MigrationPlugin()
src_env = _make_env("env-1", "Source")
tgt_env = _make_env("env-2", "Target")
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [src_env, tgt_env]
mock_src_client = _make_mock_superset_client()
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
mock_tgt_client = _make_mock_superset_client()
mock_tgt_client.import_dashboard = AsyncMock(side_effect=RuntimeError(
"Must provide a password for the database PostgreSQL"
))
mock_engine = MagicMock()
mock_engine.transform_zip.return_value = True
mock_sync = AsyncMock(return_value={
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
})
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
patch('src.plugins.migration.SupersetClient') as MockSC, \
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
patch('src.plugins.migration.SessionLocal'):
MockSC.side_effect = [mock_src_client, mock_tgt_client]
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
result = await plugin.execute({
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1],
"replace_db_config": False,
"sync_dataset_composite_keys": True,
})
mock_sync.assert_not_awaited()
assert len(result["failed_dashboards"]) == 1
@pytest.mark.asyncio
async def test_execute_composite_key_disabled_skips_fallback(self):
"""When sync_dataset_composite_keys is False, non-password failures keep the legacy entry."""
plugin = MigrationPlugin()
src_env = _make_env("env-1", "Source")
tgt_env = _make_env("env-2", "Target")
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [src_env, tgt_env]
mock_src_client = _make_mock_superset_client()
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
mock_tgt_client = _make_mock_superset_client()
mock_tgt_client.import_dashboard = AsyncMock(side_effect=RuntimeError("Generic import boom"))
mock_engine = MagicMock()
mock_engine.transform_zip.return_value = True
mock_sync = AsyncMock(return_value={
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
})
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
patch('src.plugins.migration.SupersetClient') as MockSC, \
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
patch('src.plugins.migration.SessionLocal'):
MockSC.side_effect = [mock_src_client, mock_tgt_client]
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
result = await plugin.execute({
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1],
"replace_db_config": False,
"sync_dataset_composite_keys": False,
})
mock_sync.assert_not_awaited()
entry = result["failed_dashboards"][0]
assert "composite_key_retried" not in entry
# #endregion Test.MigrationPlugin