busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

chore: checkpoint working tree onto master

Browse Source 
Carried over from 042-dashboard-scenario-registry:
- dashboard/migration backend changes + tests (dataset_key_sync)
- specs updates; drop generated doxygen artifacts
- research notes, integration artifacts, session log
This commit is contained in:
busya
2026-08-18 12:20:42 +03:00
parent 604b3dc706
commit 977f3d6d75
25 changed files with 3318 additions and 79596 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.kilo/agent-manager.json
View File

@@ -144,7 +144,7 @@
},
"tabOrder": {
"local": [
"pending:377ef4f9-0722-4904-97ca-3f3e2e7401ec"
"pending:8712a5da-7d8c-45d8-aa58-043621f160c7"
]
},
"worktreeOrder": [

12
backend/src/api/auth.py
View File

@@ -101,6 +101,18 @@ async def read_users_me(current_user: UserSchema = Depends(get_current_user)):
# #endregion Api.Auth.Me
# #region Api.Auth.AdfsConfiguration [C:2] [TYPE Function] [SEMANTICS api,auth,adfs]
# @ingroup Auth
# @BRIEF Report whether ADFS SSO is available.
# @POST Returns a public configuration flag without exposing credentials.
@router.get("/adfs-configured")
async def adfs_configured():
return {"configured": is_adfs_configured()}
# #endregion Api.Auth.AdfsConfiguration
# #region Api.Auth.SessionPolicy [C:3] [TYPE Function] [SEMANTICS api,auth,session,timeout]
# @ingroup Auth
# @BRIEF Return the idle-deadline for the current logical session.

91
backend/src/api/routes/__tests__/test_dashboards.py
View File

@@ -206,11 +206,19 @@ def test_get_dashboards_invalid_pagination(mock_deps):
# @BRIEF Validate dashboard detail returns charts and datasets for an existing dashboard.
# @TEST: GET /api/dashboards/{id} returns dashboard detail with charts and datasets
def test_get_dashboard_detail_success(mock_deps):
with patch("src.api.routes.dashboards._detail_routes.SupersetClient") as mock_client_cls:
with (
patch("src.api.routes.dashboards._detail_routes.SupersetClient") as mock_client_cls,
patch(
"src.api.routes.dashboards._detail_routes.get_superset_client",
new_callable=AsyncMock,
) as mock_get_client,
):
mock_env = MagicMock()
mock_env.id = "prod"
mock_deps["config"].get_environments.return_value = [mock_env]
mock_client = MagicMock()
mock_client.get_dashboards_page = AsyncMock(return_value=(0, []))
mock_client.get_dashboard_detail = AsyncMock()
mock_client.get_dashboard_detail.return_value = {
"id": 42,
"title": "Revenue Dashboard",
@@ -243,6 +251,7 @@ def test_get_dashboard_detail_success(mock_deps):
"dataset_count": 1,
}
mock_client_cls.return_value = mock_client
mock_get_client.return_value = MagicMock()
response = client.get("/api/dashboards/42?env_id=prod")
assert response.status_code == 200
payload = response.json()
@@ -260,68 +269,6 @@ def test_get_dashboard_detail_env_not_found(mock_deps):
assert response.status_code == 404
assert "Environment not found" in response.json()["detail"]
# #endregion Test.Tests.TestGetDashboardDetailEnvNotFound
# #region Test.Tests.TestMigrateDashboardsSuccess [TYPE Function]
# @RELATION BINDS_TO -> Test.Tests.DashboardsApiTests
# @TEST: POST /api/dashboards/migrate creates migration task
# @PRE Valid source_env_id, target_env_id, dashboard_ids
# @BRIEF Validate dashboard migration request creates an async task and returns its identifier.
# @POST Returns task_id and create_task was called
def test_migrate_dashboards_success(mock_deps):
mock_source = MagicMock()
mock_source.id = "source"
mock_target = MagicMock()
mock_target.id = "target"
mock_deps["config"].get_environments.return_value = [mock_source, mock_target]
mock_task = MagicMock()
mock_task.id = "task-migrate-123"
mock_deps["task"].create_task = AsyncMock(return_value=mock_task)
response = client.post(
"/api/dashboards/migrate",
json={
"source_env_id": "source",
"target_env_id": "target",
"dashboard_ids": [1, 2, 3],
"db_mappings": {"old_db": "new_db"},
},
)
assert response.status_code == 200
data = response.json()
assert "task_id" in data
# @POST/@SIDE_EFFECT: create_task was called
mock_deps["task"].create_task.assert_called_once()
# #endregion Test.Tests.TestMigrateDashboardsSuccess
# #region Test.Tests.TestMigrateDashboardsNoIds [TYPE Function]
# @RELATION BINDS_TO -> Test.Tests.DashboardsApiTests
# @TEST: POST /api/dashboards/migrate returns 400 for empty dashboard_ids
# @PRE dashboard_ids is empty
# @BRIEF Validate dashboard migration rejects empty dashboard identifier lists.
# @POST Returns 400 error
def test_migrate_dashboards_no_ids(mock_deps):
response = client.post(
"/api/dashboards/migrate",
json={
"source_env_id": "source",
"target_env_id": "target",
"dashboard_ids": [],
},
)
assert response.status_code == 400
assert "At least one dashboard ID must be provided" in response.json()["detail"]
# #endregion Test.Tests.TestMigrateDashboardsNoIds
# #region Test.Tests.TestMigrateDashboardsEnvNotFound [TYPE Function]
# @RELATION BINDS_TO -> Test.Tests.DashboardsApiTests
# @BRIEF Validate migration creation returns 404 when the source environment cannot be resolved.
# @PRE source_env_id and target_env_id are valid environment IDs
def test_migrate_dashboards_env_not_found(mock_deps):
"""@PRE: source_env_id and target_env_id are valid environment IDs."""
mock_deps["config"].get_environments.return_value = []
response = client.post(
"/api/dashboards/migrate",
json={"source_env_id": "ghost", "target_env_id": "t", "dashboard_ids": [1]},
)
assert response.status_code == 404
assert "Source environment not found" in response.json()["detail"]
# #endregion Test.Tests.TestMigrateDashboardsEnvNotFound
# #region Test.Tests.TestBackupDashboardsSuccess [TYPE Function]
# @RELATION BINDS_TO -> Test.Tests.DashboardsApiTests
# @TEST: POST /api/dashboards/backup creates backup task
@@ -448,11 +395,20 @@ def test_get_dashboard_tasks_history_filters_success(mock_deps):
# @BRIEF Validate dashboard thumbnail endpoint proxies image bytes and content type from Superset.
# @TEST: GET /api/dashboards/{id}/thumbnail proxies image bytes from Superset
def test_get_dashboard_thumbnail_success(mock_deps):
with patch("src.api.routes.dashboards._detail_routes.SupersetClient") as mock_client_cls:
with (
patch("src.api.routes.dashboards._detail_routes.SupersetClient") as mock_client_cls,
patch(
"src.api.routes.dashboards._detail_routes.get_superset_client",
new_callable=AsyncMock,
) as mock_get_client,
):
mock_env = MagicMock()
mock_env.id = "prod"
mock_deps["config"].get_environments.return_value = [mock_env]
mock_client = MagicMock()
mock_client.get_dashboards_page = AsyncMock(return_value=(0, []))
mock_client_cls.return_value = mock_client
async_client = MagicMock()
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.content = b"fake-image-bytes"
@@ -461,8 +417,8 @@ def test_get_dashboard_thumbnail_success(mock_deps):
if method == "POST":
return {"image_url": "/api/v1/dashboard/42/screenshot/abc123/"}
return mock_response
mock_client.network.request.side_effect = _network_request
mock_client_cls.return_value = mock_client
async_client.request = AsyncMock(side_effect=_network_request)
mock_get_client.return_value = async_client
response = client.get("/api/dashboards/42/thumbnail?env_id=prod")
assert response.status_code == 200
assert response.content == b"fake-image-bytes"
@@ -758,7 +714,7 @@ def test_get_dashboards_profile_filter_matches_display_alias_without_detail_fano
)
with (
patch("src.api.routes.dashboards._listing_routes.ProfileService") as profile_service_cls,
patch("src.api.routes.dashboards._projection.SupersetClient") as superset_client_cls,
patch("src.api.routes.dashboards._projection.AsyncSupersetClient") as superset_client_cls,
patch(
"src.api.routes.dashboards._projection.SupersetAccountLookupAdapter"
) as lookup_adapter_cls,
@@ -775,6 +731,7 @@ def test_get_dashboards_profile_filter_matches_display_alias_without_detail_fano
superset_client = MagicMock()
superset_client_cls.return_value = superset_client
lookup_adapter = MagicMock()
lookup_adapter.get_users_page = AsyncMock()
lookup_adapter.get_users_page.return_value = {
"items": [
{

84
backend/src/api/routes/dashboards/_action_routes.py
View File

@@ -1,6 +1,6 @@
# #region Api.ActionRoutes.DashboardActionRoutes [C:2] [TYPE Module] [SEMANTICS fastapi, dashboard, api, backup]
# @defgroup Api Module group.
# @BRIEF Dashboard action route handlers — migrate, backup.
# @BRIEF Dashboard action route handlers — backup.
# @LAYER API
# @RELATION DEPENDS_ON -> [EXT:frontend:DashboardRouter]]
# @RELATION DEPENDS_ON -> [Api.Schemas.DashboardSchemas]
@@ -17,80 +17,12 @@ from src.dependencies import (
from ._router import router
from ._schemas import (
BackupRequest,
MigrateRequest,
TaskResponse,
)
# #region Api.ActionRoutes.MigrateDashboards [C:2] [TYPE Function]
# @ingroup Api
# @BRIEF Trigger bulk migration of dashboards from source to target environment
# @PRE User has permission plugin:migration:execute
# @PRE source_env_id and target_env_id are valid environment IDs
# @PRE dashboard_ids is a non-empty list
# @POST Returns task_id for tracking migration progress
# @POST Task is created and queued for execution
# @RELATION DISPATCHES -> [EXT:method:MigrationPlugin:execute]
# @RELATION CALLS -> [Core.Manager.TaskManager]
@router.post("/migrate", response_model=TaskResponse)
async def migrate_dashboards(
request: MigrateRequest,
config_manager=Depends(get_config_manager),
task_manager=Depends(get_task_manager),
current_user=Depends(has_permission("plugin:migration", "EXECUTE")),
):
with belief_scope(
"migrate_dashboards",
f"source={request.source_env_id}, target={request.target_env_id}, count={len(request.dashboard_ids)}",
):
if not request.dashboard_ids:
logger.explore("No dashboard IDs provided for migrate_dashboards", error="At least one dashboard ID must be provided")
raise HTTPException(
status_code=400, detail="At least one dashboard ID must be provided"
)
environments = config_manager.get_environments()
source_env = next(
(e for e in environments if e.id == request.source_env_id), None
)
target_env = next(
(e for e in environments if e.id == request.target_env_id), None
)
if not source_env:
logger.explore("Source environment not found for migrate_dashboards", payload={"source_env_id": request.source_env_id}, error=f"Source environment not found: {request.source_env_id}")
raise HTTPException(status_code=404, detail="Source environment not found")
if not target_env:
logger.explore("Target environment not found for migrate_dashboards", payload={"target_env_id": request.target_env_id}, error=f"Target environment not found: {request.target_env_id}")
raise HTTPException(status_code=404, detail="Target environment not found")
try:
task_params = {
"source_env_id": request.source_env_id,
"target_env_id": request.target_env_id,
"selected_ids": request.dashboard_ids,
"replace_db_config": request.replace_db_config,
"db_mappings": request.db_mappings or {},
}
task_obj = await task_manager.create_task(
plugin_id="superset-migration",
params=task_params,
user_id=current_user.id,
)
logger.reflect("Migration task created", payload={"task_id": str(task_obj.id), "dashboard_count": len(request.dashboard_ids)})
return TaskResponse(task_id=str(task_obj.id))
except Exception as e:
logger.explore("Failed to create migration task", error=str(e))
raise HTTPException(
status_code=503, detail=f"Failed to create migration task: {e!s}"
)
# #endregion Api.ActionRoutes.MigrateDashboards
_config_manager_dependency = Depends(get_config_manager)
_task_manager_dependency = Depends(get_task_manager)
_backup_permission_dependency = Depends(has_permission("plugin:backup", "EXECUTE"))
# #region Api.ActionRoutes.BackupDashboards [C:2] [TYPE Function]
@@ -107,9 +39,9 @@ async def migrate_dashboards(
@router.post("/backup", response_model=TaskResponse)
async def backup_dashboards(
request: BackupRequest,
config_manager=Depends(get_config_manager),
task_manager=Depends(get_task_manager),
current_user=Depends(has_permission("plugin:backup", "EXECUTE")),
config_manager=_config_manager_dependency,
task_manager=_task_manager_dependency,
current_user=_backup_permission_dependency,
):
with belief_scope(
"backup_dashboards",
@@ -149,7 +81,7 @@ async def backup_dashboards(
logger.explore("Failed to create backup task", error=str(e))
raise HTTPException(
status_code=503, detail=f"Failed to create backup task: {e!s}"
)
) from e
# #endregion Api.ActionRoutes.BackupDashboards

17
backend/src/api/routes/dashboards/_schemas.py
View File

@@ -177,23 +177,6 @@ class DatabaseMappingsResponse(BaseModel):
# #endregion Api.Schemas.DatabaseMappingsResponse
# #region Api.Schemas.MigrateRequest [C:1] [TYPE DataClass]
# @BRIEF DTO for dashboard migration requests.
class MigrateRequest(BaseModel):
source_env_id: str = Field(..., description="Source environment ID")
target_env_id: str = Field(..., description="Target environment ID")
dashboard_ids: list[int] = Field(
..., description="List of dashboard IDs to migrate"
)
db_mappings: dict[str, str] | None = Field(
None, description="Database mappings for migration"
)
replace_db_config: bool = Field(False, description="Replace database configuration")
# #endregion Api.Schemas.MigrateRequest
# #region Api.Schemas.TaskResponse [C:1] [TYPE DataClass]
# @BRIEF DTO for async task ID return.
class TaskResponse(BaseModel):

3
backend/src/core/migration/__init__.py
View File

@@ -2,6 +2,7 @@
# @defgroup Migration Module group.
from .archive_parser import MigrationArchiveParser
from .dataset_key_sync import read_live_dataset_contracts, sync_dataset_composite_keys
from .dry_run_orchestrator import MigrationDryRunService
from .import_errors import (
build_failed_dashboard_entry,
@@ -17,6 +18,8 @@ __all__ = [
"format_superset_import_error",
"normalize_superset_passwords",
"password_paths_to_display_names",
"read_live_dataset_contracts",
"sync_dataset_composite_keys",
]
# #endregion Core.Init.MigrationPackage

209
backend/src/core/migration/dataset_key_sync.py Normal file
View File

@@ -0,0 +1,209 @@
# #region Core.DatasetKeySync.DatasetKeySyncModule [C:4] [TYPE Module] [SEMANTICS migration,dataset,composite-key,sync]
# @defgroup Migration Module group.
# @BRIEF Sync dataset composite keys (database, catalog, schema, table_name) against live Superset catalog state.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [Core.Logger.LoggerModule]
# @RELATION DEPENDS_ON -> [Core.SupersetClient.SupersetClientModule]
# @RATIONALE Composite-key drift (database, catalog, schema, table_name) blocks non-password dashboard imports
# because Superset matches datasets by their composite key. This module isolates that reconciliation
# from the migration plugin so the plugin stays under the module size limit and the sync logic is
# independently testable.
# @REJECTED Embedding dataset key sync in the migration plugin was rejected — plugin orchestration and dataset
# reconciliation have distinct failure modes, and importing MigrationEngine here would create a package
# cycle. A dedicated module keeps module boundaries clean.
from typing import Any
from src.core.logger import belief_scope, logger
_COMPOSITE_KEY_FIELDS = ("catalog", "schema")
# #region Core.DatasetKeySync.ResolveSyncPayload [C:3] [TYPE Function] [SEMANTICS migration,dataset,composite-key,payload]
# @ingroup Migration
# @BRIEF Build the dataset update payload for one contract against its live dataset.
# @PRE contract carries a truthy uuid and dataset is the matching live dataset row.
# @POST Returns {database_id?, catalog?, schema?, table_name?}; catalog/schema are present whenever the
# contract carries the key (None resets the live value), table_name is present only when not None.
# @SIDE_EFFECT Resolves the database_uuid via the Superset client when absent from the cache.
# @RELATION DEPENDS_ON -> [Core.SupersetClient.SupersetClientModule]
async def _resolve_sync_payload(
client, contract: dict, dataset: dict, db_id_cache: dict[str, Any]
) -> dict:
dataset_id = dataset.get("id")
if not dataset_id:
raise RuntimeError("dataset_has_no_id")
payload: dict[str, Any] = {}
database_uuid = contract.get("database_uuid")
if isinstance(database_uuid, str) and database_uuid:
database_id = db_id_cache.get(database_uuid)
if database_id is None:
db = await client.get_database_by_uuid(database_uuid)
if not (isinstance(db, dict) and db.get("id")):
raise RuntimeError(f"database_uuid not found: {database_uuid}")
database_id = db["id"]
db_id_cache[database_uuid] = database_id
payload["database_id"] = database_id
for field in _COMPOSITE_KEY_FIELDS:
if field in contract:
payload[field] = contract.get(field)
if contract.get("table_name") is not None:
payload["table_name"] = contract["table_name"]
return payload
# #endregion Core.DatasetKeySync.ResolveSyncPayload
# #region Core.DatasetKeySync.DatasetDiffers [C:2] [TYPE Function] [SEMANTICS migration,dataset,composite-key,diff]
# @ingroup Migration
# @BRIEF Compare a desired payload against the live dataset's current composite keys.
# @POST Returns True when at least one payload key differs from the live value.
def _dataset_differs(dataset: dict, payload: dict) -> bool:
db_ref = dataset.get("database")
current_database_id = (
db_ref.get("id")
if isinstance(db_ref, dict)
else dataset.get("database_id")
)
for key, value in payload.items():
current = current_database_id if key == "database_id" else dataset.get(key)
if value != current:
return True
return False
# #endregion Core.DatasetKeySync.DatasetDiffers
# #region Core.DatasetKeySync.SyncDatasetCompositeKeys [C:4] [TYPE Function] [SEMANTICS migration,dataset,composite-key,sync]
# @ingroup Migration
# @BRIEF Align live datasets with the desired composite keys and report per-dataset outcomes.
# @PRE client exposes get_datasets, get_database_by_uuid, and update_dataset.
# @POST Returns an aggregate report {changed, unchanged, skipped_missing, failed, errors} with isolated
# per-dataset failures — one failing dataset never aborts the remaining contracts.
# @SIDE_EFFECT Calls Superset APIs: list datasets, resolve database UUIDs, update changed datasets.
# @DATA_CONTRACT Input[list[dict]{uuid, database_uuid?, catalog?, schema?, table_name?}] -> Output[aggregate report dict]
async def sync_dataset_composite_keys(client, contracts: list[dict]) -> dict:
with belief_scope("Core.DatasetKeySync.sync_dataset_composite_keys"):
report: dict[str, Any] = {
"changed": 0,
"unchanged": 0,
"skipped_missing": 0,
"failed": 0,
"errors": [],
}
try:
_, datasets = await client.get_datasets()
except Exception as exc:
logger.explore(
"Fetching live datasets failed; marking all contracts as failed",
payload={"contract_count": len(contracts)},
error=str(exc),
)
report["failed"] = len(contracts)
for contract in contracts:
report["errors"].append(
{"uuid": contract.get("uuid"), "error": str(exc)}
)
return report
live_index = {str(ds.get("uuid")): ds for ds in datasets if ds.get("uuid")}
db_id_cache: dict[str, Any] = {}
for contract in contracts:
uuid = contract.get("uuid")
if not uuid:
report["failed"] += 1
report["errors"].append({"uuid": uuid, "error": "missing_uuid"})
continue
dataset = live_index.get(str(uuid))
if dataset is None:
report["skipped_missing"] += 1
continue
try:
payload = await _resolve_sync_payload(
client, contract, dataset, db_id_cache
)
if not _dataset_differs(dataset, payload):
report["unchanged"] += 1
continue
await client.update_dataset(
dataset.get("id"), payload, override_columns=False
)
report["changed"] += 1
except Exception as exc:
report["failed"] += 1
report["errors"].append({"uuid": uuid, "error": str(exc)})
return report
# #endregion Core.DatasetKeySync.SyncDatasetCompositeKeys
# #region Core.DatasetKeySync.ResolveDatabaseUuid [C:2] [TYPE Function] [SEMANTICS migration,dataset,database,uuid]
# @ingroup Migration
# @BRIEF Resolve the database uuid from a nested database object or a fallback id lookup.
# @POST Returns the nested uuid when present; otherwise resolves via get_database when only the id is known.
async def _resolve_database_uuid(client, db_ref: Any) -> Any:
if not isinstance(db_ref, dict):
return None
nested_uuid = db_ref.get("uuid")
if nested_uuid:
return nested_uuid
if db_ref.get("id"):
try:
db_detail = await client.get_database(db_ref["id"])
return db_detail.get("uuid") if isinstance(db_detail, dict) else None
except Exception:
return None
return None
# #endregion Core.DatasetKeySync.ResolveDatabaseUuid
# #region Core.DatasetKeySync.ReadLiveDatasetContracts [C:3] [TYPE Function] [SEMANTICS migration,dataset,composite-key,contract]
# @ingroup Migration
# @BRIEF Read live dataset composite keys from a Superset client keyed by dataset uuid.
# @PRE client exposes get_datasets and get_database.
# @POST Returns {uuid: {uuid, database_uuid?, catalog?, schema?, table_name?}}; database_uuid is resolved
# from the nested database object or via get_database when only the database id is known.
async def read_live_dataset_contracts(
client, dataset_uuids: set[str] | list[str] | None = None
) -> dict[str, dict]:
with belief_scope("Core.DatasetKeySync.read_live_dataset_contracts"):
try:
_, datasets = await client.get_datasets(
query={
"columns": [
"id",
"uuid",
"table_name",
"schema",
"catalog",
"database",
]
}
)
except Exception:
_, datasets = await client.get_datasets(query=None)
allowed = None
if dataset_uuids is not None:
allowed = {
str(u)
for u in (dataset_uuids if isinstance(dataset_uuids, (set, list)) else [dataset_uuids])
}
contracts: dict[str, dict] = {}
for dataset in datasets:
ds_uuid = dataset.get("uuid")
if not ds_uuid:
continue
uuid_str = str(ds_uuid)
if allowed is not None and uuid_str not in allowed:
continue
contracts[uuid_str] = {
"uuid": uuid_str,
"database_uuid": await _resolve_database_uuid(
client, dataset.get("database")
),
"catalog": dataset.get("catalog"),
"schema": dataset.get("schema"),
"table_name": dataset.get("table_name"),
}
return contracts
# #endregion Core.DatasetKeySync.ReadLiveDatasetContracts
__all__ = ["read_live_dataset_contracts", "sync_dataset_composite_keys"]
# #endregion Core.DatasetKeySync.DatasetKeySyncModule

47
backend/src/core/migration_engine.py
View File

@@ -253,6 +253,53 @@ class MigrationEngine:
return resources
# #endregion Core.MigrationEngine.ListDatabaseResourcesFromZip
# #region Core.MigrationEngine.ReadDatasetContractsFromZip [C:3] [TYPE Function] [SEMANTICS migration,archive,dataset,contract]
# @ingroup Core
# @BRIEF Read dataset contracts from an export ZIP in-memory without mutating the archive.
# @PRE zip_path points to a readable Superset export ZIP.
# @POST Returns a list of {uuid, database_uuid, catalog, schema, table_name} dicts; skips
# unparseable or empty dataset YAMLs and entries without a truthy uuid.
# @SIDE_EFFECT Reads the ZIP fully into memory; never writes or mutates the archive.
# @RELATION DEPENDS_ON -> [Core.MigrationEngine]
def read_dataset_contracts_from_zip(self, zip_path: str) -> list[dict]:
with belief_scope("MigrationEngine.read_dataset_contracts_from_zip"):
try:
contracts: list[dict] = []
with zipfile.ZipFile(zip_path, "r") as zf:
for name in sorted(zf.namelist()):
if not name.endswith(".yaml"):
continue
if "datasets" not in name.split("/"):
continue
try:
data = yaml.safe_load(zf.read(name)) or {}
except Exception:
continue
if not isinstance(data, dict) or not data.get("uuid"):
continue
contracts.append(
{
"uuid": data.get("uuid"),
"database_uuid": data.get("database_uuid"),
"catalog": data.get("catalog"),
"schema": data.get("schema"),
"table_name": data.get("table_name"),
}
)
logger.reflect(
"Dataset contracts read from archive",
payload={"count": len(contracts), "zip_path": zip_path},
)
return contracts
except Exception as exc:
logger.explore(
"Could not read dataset contracts from archive",
payload={"zip_path": zip_path},
error=str(exc),
)
return []
# #endregion Core.MigrationEngine.ReadDatasetContractsFromZip
# #region Core.MigrationEngine.CollectDatasetDatabaseUuids [C:2] [TYPE Function] [SEMANTICS migration,archive,database,mapping]
# @BRIEF Read source database UUIDs referenced by the datasets in an export archive.
# @PRE Each path is a readable Superset dataset YAML file.

4
backend/src/models/dashboard.py
View File

@@ -5,6 +5,8 @@
# @RELATION CALLED_BY -> [Api.Migration.MigrationApi]
from typing import Literal
from pydantic import BaseModel, Field
@@ -25,6 +27,8 @@ class DashboardSelection(BaseModel):
target_env_id: str
replace_db_config: bool = False
fix_cross_filters: bool = True
composite_key_mutation_server: Literal['target', 'source'] = 'target'
sync_dataset_composite_keys: bool = True
# #endregion Models.Dashboard.DashboardSelection

161
backend/src/plugins/migration.py
View File

@@ -23,12 +23,17 @@
# the plugin wrapper) was rejected — plugin isolation provides cancellation, progress
# reporting, and logging that raw client calls lack.
from pathlib import Path
import re
from typing import Any
from ..core.database import SessionLocal
from ..core.logger import belief_scope, logger as app_logger
from ..core.mapping_service import IdMappingService
from ..core.migration.dataset_key_sync import (
read_live_dataset_contracts,
sync_dataset_composite_keys,
)
from ..core.migration.import_errors import (
build_failed_dashboard_entry,
format_superset_import_error,
@@ -145,6 +150,102 @@ async def _lineage_refresh_after_deploy(environment_id: str, client, db_session)
# #endregion Plugin.Migration.LineageRefreshAfterDeploy
# #region Plugin.Migration.AttemptCompositeKeyFallback [C:4] [TYPE Function] [SEMANTICS migration,dataset,composite-key,fallback,retry]
# @ingroup Plugin
# @BRIEF One-shot retry after a non-password import failure: sync dataset composite keys on the selected server and re-import once.
# @PRE engine exposes read_dataset_contracts_from_zip/transform_zip; from_c/to_c expose Superset APIs; create_temp_file is available.
# @POST Returns (report, imported, retry_error). Never raises — all exceptions are captured into retry_error.
# @SIDE_EFFECT Mutates source (sync + re-export + re-transform) or target (sync) Superset state and performs exactly one retry import.
# @RATIONALE Non-password import failures are frequently composite-key drift: Superset cannot bind datasets to the mapped database.
# Aligning keys before a single retry resolves the drift without user interaction while never looping beyond one retry.
# @REJECTED Multiple retry loops were rejected — an import that still fails after composite-key sync needs operator attention,
# and unbounded retries would mask genuinely broken archives.
async def _attempt_composite_key_fallback(
*,
engine,
from_c,
to_c,
dash_id,
dash_slug,
db_mapping,
target_env_id: str | None,
fix_cross_filters: bool,
server: str = "target",
source_zip: str,
transformed_zip: str,
) -> tuple[dict, bool, BaseException | None]:
report: dict = {
"changed": 0,
"unchanged": 0,
"skipped_missing": 0,
"failed": 0,
"errors": [],
}
with belief_scope("MigrationPlugin._attempt_composite_key_fallback"):
try:
if server == "source":
source_contracts = engine.read_dataset_contracts_from_zip(source_zip)
if not source_contracts:
return report, False, RuntimeError("No dataset contracts found for source sync")
uuids = {c["uuid"] for c in source_contracts}
live_target = await read_live_dataset_contracts(to_c, uuids)
inverse_mapping = {tgt: src for src, tgt in db_mapping.items()}
derived_errors: list[dict] = []
sync_contracts: list[dict] = []
for contract in source_contracts:
live = live_target.get(contract["uuid"])
if not live:
derived_errors.append({"uuid": contract["uuid"], "error": "no_live_target_contract"})
continue
sync_contract: dict = {
"uuid": contract["uuid"],
"catalog": live.get("catalog"),
"schema": live.get("schema"),
"table_name": live.get("table_name"),
}
desired_target_db_uuid = live.get("database_uuid")
source_db_uuid = inverse_mapping.get(desired_target_db_uuid) if desired_target_db_uuid else None
if source_db_uuid:
sync_contract["database_uuid"] = source_db_uuid
else:
derived_errors.append({"uuid": contract["uuid"], "error": "cannot_derive_source_db_uuid"})
sync_contracts.append(sync_contract)
if not sync_contracts:
report["errors"].extend(derived_errors)
return report, False, RuntimeError("No source datasets matched target contracts")
report = await sync_dataset_composite_keys(from_c, sync_contracts)
report.setdefault("errors", []).extend(derived_errors)
exported, _ = await from_c.export_dashboard(dash_id)
Path(source_zip).write_bytes(exported)
ok = engine.transform_zip(
str(source_zip),
str(transformed_zip),
db_mapping,
strip_databases=False,
target_env_id=target_env_id,
fix_cross_filters=fix_cross_filters,
)
if not ok:
return report, False, RuntimeError(
"Re-transform failed after source composite-key sync"
)
await to_c.import_dashboard(
file_name=transformed_zip, dash_id=dash_id, dash_slug=dash_slug
)
return report, True, None
contracts = engine.read_dataset_contracts_from_zip(str(transformed_zip))
if not contracts:
return report, False, RuntimeError("No dataset contracts found for target sync")
report = await sync_dataset_composite_keys(to_c, contracts)
await to_c.import_dashboard(
file_name=transformed_zip, dash_id=dash_id, dash_slug=dash_slug
)
return report, True, None
except Exception as exc:
return report, False, exc
# #endregion Plugin.Migration.AttemptCompositeKeyFallback
# #region Plugin.Migration.MigrationPlugin [TYPE Class]
# @defgroup Plugin Module group.
# @BRIEF Implementation of the migration plugin workflow and transformation orchestration.
@@ -263,6 +364,19 @@ class MigrationPlugin(PluginBase):
"title": "Target DB ID",
"description": "The ID of the target database to replace with (if replacing).",
},
"sync_dataset_composite_keys": {
"type": "boolean",
"title": "Sync Dataset Composite Keys",
"description": "On non-password import failure, sync dataset composite keys (database, catalog, schema, table_name) and retry the import once.",
"default": True,
},
"composite_key_mutation_server": {
"type": "string",
"title": "Composite Key Mutation Server",
"description": "Which server to mutate on non-password import failure fallback: target syncs the transformed archive onto the target; source aligns source datasets with the target live keys, re-exports and re-transforms.",
"enum": ["target", "source"],
"default": "target",
},
},
"required": ["from_env", "to_env", "dashboard_regex"],
}
@@ -299,6 +413,14 @@ class MigrationPlugin(PluginBase):
dashboard_regex = params.get("dashboard_regex")
replace_db_config = params.get("replace_db_config", False)
fix_cross_filters = params.get("fix_cross_filters", True)
sync_dataset_composite_keys = params.get(
"sync_dataset_composite_keys", True
)
composite_key_mutation_server = params.get(
"composite_key_mutation_server", "target"
)
if composite_key_mutation_server not in ("target", "source"):
composite_key_mutation_server = "target"
task_id = params.get("_task_id")
from ..dependencies import get_task_manager
@@ -485,6 +607,45 @@ class MigrationPlugin(PluginBase):
superset_log.error(f"Superset import failed for dashboard {dash_id}: {import_exc}")
formatted = format_superset_import_error(import_exc)
if not formatted.get("is_password_required"):
if sync_dataset_composite_keys:
report, imported, retry_error = await _attempt_composite_key_fallback(
engine=engine,
from_c=from_c,
to_c=to_c,
dash_id=dash_id,
dash_slug=dash_slug,
db_mapping=db_mapping,
target_env_id=tgt_env.id if tgt_env else None,
fix_cross_filters=fix_cross_filters,
server=composite_key_mutation_server,
source_zip=str(tmp_zip_path),
transformed_zip=str(tmp_new_zip),
)
if imported:
migration_result["migrated_dashboards"].append({"id": dash_id, "title": title})
app_logger.reflect(
"Composite-key fallback retry import succeeded",
extra={"title": title},
)
continue
if retry_error is not None:
app_logger.explore(
"Composite-key fallback retry import failed; keeping original entry",
extra={"dash_id": dash_id, "title": title},
error=str(retry_error),
)
entry = _failed_entry_with_archive(
dash_id, title, import_exc, phase="import",
formatted=formatted,
task_id=task_id,
zip_path=str(tmp_new_zip),
source_zip_path=str(tmp_zip_path),
)
entry["composite_key_sync_report"] = report
entry["composite_key_mutation_server"] = composite_key_mutation_server
entry["composite_key_retried"] = True
migration_result["failed_dashboards"].append(entry)
continue
migration_result["failed_dashboards"].append(
_failed_entry_with_archive(
dash_id, title, import_exc, phase="import",

11
backend/tests/api/test_auth.py
View File

@@ -181,6 +181,17 @@ class TestLogout:
mock_blacklist.assert_called_once()
class TestAdfsConfigured:
"""GET /api/auth/adfs-configured"""
def test_returns_adfs_configuration_state(self):
with patch("src.api.auth.is_adfs_configured", return_value=False):
client = _make_client()
resp = client.get("/api/auth/adfs-configured")
assert resp.status_code == 200
assert resp.json() == {"configured": False}
class TestLoginAdfs:
"""GET /api/auth/login/adfs"""

156
backend/tests/api/test_dashboard_action_routes.py
View File

@@ -1,9 +1,7 @@
# #region Test.Api.DashboardActionRoutes [C:3] [TYPE Module] [SEMANTICS test,dashboard,action,migration,backup]
# @BRIEF Unit tests for dashboard action routes — migrate and backup.
# #region Test.Api.DashboardActionRoutes [C:3] [TYPE Module] [SEMANTICS test,dashboard,action,backup]
# @BRIEF Unit tests for dashboard action routes — backup.
# @RELATION BINDS_TO -> [Api.ActionRoutes.DashboardActionRoutes]
# @TEST_EDGE: empty_dashboard_ids -> 400
# @TEST_EDGE: source_env_not_found -> 404
# @TEST_EDGE: target_env_not_found -> 404
# @TEST_EDGE: env_not_found_backup -> 404
# @TEST_EDGE: task_creation_fail -> 503
@@ -13,12 +11,11 @@ os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:")
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:")
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-tests")
import sys
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
import sys
from unittest.mock import AsyncMock, MagicMock
import pytest
from fastapi import FastAPI, HTTPException
from fastapi import FastAPI
from fastapi.testclient import TestClient
_src = str(Path(__file__).resolve().parent.parent.parent / "src")
@@ -28,8 +25,8 @@ if _src not in sys.path:
def _make_client(overrides: dict | None = None) -> TestClient:
from src.api.routes.dashboards._action_routes import router
from src.dependencies import get_current_user, has_permission
from src.schemas.auth import User, RoleSchema
from src.dependencies import get_current_user
from src.schemas.auth import RoleSchema, User
app = FastAPI()
app.include_router(router)
@@ -51,145 +48,6 @@ def _make_client(overrides: dict | None = None) -> TestClient:
return TestClient(app)
# ── migrate_dashboards ──
class TestMigrateDashboards:
"""POST /api/dashboards/migrate"""
def _make_env(self, id: str):
env = MagicMock()
env.id = id
return env
def test_migrate_success(self):
"""Happy path: migration task created returns 200 with task_id."""
mock_config = MagicMock()
mock_config.get_environments.return_value = [
self._make_env("src-1"),
self._make_env("tgt-1"),
]
mock_task = MagicMock()
mock_task.id = "task-123"
mock_task_manager = AsyncMock()
mock_task_manager.create_task.return_value = mock_task
from src.dependencies import get_config_manager, get_task_manager
client = _make_client({
get_config_manager: lambda: mock_config,
get_task_manager: lambda: mock_task_manager,
})
resp = client.post("/api/dashboards/migrate", json={
"source_env_id": "src-1",
"target_env_id": "tgt-1",
"dashboard_ids": [1, 2, 3],
})
assert resp.status_code == 200
assert resp.json()["task_id"] == "task-123"
mock_task_manager.create_task.assert_called_once()
def test_migrate_empty_dashboard_ids(self):
"""Empty dashboard_ids returns 400."""
mock_config = MagicMock()
from src.dependencies import get_config_manager, get_task_manager
client = _make_client({
get_config_manager: lambda: mock_config,
get_task_manager: lambda: MagicMock(),
})
resp = client.post("/api/dashboards/migrate", json={
"source_env_id": "src-1",
"target_env_id": "tgt-1",
"dashboard_ids": [],
})
assert resp.status_code == 400
assert "At least one dashboard ID" in resp.text
def test_migrate_source_not_found(self):
"""Non-existent source env returns 404."""
mock_config = MagicMock()
mock_config.get_environments.return_value = [self._make_env("tgt-1")]
from src.dependencies import get_config_manager, get_task_manager
client = _make_client({
get_config_manager: lambda: mock_config,
get_task_manager: lambda: MagicMock(),
})
resp = client.post("/api/dashboards/migrate", json={
"source_env_id": "src-missing",
"target_env_id": "tgt-1",
"dashboard_ids": [1],
})
assert resp.status_code == 404
assert "Source environment not found" in resp.text
def test_migrate_target_not_found(self):
"""Non-existent target env returns 404."""
mock_config = MagicMock()
mock_config.get_environments.return_value = [self._make_env("src-1")]
from src.dependencies import get_config_manager, get_task_manager
client = _make_client({
get_config_manager: lambda: mock_config,
get_task_manager: lambda: MagicMock(),
})
resp = client.post("/api/dashboards/migrate", json={
"source_env_id": "src-1",
"target_env_id": "tgt-missing",
"dashboard_ids": [1],
})
assert resp.status_code == 404
assert "Target environment not found" in resp.text
def test_migrate_task_creation_fail(self):
"""Task creation failure returns 503."""
mock_config = MagicMock()
mock_config.get_environments.return_value = [
self._make_env("src-1"),
self._make_env("tgt-1"),
]
mock_task_manager = AsyncMock()
mock_task_manager.create_task.side_effect = Exception("DB down")
from src.dependencies import get_config_manager, get_task_manager
client = _make_client({
get_config_manager: lambda: mock_config,
get_task_manager: lambda: mock_task_manager,
})
resp = client.post("/api/dashboards/migrate", json={
"source_env_id": "src-1",
"target_env_id": "tgt-1",
"dashboard_ids": [1],
})
assert resp.status_code == 503
def test_migrate_with_db_mappings(self):
"""Migration with replace_db_config and db_mappings."""
mock_config = MagicMock()
mock_config.get_environments.return_value = [
self._make_env("src-1"),
self._make_env("tgt-1"),
]
mock_task = MagicMock()
mock_task.id = "task-dbm"
mock_task_manager = AsyncMock()
mock_task_manager.create_task.return_value = mock_task
from src.dependencies import get_config_manager, get_task_manager
client = _make_client({
get_config_manager: lambda: mock_config,
get_task_manager: lambda: mock_task_manager,
})
resp = client.post("/api/dashboards/migrate", json={
"source_env_id": "src-1",
"target_env_id": "tgt-1",
"dashboard_ids": [1],
"replace_db_config": True,
"db_mappings": {"old_db": "new_db"},
})
assert resp.status_code == 200
# ── backup_dashboards ──
class TestBackupDashboards:

345
backend/tests/core/test_dataset_key_sync.py Normal file
View File

@@ -0,0 +1,345 @@
# #region Test.DatasetKeySync [C:3] [TYPE Module] [SEMANTICS test,migration,dataset,composite-key,sync]
# @BRIEF Unit tests for dataset composite-key sync against a mocked Superset client.
# @RELATION BINDS_TO -> [Core.DatasetKeySync.DatasetKeySyncModule]
# @TEST_EDGE: update_failure_isolated -> One failing update does not abort remaining contracts.
# @TEST_EDGE: unresolvable_database_uuid -> None database lookup surfaces as failed with a database_uuid error.
# @TEST_EDGE: fetch_failure -> get_datasets raising marks every contract as failed.
from pathlib import Path
import pytest
import sys
from unittest.mock import AsyncMock
backend_dir = str(Path(__file__).parent.parent.parent.resolve())
if backend_dir not in sys.path:
sys.path.insert(0, backend_dir)
from src.core.migration.dataset_key_sync import (
read_live_dataset_contracts,
sync_dataset_composite_keys,
)
# #region Test.DatasetKeySync.MockClientFixture [C:2] [TYPE Function]
# @BRIEF Shared AsyncMock Superset client with a default two-dataset catalog.
@pytest.fixture
def mock_client():
client = AsyncMock()
client.get_datasets = AsyncMock(
return_value=(
2,
[
{
"id": 7,
"uuid": "ds-1",
"database": {"id": 1, "uuid": "db-live-1"},
"catalog": None,
"schema": "old",
"table_name": "users",
},
{
"id": 8,
"uuid": "ds-2",
"database": {"id": 2, "uuid": "db-live-2"},
"catalog": "c1",
"schema": "public",
"table_name": "orders",
},
],
)
)
client.get_database_by_uuid = AsyncMock(return_value={"id": 7})
client.update_dataset = AsyncMock(return_value={})
return client
# #endregion Test.DatasetKeySync.MockClientFixture
# #region Test.DatasetKeySync.TestSyncChangesDataset [C:2] [TYPE Function]
# @BRIEF A live dataset with a different database id and schema is updated with the desired composite keys.
@pytest.mark.asyncio
async def test_sync_changes_dataset(mock_client):
contracts = [
{
"uuid": "ds-1",
"database_uuid": "db-src-1",
"catalog": None,
"schema": "public",
"table_name": "users",
}
]
report = await sync_dataset_composite_keys(mock_client, contracts)
assert report == {
"changed": 1,
"unchanged": 0,
"skipped_missing": 0,
"failed": 0,
"errors": [],
}
mock_client.update_dataset.assert_awaited_once_with(
7,
{"database_id": 7, "catalog": None, "schema": "public", "table_name": "users"},
override_columns=False,
)
# #endregion Test.DatasetKeySync.TestSyncChangesDataset
# #region Test.DatasetKeySync.TestSyncResetsConcreteCatalogToNone [C:2] [TYPE Function]
# @BRIEF A contract carrying catalog=None overwrites a live concrete catalog value with None.
@pytest.mark.asyncio
async def test_sync_resets_concrete_catalog_to_none(mock_client):
contracts = [{"uuid": "ds-2", "catalog": None}]
report = await sync_dataset_composite_keys(mock_client, contracts)
assert report == {
"changed": 1,
"unchanged": 0,
"skipped_missing": 0,
"failed": 0,
"errors": [],
}
mock_client.update_dataset.assert_awaited_once_with(
8,
{"catalog": None},
override_columns=False,
)
# #endregion Test.DatasetKeySync.TestSyncResetsConcreteCatalogToNone
# #region Test.DatasetKeySync.TestSyncOmitsNoneTableName [C:2] [TYPE Function]
# @BRIEF A contract with table_name=None never sends a table_name key in the update payload.
@pytest.mark.asyncio
async def test_sync_omits_none_table_name(mock_client):
contracts = [
{"uuid": "ds-2", "catalog": None, "schema": "public", "table_name": None}
]
report = await sync_dataset_composite_keys(mock_client, contracts)
assert report == {
"changed": 1,
"unchanged": 0,
"skipped_missing": 0,
"failed": 0,
"errors": [],
}
mock_client.update_dataset.assert_awaited_once_with(
8,
{"catalog": None, "schema": "public"},
override_columns=False,
)
# #endregion Test.DatasetKeySync.TestSyncOmitsNoneTableName
# #region Test.DatasetKeySync.TestSyncCachesDatabaseUuidLookup [C:2] [TYPE Function]
# @BRIEF Repeated contracts with the same database_uuid resolve the database id only once.
@pytest.mark.asyncio
async def test_sync_caches_database_uuid_lookup(mock_client):
mock_client.get_datasets = AsyncMock(
return_value=(
3,
[
{
"id": 7,
"uuid": "ds-1",
"database": {"id": 1, "uuid": "db-live-1"},
"catalog": None,
"schema": "old",
"table_name": "users",
},
{
"id": 8,
"uuid": "ds-2",
"database": {"id": 2, "uuid": "db-live-2"},
"catalog": "c1",
"schema": "public",
"table_name": "orders",
},
{
"id": 9,
"uuid": "ds-4",
"database": {"id": 3, "uuid": "db-live-3"},
"catalog": None,
"schema": "s4",
"table_name": "items",
},
],
)
)
contracts = [
{"uuid": "ds-1", "database_uuid": "db-src-1", "table_name": "users"},
{"uuid": "ds-4", "database_uuid": "db-src-1", "table_name": "items"},
]
report = await sync_dataset_composite_keys(mock_client, contracts)
assert report["changed"] == 2
assert mock_client.get_database_by_uuid.await_count == 1
mock_client.get_database_by_uuid.assert_awaited_once_with("db-src-1")
# #endregion Test.DatasetKeySync.TestSyncCachesDatabaseUuidLookup
# #region Test.DatasetKeySync.TestSyncNoopWhenMatching [C:2] [TYPE Function]
# @BRIEF When every desired key already matches the live dataset, no update is issued.
@pytest.mark.asyncio
async def test_sync_noop_when_matching(mock_client):
contracts = [
{"uuid": "ds-2", "catalog": "c1", "schema": "public", "table_name": "orders"}
]
report = await sync_dataset_composite_keys(mock_client, contracts)
assert report == {
"changed": 0,
"unchanged": 1,
"skipped_missing": 0,
"failed": 0,
"errors": [],
}
mock_client.update_dataset.assert_not_called()
# #endregion Test.DatasetKeySync.TestSyncNoopWhenMatching
# #region Test.DatasetKeySync.TestSyncMissingContract [C:2] [TYPE Function]
# @BRIEF A contract whose uuid is absent from the live catalog is counted as skipped_missing.
@pytest.mark.asyncio
async def test_sync_missing_contract(mock_client):
contracts = [{"uuid": "ds-missing", "table_name": "ghost"}]
report = await sync_dataset_composite_keys(mock_client, contracts)
assert report == {
"changed": 0,
"unchanged": 0,
"skipped_missing": 1,
"failed": 0,
"errors": [],
}
mock_client.update_dataset.assert_not_called()
# #endregion Test.DatasetKeySync.TestSyncMissingContract
# #region Test.DatasetKeySync.TestSyncUpdateFailureIsolated [C:2] [TYPE Function]
# @BRIEF A failing update is recorded as failed while a healthy contract is still processed.
@pytest.mark.asyncio
async def test_sync_update_failure_isolated(mock_client):
mock_client.update_dataset = AsyncMock(
side_effect=[RuntimeError("update boom"), None]
)
contracts = [
{
"uuid": "ds-1",
"database_uuid": "db-src-1",
"schema": "public",
"table_name": "users",
},
{"uuid": "ds-2", "catalog": "c2", "schema": "public", "table_name": "orders"},
]
report = await sync_dataset_composite_keys(mock_client, contracts)
assert report["changed"] == 1
assert report["failed"] == 1
assert report["unchanged"] == 0
assert report["skipped_missing"] == 0
assert report["errors"][0]["uuid"] == "ds-1"
assert "update boom" in report["errors"][0]["error"]
# #endregion Test.DatasetKeySync.TestSyncUpdateFailureIsolated
# #region Test.DatasetKeySync.TestSyncUnresolvableDatabaseUuid [C:2] [TYPE Function]
# @BRIEF get_database_by_uuid returning None surfaces as a failed contract mentioning the database_uuid.
@pytest.mark.asyncio
async def test_sync_unresolvable_database_uuid(mock_client):
mock_client.get_database_by_uuid = AsyncMock(return_value=None)
contracts = [
{"uuid": "ds-1", "database_uuid": "missing-db", "table_name": "users"}
]
report = await sync_dataset_composite_keys(mock_client, contracts)
assert report["failed"] == 1
assert "database_uuid" in report["errors"][0]["error"]
assert "missing-db" in report["errors"][0]["error"]
mock_client.update_dataset.assert_not_called()
# #endregion Test.DatasetKeySync.TestSyncUnresolvableDatabaseUuid
# #region Test.DatasetKeySync.TestSyncFetchFailure [C:2] [TYPE Function]
# @BRIEF When fetching the live catalog raises, every contract is marked failed with the fetch error.
@pytest.mark.asyncio
async def test_sync_fetch_failure(mock_client):
mock_client.get_datasets = AsyncMock(side_effect=RuntimeError("fetch boom"))
contracts = [
{"uuid": "ds-1", "table_name": "users"},
{"uuid": "ds-2", "table_name": "orders"},
]
report = await sync_dataset_composite_keys(mock_client, contracts)
assert report["failed"] == len(contracts)
assert len(report["errors"]) == 2
assert all("fetch boom" in e["error"] for e in report["errors"])
# #endregion Test.DatasetKeySync.TestSyncFetchFailure
# #region Test.DatasetKeySync.TestReadLiveDatasetContracts [C:2] [TYPE Function]
# @BRIEF Live contracts are filtered by uuid and database_uuid is resolved from the nested object or via get_database.
@pytest.mark.asyncio
async def test_read_live_dataset_contracts(mock_client):
mock_client.get_datasets = AsyncMock(
return_value=(
3,
[
{
"id": 1,
"uuid": "ds-1",
"database": {"id": 10, "uuid": "db-live-1"},
"catalog": None,
"schema": "public",
"table_name": "users",
},
{
"id": 2,
"uuid": "ds-2",
"database": {"id": 99},
"catalog": "c1",
"schema": "public",
"table_name": "orders",
},
{
"id": 3,
"uuid": "ds-3",
"database": {},
"catalog": None,
"schema": None,
"table_name": "items",
},
],
)
)
mock_client.get_database = AsyncMock(return_value={"id": 99, "uuid": "db-resolved"})
contracts = await read_live_dataset_contracts(mock_client, dataset_uuids={"ds-1", "ds-2"})
assert set(contracts) == {"ds-1", "ds-2"}
assert contracts["ds-1"]["database_uuid"] == "db-live-1"
assert contracts["ds-2"]["database_uuid"] == "db-resolved"
mock_client.get_database.assert_awaited_once_with(99)
contracts_all = await read_live_dataset_contracts(mock_client)
assert set(contracts_all) == {"ds-1", "ds-2", "ds-3"}
assert contracts_all["ds-3"]["database_uuid"] is None
# #endregion Test.DatasetKeySync.TestReadLiveDatasetContracts
# #region Test.DatasetKeySync.TestReadLiveDatasetContractsRetriesWithoutQuery [C:2] [TYPE Function]
# @BRIEF The defensive retry falls back to a column-less fetch when the columns query is rejected.
@pytest.mark.asyncio
async def test_read_live_dataset_contracts_retries_without_query(mock_client):
mock_client.get_datasets = AsyncMock(
side_effect=[
RuntimeError("column not allowed"),
(
1,
[
{
"id": 1,
"uuid": "ds-1",
"database": {"id": 10, "uuid": "db-live-1"},
"catalog": None,
"schema": "s",
"table_name": "t",
}
],
),
]
)
contracts = await read_live_dataset_contracts(mock_client, {"ds-1"})
assert contracts["ds-1"]["uuid"] == "ds-1"
assert mock_client.get_datasets.await_count == 2
# #endregion Test.DatasetKeySync.TestReadLiveDatasetContractsRetriesWithoutQuery
# #endregion Test.DatasetKeySync

34
backend/tests/core/test_migration_engine.py
View File

@@ -696,4 +696,38 @@ def test_transform_zip_without_fix_cross_filters():
assert data["database_uuid"] == "new-uuid"
# #endregion Test.MigrationEngine.TestTransformZipWithoutFixCrossFilters
# #region Test.MigrationEngine.TestReadDatasetContractsFromZip [C:2] [TYPE Function]
# @BRIEF read_dataset_contracts_from_zip parses dataset YAMLs in-memory, skipping invalid and uuid-less entries.
def test_read_dataset_contracts_from_zip():
"""@BRIEF Dataset contracts are read from the ZIP without mutating the archive."""
engine = MigrationEngine()
with tempfile.TemporaryDirectory() as td:
td_path = Path(td)
zip_path = td_path / "datasets.zip"
with zipfile.ZipFile(zip_path, "w") as zf:
zf.writestr(
"datasets/a.yaml",
"uuid: ds-1\ndatabase_uuid: db-src-1\nschema: public\ntable_name: users\n",
)
zf.writestr(
"datasets/b.yaml",
"uuid: ds-2\ndatabase_uuid: db-src-2\ncatalog: c1\ntable_name: orders\n",
)
zf.writestr("datasets/nouuid.yaml", "table_name: x\n")
zf.writestr("datasets/bad.yaml", "{invalid: yaml: [}")
contracts = engine.read_dataset_contracts_from_zip(str(zip_path))
assert len(contracts) == 2
by_uuid = {c["uuid"]: c for c in contracts}
assert by_uuid["ds-1"]["database_uuid"] == "db-src-1"
assert by_uuid["ds-1"]["schema"] == "public"
assert by_uuid["ds-1"]["table_name"] == "users"
assert by_uuid["ds-1"].get("catalog") is None
assert by_uuid["ds-2"]["catalog"] == "c1"
assert by_uuid["ds-2"]["database_uuid"] == "db-src-2"
assert by_uuid["ds-2"]["table_name"] == "orders"
assert by_uuid["ds-2"].get("schema") is None
# #endregion Test.MigrationEngine.TestReadDatasetContractsFromZip
# #endregion Test.MigrationEngine.TestMigrationEngine

421
backend/tests/plugins/test_migration_plugin.py
View File

@@ -109,6 +109,21 @@ class TestMigrationPluginGetSchema:
assert schema["properties"]["from_env"]["enum"] == ["dev", "prod"]
def test_get_schema_includes_composite_key_fields(self):
"""Composite-key sync fields are exposed with correct types and defaults."""
plugin = MigrationPlugin()
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [_make_env("e1", "Dev"), _make_env("e2", "Prod")]
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm):
schema = plugin.get_schema()
props = schema["properties"]
assert props["sync_dataset_composite_keys"]["type"] == "boolean"
assert props["sync_dataset_composite_keys"]["default"] is True
assert props["composite_key_mutation_server"]["enum"] == ["target", "source"]
assert props["composite_key_mutation_server"]["default"] == "target"
class TestMigrationPluginExecute:
"""Verify MigrationPlugin.execute with various scenarios."""
@@ -917,4 +932,410 @@ class TestMigrationPluginExecute:
assert failed.get("error_type") == "GENERIC_COMMAND_ERROR"
assert 1010 in (failed.get("issue_codes") or [])
class TestMigrationPluginCompositeKeyFallback:
"""Composite-key sync fallback on non-password import failures."""
@pytest.mark.asyncio
async def test_execute_composite_key_target_fallback(self):
"""Target fallback syncs datasets on the target and retries import once."""
plugin = MigrationPlugin()
src_env = _make_env("env-1", "Source")
tgt_env = _make_env("env-2", "Target")
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [src_env, tgt_env]
contracts = [{"uuid": "ds-1", "database_uuid": "tgt-db", "catalog": None, "schema": "public", "table_name": "users"}]
mock_src_client = _make_mock_superset_client()
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
mock_tgt_client = _make_mock_superset_client()
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[
RuntimeError("Generic import boom"),
None,
])
mock_engine = MagicMock()
mock_engine.transform_zip.return_value = True
mock_engine.read_dataset_contracts_from_zip.return_value = contracts
mock_sync = AsyncMock(return_value={
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
})
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
patch('src.plugins.migration.SupersetClient') as MockSC, \
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
patch('src.plugins.migration.SessionLocal'):
MockSC.side_effect = [mock_src_client, mock_tgt_client]
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
result = await plugin.execute({
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1],
"replace_db_config": False,
"sync_dataset_composite_keys": True,
"composite_key_mutation_server": "target",
})
assert result["status"] == "SUCCESS"
assert len(result["migrated_dashboards"]) == 1
mock_sync.assert_awaited_once_with(mock_tgt_client, contracts)
@pytest.mark.asyncio
async def test_execute_composite_key_source_fallback(self, tmp_path):
"""Source fallback aligns source datasets with target live keys, re-exports and retries."""
plugin = MigrationPlugin()
src_env = _make_env("env-1", "Source")
tgt_env = _make_env("env-2", "Target")
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [src_env, tgt_env]
tmp_source = tmp_path / "source.zip"
tmp_transformed = tmp_path / "transformed.zip"
tmp_source.write_bytes(b"zip")
source_contracts = [{"uuid": "ds-1", "database_uuid": "src-db", "catalog": None, "schema": "public", "table_name": "users"}]
live_target = {"ds-1": {"uuid": "ds-1", "database_uuid": "tgt-db", "catalog": None, "schema": "public", "table_name": "users"}}
mock_src_client = _make_mock_superset_client()
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
mock_src_client.export_dashboard = AsyncMock(side_effect=[(b"zip", "meta"), (b"zip2", "meta")])
mock_tgt_client = _make_mock_superset_client()
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[RuntimeError("boom"), None])
mock_engine = MagicMock()
mock_engine.transform_zip.return_value = True
mock_engine.read_dataset_contracts_from_zip.return_value = source_contracts
mock_sync = AsyncMock(return_value={
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
})
mock_read_live = AsyncMock(return_value=live_target)
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
patch('src.plugins.migration.SupersetClient') as MockSC, \
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
patch('src.plugins.migration.read_live_dataset_contracts', new=mock_read_live), \
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
patch('src.plugins.migration.SessionLocal'):
MockSC.side_effect = [mock_src_client, mock_tgt_client]
mock_ctf.return_value.__enter__ = MagicMock(
side_effect=[str(tmp_source), str(tmp_transformed)]
)
result = await plugin.execute({
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1],
"replace_db_config": False,
"db_mappings": {"src-db": "tgt-db"},
"sync_dataset_composite_keys": True,
"composite_key_mutation_server": "source",
})
assert result["status"] == "SUCCESS"
assert len(result["migrated_dashboards"]) == 1
expected_sync_contract = {
"uuid": "ds-1",
"catalog": None,
"schema": "public",
"table_name": "users",
"database_uuid": "src-db",
}
mock_sync.assert_awaited_once_with(mock_src_client, [expected_sync_contract])
assert mock_src_client.export_dashboard.await_count == 2
assert mock_engine.transform_zip.call_count == 2
assert mock_engine.transform_zip.call_args.args[:2] == (
str(tmp_source), str(tmp_transformed),
)
mock_tgt_client.import_dashboard.assert_awaited_with(
file_name=str(tmp_transformed), dash_id=1, dash_slug="slug-1"
)
@pytest.mark.asyncio
async def test_execute_composite_key_source_no_inverse_db_uuid(self, tmp_path):
"""When the target db uuid has no inverse mapping, the contract syncs without database_uuid."""
plugin = MigrationPlugin()
src_env = _make_env("env-1", "Source")
tgt_env = _make_env("env-2", "Target")
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [src_env, tgt_env]
tmp_source = tmp_path / "source.zip"
tmp_transformed = tmp_path / "transformed.zip"
tmp_source.write_bytes(b"zip")
source_contracts = [{"uuid": "ds-1", "database_uuid": "src-db", "catalog": None, "schema": "public", "table_name": "users"}]
live_target = {"ds-1": {"uuid": "ds-1", "database_uuid": "tgt-db", "catalog": None, "schema": "public", "table_name": "users"}}
mock_src_client = _make_mock_superset_client()
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
mock_src_client.export_dashboard = AsyncMock(side_effect=[(b"zip", "meta"), (b"zip2", "meta")])
mock_tgt_client = _make_mock_superset_client()
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[RuntimeError("boom"), None])
mock_engine = MagicMock()
mock_engine.transform_zip.return_value = True
mock_engine.read_dataset_contracts_from_zip.return_value = source_contracts
mock_sync = AsyncMock(return_value={
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0,
"errors": [{"uuid": "ds-1", "error": "cannot_derive_source_db_uuid"}],
})
mock_read_live = AsyncMock(return_value=live_target)
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
patch('src.plugins.migration.SupersetClient') as MockSC, \
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
patch('src.plugins.migration.read_live_dataset_contracts', new=mock_read_live), \
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
patch('src.plugins.migration.SessionLocal'):
MockSC.side_effect = [mock_src_client, mock_tgt_client]
mock_ctf.return_value.__enter__ = MagicMock(
side_effect=[str(tmp_source), str(tmp_transformed)]
)
result = await plugin.execute({
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1],
"replace_db_config": False,
"db_mappings": {},
"sync_dataset_composite_keys": True,
"composite_key_mutation_server": "source",
})
assert result["status"] == "SUCCESS"
assert len(result["migrated_dashboards"]) == 1
sync_contract = mock_sync.await_args.args[1][0]
assert "database_uuid" not in sync_contract
assert sync_contract["schema"] == "public"
assert sync_contract["table_name"] == "users"
assert any(
e.get("error") == "cannot_derive_source_db_uuid"
for e in mock_sync.return_value["errors"]
)
assert mock_src_client.export_dashboard.await_count == 2
assert mock_engine.transform_zip.call_count == 2
assert mock_engine.transform_zip.call_args.args[:2] == (
str(tmp_source), str(tmp_transformed),
)
mock_tgt_client.import_dashboard.assert_awaited_with(
file_name=str(tmp_transformed), dash_id=1, dash_slug="slug-1"
)
@pytest.mark.asyncio
async def test_execute_composite_key_source_skips_sync_when_no_contracts(self, tmp_path):
"""When the source archive yields no dataset contracts, sync and live reads are skipped entirely."""
plugin = MigrationPlugin()
src_env = _make_env("env-1", "Source")
tgt_env = _make_env("env-2", "Target")
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [src_env, tgt_env]
tmp_source = tmp_path / "source.zip"
tmp_transformed = tmp_path / "transformed.zip"
tmp_source.write_bytes(b"zip")
mock_src_client = _make_mock_superset_client()
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
mock_src_client.export_dashboard = AsyncMock(side_effect=[(b"zip", "meta"), (b"zip2", "meta")])
mock_tgt_client = _make_mock_superset_client()
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[RuntimeError("boom"), None])
mock_engine = MagicMock()
mock_engine.transform_zip.return_value = True
mock_engine.read_dataset_contracts_from_zip.return_value = []
mock_sync = AsyncMock(return_value={
"changed": 0, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
})
mock_read_live = AsyncMock(return_value={})
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
patch('src.plugins.migration.SupersetClient') as MockSC, \
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
patch('src.plugins.migration.read_live_dataset_contracts', new=mock_read_live), \
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
patch('src.plugins.migration.SessionLocal'):
MockSC.side_effect = [mock_src_client, mock_tgt_client]
mock_ctf.return_value.__enter__ = MagicMock(
side_effect=[str(tmp_source), str(tmp_transformed)]
)
result = await plugin.execute({
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1],
"replace_db_config": False,
"db_mappings": {"src-db": "tgt-db"},
"sync_dataset_composite_keys": True,
"composite_key_mutation_server": "source",
})
assert result["status"] == "PARTIAL_SUCCESS"
assert len(result["failed_dashboards"]) == 1
mock_sync.assert_not_awaited()
mock_read_live.assert_not_awaited()
assert mock_src_client.export_dashboard.await_count == 1
@pytest.mark.asyncio
async def test_execute_composite_key_fallback_failure_preserves_entry(self):
"""When the retry also fails, the original entry is preserved with the sync report attached."""
plugin = MigrationPlugin()
src_env = _make_env("env-1", "Source")
tgt_env = _make_env("env-2", "Target")
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [src_env, tgt_env]
mock_src_client = _make_mock_superset_client()
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
mock_tgt_client = _make_mock_superset_client()
mock_tgt_client.import_dashboard = AsyncMock(side_effect=[
RuntimeError("Generic import boom"),
RuntimeError("Still broken"),
])
mock_engine = MagicMock()
mock_engine.transform_zip.return_value = True
mock_engine.read_dataset_contracts_from_zip.return_value = [
{"uuid": "ds-1", "database_uuid": "tgt-db", "catalog": None, "schema": "public", "table_name": "users"}
]
report = {"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": []}
mock_sync = AsyncMock(return_value=report)
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
patch('src.plugins.migration.SupersetClient') as MockSC, \
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
patch('src.plugins.migration.SessionLocal'):
MockSC.side_effect = [mock_src_client, mock_tgt_client]
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
result = await plugin.execute({
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1],
"replace_db_config": False,
"sync_dataset_composite_keys": True,
"composite_key_mutation_server": "target",
})
assert result["status"] == "PARTIAL_SUCCESS"
entry = result["failed_dashboards"][0]
assert "Generic import boom" in entry["error"]
assert entry["composite_key_sync_report"] == report
assert entry["composite_key_mutation_server"] == "target"
assert entry["composite_key_retried"] is True
@pytest.mark.asyncio
async def test_execute_composite_key_never_syncs_password_failure(self):
"""Password-required failures never trigger the composite-key fallback."""
plugin = MigrationPlugin()
src_env = _make_env("env-1", "Source")
tgt_env = _make_env("env-2", "Target")
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [src_env, tgt_env]
mock_src_client = _make_mock_superset_client()
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
mock_tgt_client = _make_mock_superset_client()
mock_tgt_client.import_dashboard = AsyncMock(side_effect=RuntimeError(
"Must provide a password for the database PostgreSQL"
))
mock_engine = MagicMock()
mock_engine.transform_zip.return_value = True
mock_sync = AsyncMock(return_value={
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
})
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
patch('src.plugins.migration.SupersetClient') as MockSC, \
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
patch('src.plugins.migration.SessionLocal'):
MockSC.side_effect = [mock_src_client, mock_tgt_client]
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
result = await plugin.execute({
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1],
"replace_db_config": False,
"sync_dataset_composite_keys": True,
})
mock_sync.assert_not_awaited()
assert len(result["failed_dashboards"]) == 1
@pytest.mark.asyncio
async def test_execute_composite_key_disabled_skips_fallback(self):
"""When sync_dataset_composite_keys is False, non-password failures keep the legacy entry."""
plugin = MigrationPlugin()
src_env = _make_env("env-1", "Source")
tgt_env = _make_env("env-2", "Target")
mock_cm = MagicMock()
mock_cm.get_environments.return_value = [src_env, tgt_env]
mock_src_client = _make_mock_superset_client()
mock_src_client.get_dashboards = AsyncMock(return_value=(True, [_make_dashboard(1, "Dash")]))
mock_src_client.export_dashboard = AsyncMock(return_value=(b"zip", "meta"))
mock_tgt_client = _make_mock_superset_client()
mock_tgt_client.import_dashboard = AsyncMock(side_effect=RuntimeError("Generic import boom"))
mock_engine = MagicMock()
mock_engine.transform_zip.return_value = True
mock_sync = AsyncMock(return_value={
"changed": 1, "unchanged": 0, "skipped_missing": 0, "failed": 0, "errors": [],
})
with patch('src.plugins.migration.get_config_manager', return_value=mock_cm), \
patch('src.plugins.migration.SupersetClient') as MockSC, \
patch('src.plugins.migration.MigrationEngine', return_value=mock_engine), \
patch('src.plugins.migration.create_temp_file') as mock_ctf, \
patch('src.plugins.migration.sync_dataset_composite_keys', new=mock_sync), \
patch('src.plugins.migration.IdMappingService', return_value=_make_mock_mapping_service()), \
patch('src.plugins.migration.SessionLocal'):
MockSC.side_effect = [mock_src_client, mock_tgt_client]
mock_ctf.return_value.__enter__ = MagicMock(return_value="/tmp/test.zip")
result = await plugin.execute({
"source_env_id": "env-1",
"target_env_id": "env-2",
"selected_ids": [1],
"replace_db_config": False,
"sync_dataset_composite_keys": False,
})
mock_sync.assert_not_awaited()
entry = result["failed_dashboards"][0]
assert "composite_key_retried" not in entry
# #endregion Test.MigrationPlugin

59
backend/tests/test_dashboards_api.py
View File

@@ -15,6 +15,7 @@ from src.api.routes.dashboards import (
DashboardTaskHistoryResponse,
DatabaseMappingsResponse,
)
from src.api.routes.dashboards._projection import _task_matches_dashboard
from src.app import app
from src.dependencies import (
get_config_manager,
@@ -460,64 +461,11 @@ def test_get_dashboard_thumbnail_202(mock_deps):
assert "Thumbnail is being generated" in response.json()["message"]
# --- 6. migrate_dashboards tests ---
# #endregion Test.DashboardsApi.TestGetDashboardThumbnail202
# #region Test.DashboardsApi.TestMigrateDashboardsSuccess [C:2] [TYPE Function]
# @RELATION BINDS_TO ->[Test.DashboardsApi.TestDashboardsApi]
def test_migrate_dashboards_success(mock_deps):
mock_s = MagicMock()
mock_s.id = "s"
mock_t = MagicMock()
mock_t.id = "t"
mock_deps["config"].get_environments.return_value = [mock_s, mock_t]
mock_deps["task"].create_task = AsyncMock(return_value=MagicMock(id="task-123"))
response = client.post(
"/api/dashboards/migrate",
json={"source_env_id": "s", "target_env_id": "t", "dashboard_ids": [1]},
)
assert response.status_code == 200
assert response.json()["task_id"] == "task-123"
# #endregion Test.DashboardsApi.TestMigrateDashboardsSuccess
# #region Test.DashboardsApi.TestMigrateDashboardsPreChecks [C:2] [TYPE Function]
# @RELATION BINDS_TO ->[Test.DashboardsApi.TestDashboardsApi]
def test_migrate_dashboards_pre_checks(mock_deps):
# Missing IDs
response = client.post(
"/api/dashboards/migrate",
json={"source_env_id": "s", "target_env_id": "t", "dashboard_ids": []},
)
assert response.status_code == 400
assert "At least one dashboard ID must be provided" in response.json()["detail"]
# #endregion Test.DashboardsApi.TestMigrateDashboardsPreChecks
# #region Test.DashboardsApi.TestMigrateDashboardsEnvNotFound [C:2] [TYPE Function]
# @RELATION BINDS_TO ->[Test.DashboardsApi.TestDashboardsApi]
def test_migrate_dashboards_env_not_found(mock_deps):
"""@PRE: source_env_id and target_env_id are valid environment IDs."""
mock_deps["config"].get_environments.return_value = []
response = client.post(
"/api/dashboards/migrate",
json={"source_env_id": "ghost", "target_env_id": "t", "dashboard_ids": [1]},
)
assert response.status_code == 404
assert "Source environment not found" in response.json()["detail"]
# --- 7. backup_dashboards tests ---
# #endregion Test.DashboardsApi.TestMigrateDashboardsEnvNotFound
# #region Test.DashboardsApi.TestBackupDashboardsSuccess [C:2] [TYPE Function]
# @RELATION BINDS_TO ->[Test.DashboardsApi.TestDashboardsApi]
@@ -539,7 +487,7 @@ def test_backup_dashboards_success(mock_deps):
# #region Test.DashboardsApi.TestBackupDashboardsPreChecks [C:2] [TYPE Function]
# @RELATION BINDS_TO ->[Test.DashboardsApi.TestDashboardsApi]
def test_backup_dashboards_pre_checks(mock_deps):
def test_backup_dashboards_pre_checks():
response = client.post(
"/api/dashboards/backup", json={"env_id": "prod", "dashboard_ids": []}
)
@@ -589,9 +537,6 @@ def test_backup_dashboards_with_schedule(mock_deps):
# --- 8. Internal logic: _task_matches_dashboard ---
# #endregion Test.DashboardsApi.TestBackupDashboardsWithSchedule
from src.api.routes.dashboards._projection import _task_matches_dashboard
# #region Test.DashboardsApi.TestTaskMatchesDashboardLogic [C:2] [TYPE Function]
# @RELATION BINDS_TO ->[Test.DashboardsApi.TestDashboardsApi]
def test_task_matches_dashboard_logic():

21169
docs/api/doxygen_docs.h
View File

File diff suppressed because it is too large Load Diff

21577
docs/api/doxygen_grace.txt
View File

File diff suppressed because it is too large Load Diff

18019
docs/api/doxygen_stripped.c
View File

File diff suppressed because it is too large Load Diff

18019
docs/api/doxygen_stripped.txt
View File

File diff suppressed because it is too large Load Diff

2025
research/DWH-14943 — UUID-authoritative импорт Dashboard и Dataset с безопасным reparenting.md Normal file
View File

File diff suppressed because it is too large Load Diff

5
specs/019-superset-ux-redesign/contracts/modules.md
View File

@@ -438,9 +438,8 @@ export const activityStore = derived(
# Query: env_id (required), search (optional)
# Response: { dashboards: [...] }
# Endpoint: POST /api/dashboards/migrate
# Body: { source_env_id, target_env_id, dashboard_ids, db_mappings }
# Response: { task_id }
# Dashboard migration is handled by POST /api/migration/execute.
# The dashboard list passes env_id and selected IDs to the unified /migration wizard.
# Endpoint: POST /api/dashboards/backup
# Body: { env_id, dashboard_ids, schedule (optional cron) }

4
specs/019-superset-ux-redesign/tasks.md
View File

@@ -167,11 +167,11 @@ All implementation tasks MUST follow the Design-by-Contract specifications:
- [x] T033 [US3] Implement dashboard list fetching with Git status and last task status
- [x] T034 [US3] Add pagination support to GET /api/dashboards endpoint (page, page_size parameters)
_Contract: @POST: Response includes pagination metadata_
- [x] T035 [US3] Implement bulk migration endpoint POST /api/dashboards/migrate with target environment and dashboard IDs
- [x] T035 [US3] Route bulk migration selections into the unified `/migration` wizard; execute via POST /api/migration/execute
_Contract: @PRE: User has permission plugin:migration:execute_
- [x] T036 [US3] Implement bulk backup endpoint POST /api/dashboards/backup with optional cron schedule
_Contract: @PRE: User has permission plugin:backup:execute_
- [x] T037 [US3] Add database mappings retrieval from MappingService for migration modal
- [x] T037 [US3] Add database mappings retrieval from MappingService for the migration wizard
- [x] T064 [US3] Fix "API endpoint not found" for databases by correcting endpoint path in `frontend/src/lib/api.js`
### Frontend for User Story 3

200
specs/REVIEW-042-047-CLOSURE.md
View File

@@ -1,200 +0,0 @@
# Review Closure — 042–047 Architectural Gaps (2026-08-07)
**Purpose**: Trace every review finding from the 042–047 architecture review to a concrete resolution and the spec/file that changes. Prevents silent regression and keeps the audit trail. P0 items are binding; P1/P2 documented.
**Reading**: `#` = review finding (1–27 from the review). `→` = resolution. `[FILE]` = file updated.
---
## P0 — Binding fixes
### #1. Scenario never reaches the Registry after creation
**Problem**: 042 has `GET /scenarios` but no `POST /scenarios`; 039 Save → 042 registration gap.
**Resolution**: Add `CreateScenario` transaction to 042: input `{validated_revision, draft_pack, owner, dashboard}`, atomic `ScenarioRegistryEntry + ScenarioRevision #1 + artifact materialization binding`, output `{scenario_id, revision_id + content_hash}`. 039 Save calls it. Add task + route + spec story.
**[FILE]** `042/contracts/openapi.yaml` (POST /scenarios), `042/data-model.md` (CreateScenario), `042/tasks.md` (T0xx), `042/spec.md` (US: Save→Register).
### #2. ScenarioRevision ↔ runner.plan.json can diverge
**Problem**: r18 graph edited, runner.plan.json still r17 → run claims r18 but executes r17. Kills reproducibility.
**Resolution**: **RunnerPlan is a deterministic derivation from ScenarioRevision at run start**, never a stored runtime source of truth. The run compares its derived revision/program/action-registry hashes to the selected revision before execution.
**[FILE]** `044/data-model.md`, `044/contracts/modules.md` (RunnerPlan derivation), `042/data-model.md` (revision carries runner_plan_hash).
### #3. Wrong reuse of 036 ApprovalGate for human checkpoint / ScenarioRun
**Problem**: 036 gate contract is only `repository_write | baseline_approval`, owned by an AgentRun; ScenarioRun is separate; `false_positive/inconclusive` aren't 036 decisions.
**Resolution**: Split into two domain concepts:
- `ActionApprovalGate` — PROD execution approval, baseline approval, repository mutation. Reuses 036 gate mechanism, but generalized owner (see #4).
- `HumanCheckpoint` — test observation disposition: confirm | false_positive | inconclusive. A separate entity with its own lifecycle, not a 036 gate.
**[FILE]** `044/data-model.md`, `044/contracts/modules.md`, `044/spec.md` (Clarifications), `044/contracts/openapi.yaml` (human/decision schema).
### #4. Execution artifacts still tied to AgentRun
**Problem**: 038 capture / VLM require `agent_run_id`; ScenarioRun (agent_run_id=null) can't supply it. Evidence/report/xlsx/screenshot shouldn't be DraftArtifact of an artificial AgentRun.
**Resolution**: Introduce a **generic artifact owner** — `Artifact { id, owner_type: agent_run|scenario_run|verification_run|load_run, owner_id, kind, sha256, content_ref, retention_class, ... }`. ScenarioRun evidence uses `owner_type=scenario_run`. Add `ScenarioRunArtifact` projection. Evidence bridge accepts owner_type (036 evidence adapter generalized).
**[FILE]** `044/data-model.md`, `044/contracts/modules.md`, `042/data-model.md` (retention), `044/spec.md`.
### #5. Backend contract for 045 missing (history/result/compare/retry/SSE)
**Problem**: 045 UI promises RunHistory/RunComparison but 044 has no `GET /scenarios/{id}/runs`, `GET /scenario-runs/{id}/result`, `GET /scenario-runs/compare`, step retry, or an SSE event schema.
**Resolution**: Add to 044 OpenAPI: `GET /scenarios/{id}/runs`, `GET /scenario-runs/{id}/result`, `GET /scenario-runs/compare?a=&b=`, `POST /scenario-runs/{id}/steps/{logical_step_id}/retry`, and an **SSE event contract** (id, sequence, event_type, run_id, logical_step_id?, attempt?, occurred_at, payload; Last-Event-ID replay, heartbeat, terminal close) — reuse the 036 AgentRunEvent pattern.
**[FILE]** `044/contracts/openapi.yaml`, `044/contracts/modules.md`, `044/spec.md`.
### #6. Durable worker semantics (lease/heartbeat/idempotency/crash recovery)
**Problem**: crash-recovery claim needs at-least-once primitives, not a service loop.
**Resolution**: Add worker runtime primitives: worker lease, heartbeat, run claim, step claim, lease expiration, idempotency key, recovery scheduler. Each executor declares `idempotent? | retry-safe? | side-effect key? | external request id?`. `POST /scenario-runs` requires `Idempotency-Key` to prevent double-run on double-click.
**[FILE]** `044/data-model.md`, `044/contracts/modules.md`, `044/contracts/openapi.yaml`, `044/tasks.md`.
### #7. 043 edit contract bypass (arbitrary draft)
**Problem**: `save` accepts `draft: object` → client can bypass the constrained editor.
**Resolution**: **Server-stored WorkingDraft**. `apply` persists a WorkingDraft server-side and returns `draft_id + digest`. `save(draft_id, digest)` reloads it server-side, re-validates, canonicalizes, re-hashes, compares base revision. Client never returns the full graph.
**[FILE]** `043/data-model.md`, `043/contracts/modules.md`, `043/contracts/openapi.yaml`, `043/tasks.md`.
### #8. Stable logical step identity
**Problem**: an ordinal-derived step identity can shift on edit; breaks 045 compare + 047 flakiness across revisions.
**Resolution**: `logical_step_id = UUID` (immutable) + `step_position` (mutable) + `step_content_hash` (mutable). Analytics key on logical_step_id.
**[FILE]** `044/data-model.md`, `042/data-model.md`, `045/data-model.md`, `047/data-model.md`.
### #9. Revalidation / Migration workflow (staleness remediation)
**Problem**: "Revalidate" button has no defined flow.
**Resolution**: Scenario Migration workflow in 043: stale → revalidate against current dashboard → automatic mappings + manual conflicts → proposed r18 → diff → approve. Add spec story + tasks.
**[FILE]** `043/spec.md`, `043/data-model.md`, `043/tasks.md`, `042/spec.md`.
### #10. Global Run Operations Center + Automation Management UI
**Problem**: 045 covers Scenario→Runs only; 046 is backend-only.
**Resolution**:
- 045: add **Global Run Operations Center** (`/dashboard-testing/runs`) — all active/queued/waiting-human/failed/recent runs with filters, incl. **"Waiting for me"** for human checkpoints.
- 046: add **Automation Management UI** — schedules/triggers/policies CRUD surface.
**[FILE]** `045/spec.md`, `045/data-model.md`, `045/tasks.md`, `046/spec.md`, `046/contracts/ux/`.
---
## P1/P2 — Documented resolutions
### #12. Run Configuration vs 044 start API mismatch
**Resolution**: 044 `POST /scenario-runs` gains `release`, `baseline_set`, `execution_toggles`, `approval_ref` (optional), matching 045 RunConfiguration.
**[FILE]** `044/contracts/openapi.yaml`.
### #16. Result aggregation truth table
**Resolution**: Define step→assertion→scenario aggregation formally. A step outcome maps to a normalized contribution; scenario result derived with explicit rules (e.g., any failed → failed; skipped doesn't count against pass; blocked descendants counted as blocked, not failed). Add truth table.
**[FILE]** `044/data-model.md`.
### #17. Execution toggles safety
**Resolution**: Toggles may only disable **optional evidence enrichment** (diagnostic screenshots, verbose logs, optional VLM commentary). Mandatory graph steps cannot be toggled off; otherwise run would false-PASS.
**[FILE]** `045/spec.md`, `044/contracts/openapi.yaml`.
### #18. (covered by #9) Revalidation = 043 migration workflow.
### #20. Automation Management UI → 046 (see #10).
### #21. API trigger operation
**Resolution**: Add `POST /scenarios/{id}/trigger` (external API run trigger) to 046, distinct from creating a `ScenarioTriggerRule(trigger=api)`.
**[FILE]** `046/contracts/openapi.yaml`.
### #22. Scheduler semantics
**Resolution**: Specify timezone, DST, `misfire_grace_time`, `coalesce`, `max_instances`, missed-execution policy (run-immediately | skip | queue), scheduler-restart handling.
**[FILE]** `046/data-model.md`, `046/spec.md`.
### #23. Retention vs analytics conflict
**Resolution**: Layered retention tiers (run metadata 180d, triage/audit 365d/policy, step metrics 90d, heavy artifacts 30d, screenshots 30d, raw VLM 7d) and an **analytics minimum history window** independent of retention.
**[FILE]** `046/data-model.md`, `047/data-model.md`.
### #24. 047 resource shape
**Resolution**: Change 047 endpoints to `GET /scenarios/{scenario_id}/health|trends|recurring-failures` (health/trends aggregate scenario history, not one run).
**[FILE]** `047/contracts/openapi.yaml`.
### #25. Strict flakiness rules
**Resolution**: Define window + eligibility (same environment class, logical step, compatibility_family, baseline family) + flaky iff pass AND fail observed AND failure ratio within (X,Y) AND infra failures excluded.
**[FILE]** `047/data-model.md`.
### #26. Immutable recurring fingerprint
**Resolution**: Fingerprint built from immutable raw evidence: `logical_step_id + error_code + normalized_error_signature + assertion_kind + affected_ref`. Triage classification is a separate group attribute, never part of fingerprint.
**[FILE]** `047/data-model.md`.
### #27. Triage split
**Resolution**: `RunResult` = immutable truth; triage = `{ investigation_status: new|investigating|resolved, classification, resolution: fixed|accepted_risk|duplicate|wont_fix }`. Run stays FAILED; triage is orthogonal.
**[FILE]** `047/data-model.md`, `047/contracts/modules.md`.
---
## Updated scores (target after closure)
| Spec | Concept | Contractual closure |
|------|:------:|:-------------------:|
| 042 Registry | 8.5 | 8.0 |
| 043 Editor | 8.5 | 8.0 |
| 044 Execution | 9.0 | 8.0 |
| 045 Run Monitor | 9.0 | 8.0 |
| 046 Automation | 8.0 | 7.5 |
| 047 Analytics | 8.0 | 7.5 |
## Cross-Spec Canonicalization Pass (2026-08-07, second pass)
Reconciled the stale 038 core with 042–047. Bindings applied to **038** and all **normative** (spec/research/checklists/ux/prototype):
- **#1/#7** 038 identity: `scenario_id` slug → **`scenario_key`** (semantic); `scenario_id` (UUID) + `revision_id` (UUID) assigned by 042 at Save; compiler emits `content_hash` only.
- **#2/#8/#11** Replaced `revision_id + content_hash`/`parent_revision_id + content_hash`/`scenario_revision_id + scenario_content_hash` with `revision_id`/`content_hash`/`parent_revision_id` across 042/043/044/045 (openapi, data-model, modules, spec, research, checklists, tasks). 043 unified fully.
- **#3/#4/#13** 038 step schema: added `logical_step_id` (UUID, immutable) + `step_key`/`position`/`step_content_hash`; runtime `VlmFinding`/`HumanDisposition` moved to 044; `VlmAnalysisSpec`/`ScreenshotCaptureSpec` stay in 038.
- **#5/#6/#12** 038 runtime capture/VLM/disposition endpoints marked `deprecated`→410 MOVED_TO_044; `agent_run_id` removed from compile/capture (provenance optional, source_type: agent_run|editor|migration|api). Runtime evidence = `Artifact(owner_type=scenario_run)`, never authoring DraftPack.
- **#8** `false_positive` vocabulary unified; `dismiss` removed (037 dispositions split; 047 triage split investigation_status/classification/resolution).
- **#9/#10** 038 `validation.md` PASS **nullified** (self-contradictory COMPLETE vs OPEN); refocused as compiler-layer PASS only; T057–T059 moved to 044; `tasks.md` T046 rewritten.
## Machine-Contract Reconciliation Gate (2026-08-09)
Closed the P0 "prose-fixed, contracts-stale" gap by validating the machine-readable source of truth:
- **OpenAPI YAML**: fixed 3 syntax failures (`038` line 472 unquoted `:`, `043` line 32 flow `{...}`, `046` line 23 & 51 `Schedule[]`/`TriggerRule[]`). All 036–047 OpenAPI now parse.
- **038 JSON Schema** (`dashboard-test-scenario.schema.json`): migrated to canonical identity — `scenario_id`→`scenario_key`, `revision_id + content_hash`→`content_hash`; step `id`→`logical_step_id`+`step_key`+`position`+`step_content_hash`; `vlm_analysis`→`vlm_analysis_spec`; `affected_step_ids`→`affected_logical_step_ids`; `coverage.step_ids`→`logical_step_ids`; disposition `dismissed`→`false_positive`.
- **038 fixtures** (`fixtures/api/*.json`, 6): migrated to canonical identity via transform script; fixed 66-char fingerprint; **6/6 validate against the JSON Schema** (jsonschema).
- **038 validation.md**: regenerated cleanly (was self-contradictory PASS). Compiler-scope PASS; runtime gated by 044.
- **039 prototype**: `scenario_id`→`scenario_key` in JSON display; **038 prototype** `revision_id + content_hash`→`content_hash`.
- **New tool** `reconcile_contracts.py`: parses all OpenAPI/JSON, checks forbidden old-identity tokens in machine files (with compiled-output scoping for `scenario_id`), validates fixtures vs schema. Gate: **PASS (0 findings)** on 038–047 and `all`.
**Follow-up items** listed here were resolved by the 2026-08-10/11 canonical contracts below; implementation must use those newer sections as normative source.
## Status
- [x] Closure doc written
- [x] 042-047 edits applied (previous passes)
- [x] **Machine-Contract Reconciliation Gate applied** (038 schema+fixtures+validation, OpenAPI YAML fixes, reconcile_contracts.py) — PASS 0 findings
## Contract Closure / Canonicalization Pass (2026-08-10)
The HOLD review has been incorporated as binding canonical contracts before implementation:
- `reconcile_contracts.py` now performs OpenAPI path-template semantic validation and a narrow rejected-decision drift scan; all 036–047 contracts pass.
- 036 `ActionApprovalGate` is generic (`owner_type`, `owner_id`, scenario/load operations); the agent route is explicitly an adapter.
- 044 creates durable `ScenarioRun(pending_approval)` plus a gate for PROD, pins `ParameterBinding`, `TargetSnapshot`, and `ExecutionPrincipal` provenance, defines CAS `HumanCheckpoint`, and limits browser recovery to deterministic checkpoint replay.
- 038 contains immutable `ParameterDefinition` only; runtime values/statuses were removed from its JSON Schema and fixtures. Logical step UUIDs are minted once and carried through revisions; neither position nor ordinal affects identity.
- 042 owns metadata versions separately from executable revisions, records clone provenance without cross-scenario revision parents, aggregates/deduplicates active staleness signals, and defines the object-level ACL intersection.
- 043 separates metadata and executable edits, types parameter values as JSON validated against ParameterDefinition, and completes proposal/revalidation → WorkingDraft → save.
- 045 uses the pre-run scenario configuration resource, exposes every executor state/filter, and separates mandatory checks from optional diagnostics.
- 046 exposes schedule/trigger/policy CRUD schemas, derives coalescing from missed-execution policy, requires external-trigger idempotency, and delegates cross-run quota to 044's shared capacity manager.
- 047 keys analytics on `compatibility_family` and `logical_step_id`, splits health dimensions, and uses alertable recurring-failure episodes after resolution.
**Verification:** `backend/.venv/bin/python reconcile_contracts.py 036-047` → **PASS (0 findings)**; `git diff --check` → **PASS**.
## Final Execution-Semantics Closure (2026-08-10)
The second HOLD review is resolved with these binding decisions:
1. 038 authoring validity permits unbound required ParameterDefinitions; 044 alone performs per-launch RunPreflight/ParameterBinding.
2. 036 exposes generic ActionApprovalGate read/decision/consume routes and operation-specific ScenarioExecution approval requests.
3. All entry points share the 044 PROD lifecycle: accepted run is `queued` or `pending_approval`, never an approval-only 403.
4. 038 ActionRegistry is version-pinned; 044 BrowserExecutor dispatches only registered `{tool, action}` contracts.
5. Mutation has a distinct immutable contract. MVP prohibits all PROD mutation and requires controlled test-data scope, cleanup and non-retry default elsewhere.
6. Idempotent replay returns the prior run for the same key/request hash; different request reuse is 409.
7. Metadata has an ETag PATCH route; executable draft save accepts confirmation only and derives its audit summary server-side.
8. Revalidation includes conflict-resolution before proposal acceptance.
9. 044 publishes typed result, comparison and SSE schemas; checkpoint outcome is selected by checkpoint type/policy.
10. 046 separates dedup identity from capacity concurrency; 042 consumes (not derives) 047 health; 047 uses compatibility family and alertable FailureEpisodes.
`reconcile_contracts.py` now enforces 20 final-closure clauses in addition to parsability, schemas, semantic path parameters, fixtures, identity drift and rejected-decision patterns. The stale 041 task digest has also been corrected; future validation-table generation remains a separate tooling improvement.
## Verification Program Reconciliation (2026-08-11)
The canonical system model is **agent-authored, deterministically executed verification programs**:
- 038 now makes `VerificationProgram` required, content-hashed IR: navigation, evidence, bounded transforms, assertions and explicitly declared semantic evaluation.
- Source-mart `SqlEvidenceSpec` is permitted only during authoring/edit/revalidation/investigation proposal, passes AST/policy/schema/preview validation and is executed at runtime unchanged through the Superset SQL Lab adapter with typed bindings and pinned security context.
- `TransformSpec` is bounded DSL; `ComparisonSpec` is first-class; arbitrary code and runtime SQL/DSL rewrite remain forbidden.
- 044 orchestration stays deterministic but supports bounded, versioned `AgentEvaluationSpec`; immutable AgentEvaluation evidence becomes StepOutcome only through DecisionPolicy.
- 039 previews program structure, evidence sources and SQL/DSL/assertion/evaluation diffs; ChangeRequestContext is explicit and required for compilation.
- 045 renders typed events/results, including DecisionPolicy outcomes and AgentEvaluation summaries, without prose parsing.
- 047 distinguishes deterministic failure from model disagreement, low confidence and model instability; only deterministic outcomes feed deterministic flakiness.
**Generated gate command:** `backend/.venv/bin/python reconcile_contracts.py 036-047` validates YAML/JSON, OpenAPI paths, JSON fixtures, legacy drift, Verification Program schema/action registry and runtime mutation boundary. Current result: **PASS (0 findings)**.

240
specs/research-task-execution-architecture.md
View File

@@ -1,240 +0,0 @@
# Task Execution Architecture — superset-tools
**Date:** 2026-07-02
**Purpose:** Comprehensive audit of task execution paths — what runs through TaskManager, what bypasses it, and why.
**Context:** User noted translation tasks are missing from `/reports` (Task Status Center), which only shows tasks from the generic TaskManager pipeline.
---
## 1. TaskManager Pipeline (the unified path)
### Core files
| File | Purpose |
|------|---------|
| `src/core/task_manager/manager.py` | Thin facade composing Graph, EventBus, Lifecycle |
| `src/core/task_manager/graph.py` | In-memory Task registry with CRUD, pagination, filters |
| `src/core/task_manager/lifecycle.py` | State machine: PENDING→RUNNING→SUCCESS/FAILED/WAITING |
| `src/core/task_manager/event_bus.py` | Async log buffer, persistence flush, WebSocket fan-out |
| `src/core/task_manager/context.py` | TaskContext container passed to plugin.execute() |
| `src/core/task_manager/models.py` | Task, TaskStatus, LogEntry, LogFilter (Pydantic) |
| `src/core/plugin_loader.py` | Filesystem-based PluginBase discovery and registration |
| `src/core/plugin_base.py` | ABC PluginBase with id, name, execute, get_schema |
| `src/core/scheduler.py` | APScheduler service (backup, validation, translation jobs) |
| `src/core/async_job_runner.py` | Bridge: sync APScheduler ↔ async event loop |
| `src/dependencies.py` | Singleton factory for TaskManager, PluginLoader, SchedulerService |
| `src/api/routes/tasks.py` | REST API: POST/GET /api/tasks, WebSocket status/logs |
### Pipeline flow
```
POST /api/tasks {"plugin_id": "...", "params": {...}}
│
▼
TaskManager.create_task(plugin_id, params) [manager.py:306]
│
▼
JobLifecycle.create_task(plugin_id, params) [lifecycle.py:96]
├─ PluginLoader.has_plugin(plugin_id) → raise ValueError if missing
├─ Task(plugin_id=..., params=..., status=PENDING)
├─ TaskGraph.add_task(task) [graph.py:125]
├─ TaskPersistenceService.persist_task(task) [lifecycle.py:111]
└─ returns Task object
│
▼
asyncio.create_task( lifecycle._run_task(task_id) ) [manager.py:314]
│
▼
JobLifecycle._run_task(task_id) [lifecycle.py:128]
├─ TaskGraph.get_task(task_id)
├─ PluginLoader.get_plugin(task.plugin_id)
├─ task.status = RUNNING; persisted; broadcast_status
├─ Creates TaskContext(task_id, add_log_fn, params) [context.py:70]
├─ Inspects plugin.execute() signature:
│ └─ If accepts `context`: plugin.execute(params, context=context)
│ If sync: wrapped in asyncio.to_thread()
│ If async: awaited directly
├─ On success: task.result = result; task.status = SUCCESS
├─ On failure: task.status = FAILED
├─ Finally: task.finished_at, flush_task_logs(), persist_task(), broadcast
└─ Additional: broadcasts dataset.updated for "dataset-mapper"/"llm_documentation"
```
### TaskStatus values
- `PENDING`, `RUNNING`, `SUCCESS`, `FAILED`, `AWAITING_MAPPING`, `AWAITING_INPUT`
### Registered plugin_id values (PluginBase subclasses)
All discovered by PluginLoader scanning `backend/src/plugins/`. Classes inheriting `PluginBase` are instantiated and registered by their `id` property:
| plugin_id | PluginBase subclass | Source file |
|-----------|-------------------|-------------|
| `superset-backup` | BackupPlugin | `backup.py` |
| `superset-migration` | MigrationPlugin | `migration.py` |
| `search-datasets` | SearchPlugin | `search.py` |
| `dataset-mapper` | MapperPlugin | `mapper.py` |
| `system-debug` | DebugPlugin | `debug.py` |
| `maintenance_banner_apply` | MaintenanceBannerPlugin | `maintenance_banner.py` |
| `git-integration` | GitPlugin | `git_plugin.py` |
| `llm_dashboard_validation` | DashboardValidationPlugin | `llm_analysis/plugin.py` |
| `llm_documentation` | DocumentationPlugin | `llm_analysis/plugin.py` |
---
## 2. Scheduler Service
SchedulerService (`src/core/scheduler.py`) manages three types of jobs via APScheduler:
| Job Type | Trigger Mechanism | Uses TaskManager? |
|----------|------------------|-------------------|
| Backup (`backup_{env_id}`) | `task_manager.create_task("superset-backup")` via AsyncJobRunner.run() | **Yes** |
| Translation (`translate_{schedule_id}`) | Direct call to execute_scheduled_translation() → TranslationOrchestrator | **No** |
| Validation (`validation_{policy_id}`) | `task_manager.create_task("llm_dashboard_validation")` via AsyncJobRunner.run() | **Yes** |
---
## 3. Translation System (standalone, bypasses TaskManager)
Translation tasks use a **separate, parallel execution pipeline**. They never go through `TaskManager.create_task()` or `PluginBase.execute()`.
### Key files
| File | Purpose |
|------|---------|
| `src/plugins/translate/orchestrator.py` | TranslationOrchestrator — run lifecycle coordination |
| `src/plugins/translate/orchestrator_planner.py` | TranslationPlanner — plan generation |
| `src/plugins/translate/orchestrator_runner.py` | TranslationStageRunner — execution, retry, cancel |
| `src/plugins/translate/orchestrator_sql.py` | SQL INSERT orchestrator |
| `src/plugins/translate/scheduler.py` | TranslationScheduler CRUD + execute_scheduled_translation() |
| `src/api/routes/translate/_run_routes.py` | POST /api/translate/jobs/{job_id}/run |
| `src/api/routes/translate/_schedule_routes.py` | Translation schedule CRUD |
### Translation execution flow
```
POST /api/translate/jobs/{job_id}/run [_run_routes.py:32]
│
▼
TranslationOrchestrator(db, config_manager, username) [orchestrator.py:46]
│
▼
TranslationPlanner.plan_run(job_id) [orchestrator_planner.py]
├─ Creates TranslationRun DB row (status=PENDING)
└─ Returns TranslationRun object
│
▼
asyncio.create_task( _background_execute() ) [_run_routes.py:123]
│ (separate DB session, separate orchestrator)
▼
TranslationOrchestrator.execute_run(bg_run) [orchestrator.py:89]
│
▼
TranslationStageRunner.execute_run(run) [orchestrator_runner.py:45]
│
▼
TranslationExecutionEngine.execute_run(run)
├─ Fetches data from source
├─ Creates batches
├─ Calls LLM for translation (per-batch)
├─ Generates SQL INSERT statements
├─ Submits SQL to Superset SQL Lab
└─ Records results in TranslationRun/TranslationBatch/TranslationRecord DB rows
```
### Translation scheduling flow (also bypasses)
```
SchedulerService.load_schedules() [scheduler.py:69]
├─ Queries TranslationSchedule table for is_active=True
└─ scheduler.add_job(
execute_scheduled_translation, [translate/scheduler.py:278]
CronTrigger(...)
)
│
▼ (on trigger)
execute_scheduled_translation(schedule_id, job_id, ...)
├─ TranslationOrchestrator(db, config_manager, "scheduler")
├─ orch.start_run(job_id=job_id, is_scheduled=True)
├─ orch.execute_run(run) via AsyncJobRunner.run()
└─ TranslationRun.status set in DB
```
### Key differences: TaskManager vs Translation
| Feature | TaskManager (PluginBase) | Translation Runs |
|---------|------------------------|------------------|
| State model | Pydantic `Task` in memory (SQL persistence) | SQLAlchemy `TranslationRun` in DB |
| Logger | `TaskContext.logger` → EventBus → WebSocket push | `TranslationEventLog` → DB rows |
| WebSocket | `/ws/logs/{task_id}` (push) + `/ws/task-events` | `/ws/translate/run/{run_id}` (poll, 1s interval) |
| Execution model | `plugin.execute(params, context)` | `TranslationOrchestrator.execute_run(run)` |
| Pause/Resume | Built-in (AWAITING_INPUT, AWAITING_MAPPING) | Not supported |
| Cancellation | `TaskManager.cancel_task()` | `TranslationStageRunner.cancel_run()` |
| Discovery | PluginLoader filesystem scan | Hardcoded orchestrator class |
| Results in `/reports` | Yes | **No** |
---
## 4. Complete Audit: All Execution Paths Bypassing TaskManager
### 🔴 Critical bypasses (full task execution, NOT in TaskManager)
| # | Path | File:Line | Launcher | Work Done | State Tracking |
|---|------|-----------|----------|-----------|----------------|
| **A1** | Manual translation run | `_run_routes.py:123` | `asyncio.create_task(_background_execute())` | LLM translation, SQL generation, Superset API | `TranslationRun` table (SQLAlchemy) |
| **A2** | Scheduled translation run | `translate/scheduler.py:278` | APScheduler → AsyncJobRunner.run() | LLM translation, SQL generation, Superset API | `TranslationRun` table (SQLAlchemy) |
### 🟡 Semi-bypasses (blocking HTTP, could be TaskManager async)
| # | Path | File:Line | Work Done | Notes |
|---|------|-----------|-----------|-------|
| **D1** | Retry failed batches | `_run_routes.py:140` | LLM calls + SQL gen | Blocks HTTP response, no 202 Accepted |
| **D2** | Retry SQL insert | `_run_routes.py:168` | Superset SQL submit | Blocks HTTP response, no 202 Accepted |
### 🟢 Non-task execution paths (should NOT be in TaskManager)
| # | Path | File:Line | Work Done | Reason for staying out |
|---|------|-----------|-----------|----------------------|
| **A3** | Agent LLM title generation | `agent/app.py:488` | `asyncio.create_task(generate_llm_title(...))` | Best-effort, sub-second, non-critical. No business state beyond title text. |
| **B1** | EventBus async flusher | `event_bus.py:72` | Flush log buffer to DB every 2s | Internal TaskManager infrastructure |
| **B2** | TaskLogger fire-forget log writes | `task_logger.py:100` | Async log delivery to EventBus | Internal TaskManager infrastructure |
| **C1-C5** | WebSocket event consumers | `app.py:664,770,824,866,895` | Relay events to browser clients | Event relay, not task execution |
| **E** | Thread pool executors | `utils/executors.py:50` | 3× ThreadPoolExecutor | Infra for blocking I/O offloading |
---
## 5. Impact Summary
```
TaskManager (unified)
├─ backup ✅
├─ migration ✅
├─ llm_validation ✅
├─ llm_documentation ✅
├─ dataset-mapper ✅
├─ search-datasets ✅
├─ git-integration ✅
├─ maintenance ✅
├─ debug ✅
│
└─ translation ❌ ← bypasses completely (A1 + A2)
dataset review? need to verify
```
### If translation were unified:
1. Create `TranslatePlugin extends PluginBase` with `id = "translate-run"` or similar
2. Its `execute(params, context)` method would:
- Receive `job_id` and `run_id` from params
- Open its own DB session
- Call `TranslationOrchestrator(db, ...).execute_run(run)`
- Report progress via `context.logger` (→ automatic WebSocket push)
3. The scheduler would call `task_manager.create_task("translate-run", {schedule_id, job_id})` instead of `execute_scheduled_translation()`
4. Manual POST would call `task_manager.create_task()` instead of the orchestrator directly
5. Translation tasks would automatically appear in `/reports` with log streaming, status broadcasts, and cancel support
### Key benefit:
- Single API: `POST /api/tasks` for ALL background work
- Single monitoring: `/reports` shows ALL tasks including translations
- Unified WebSocket: push-based logs instead of poll-based
- Elimination of ~200 lines of duplicated concurrency/DB-session/stale-run cleanup code