busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

032: fix remaining sync→async propagation (17 call sites)

Browse Source 
Core fixes:
  - service_datasource.py: fetch_datasource_metadata() → async
  - service.py: create_job(), update_job() → async (callers await)
  - _job_routes.py: await create_job/update_job

Maintenance scanners:
  - _dashboard_scanner.py: 4 functions → async (find_affected, _get_linked,
    _apply_filters, _resolve_title)
  - _chart_manager.py: 3 functions → async
  - _banner_renderer.py: rebuild_banner → async
  - _orchestrators.py: 3 orchestrators → async
  - maintenance_banner.py: await async calls

Migration:
  - dry_run_orchestrator.py: run(), _build_target_signatures() → async
  - risk_assessor.py: build_risks() → async
  - migration.py: await service.run()
  - mapping_service.py: sync_environment() → async

Dead code:
  - _helpers.py: _find_dashboard_id_by_slug marked DEPRECATED
This commit is contained in:
busya
2026-06-05 08:56:37 +03:00
parent a5cd06546f
commit c7d8f4431e
13 changed files with 67 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/api/routes/dashboards/_helpers.py
View File

@@ -20,6 +20,10 @@ def _find_dashboard_id_by_slug(
client: SupersetClient,
dashboard_slug: str,
) -> int | None:
"""@DEPRECATED — dead code, use _find_dashboard_id_by_slug_async."""
# This function is no longer called (only _async version is used).
# It remains as a reference but will raise AttributeError at runtime
# because SupersetClient methods are now async.
query_variants = [
{
"filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}],

2
backend/src/api/routes/migration.py
View File

@@ -184,7 +184,7 @@ async def dry_run_migration(
target_client = AsyncSupersetClient(target_env)
try:
result = service.run(
result = await service.run(
selection=selection,
source_client=source_client,
target_client=target_client,

4
backend/src/api/routes/translate/_job_routes.py
View File

@@ -106,7 +106,7 @@ async def create_job(
logger.reason(f"create_job — User: {current_user.username}", extra={"src": "translate_routes"})
try:
service = TranslateJobService(db, config_manager, current_user.username)
job = service.create_job(payload)
job = await service.create_job(payload)
dict_ids = service.get_job_dictionary_ids(job.id)
return job_to_response(job, dict_ids)
except ValueError as e:
@@ -132,7 +132,7 @@ async def update_job(
logger.reason(f"update_job — Job: {job_id}, User: {current_user.username}", extra={"src": "translate_routes"})
try:
service = TranslateJobService(db, config_manager, current_user.username)
job = service.update_job(job_id, payload)
job = await service.update_job(job_id, payload)
dict_ids = service.get_job_dictionary_ids(job.id)
return job_to_response(job, dict_ids)
except ValueError as e:

7
backend/src/core/mapping_service.py
View File

@@ -73,7 +73,8 @@ class IdMappingService:
for env_id in environments:
client = superset_client_factory(env_id)
if client:
self.sync_environment(env_id, client)
import asyncio
asyncio.run(self.sync_environment(env_id, client))
self._sync_job = self.scheduler.add_job(
sync_all,
CronTrigger.from_crontab(cron_string),
@@ -96,7 +97,7 @@ class IdMappingService:
# @PARAM superset_client - Instance capable of hitting the Superset API.
# @PRE environment_id exists in the database.
# @POST ResourceMapping records for the environment are created or updated.
def sync_environment(
async def sync_environment(
self, environment_id: str, superset_client, incremental: bool = False
) -> None:
"""
@@ -150,7 +151,7 @@ class IdMappingService:
logger.debug(
f"[IdMappingService.sync_environment] Incremental sync since {since_dttm}"
)
resources = superset_client.get_all_resources(
resources = await superset_client.get_all_resources(
endpoint, since_dttm=since_dttm
)
# Track which UUIDs we see in this sync cycle

22
backend/src/core/migration/dry_run_orchestrator.py
View File

@@ -54,7 +54,7 @@ class MigrationDryRunService:
# @PRE source/target clients are authenticated and selection validated by caller.
# @POST Returns JSON-serializable pre-flight payload with summary, diff and risk.
# @SIDE_EFFECT Reads source export archives and target metadata via network.
def run(
async def run(
self,
selection: DashboardSelection,
source_client: SupersetClient,
@@ -73,14 +73,14 @@ class MigrationDryRunService:
else {}
)
transformed = {"dashboards": {}, "charts": {}, "datasets": {}}
dashboards_preview = source_client.get_dashboards_summary()
dashboards_preview = await source_client.get_dashboards_summary()
selected_preview = {
item["id"]: item
for item in dashboards_preview
if item.get("id") in selection.selected_ids
}
for dashboard_id in selection.selected_ids:
exported_content, _ = source_client.export_dashboard(int(dashboard_id))
exported_content, _ = await source_client.export_dashboard(int(dashboard_id))
with create_temp_file(
content=exported_content, suffix=".zip"
) as source_zip, create_temp_file(suffix=".zip") as transformed_zip:
@@ -103,7 +103,7 @@ class MigrationDryRunService:
source_objects = {
key: list(value.values()) for key, value in transformed.items()
}
target_objects = self._build_target_signatures(target_client)
target_objects = await self._build_target_signatures(target_client)
diff = {
"dashboards": self._build_object_diff(
source_objects["dashboards"], target_objects["dashboards"]
@@ -115,7 +115,7 @@ class MigrationDryRunService:
source_objects["datasets"], target_objects["datasets"]
),
}
risk = self._build_risks(
risk = await self._build_risks(
source_objects, target_objects, diff, target_client
)
summary = {
@@ -219,10 +219,10 @@ class MigrationDryRunService:
# #endregion _build_object_diff
# #region _build_target_signatures [TYPE Function]
# @PURPOSE: Pull target metadata and normalize it into comparable signatures.
def _build_target_signatures(
async def _build_target_signatures(
self, client: SupersetClient
) -> dict[str, list[dict[str, Any]]]:
_, dashboards = client.get_dashboards(
_, dashboards = await client.get_dashboards(
query={
"columns": [
"uuid",
@@ -235,7 +235,7 @@ class MigrationDryRunService:
],
}
)
_, datasets = client.get_datasets(
_, datasets = await client.get_datasets(
query={
"columns": [
"uuid",
@@ -248,7 +248,7 @@ class MigrationDryRunService:
],
}
)
_, charts = client.get_charts(
_, charts = await client.get_charts(
query={
"columns": [
"uuid",
@@ -330,14 +330,14 @@ class MigrationDryRunService:
# #endregion _build_target_signatures
# #region _build_risks [TYPE Function]
# @PURPOSE: Build risk items for missing datasource, broken refs, overwrite, owner mismatch.
def _build_risks(
async def _build_risks(
self,
source_objects: dict[str, list[dict[str, Any]]],
target_objects: dict[str, list[dict[str, Any]]],
diff: dict[str, dict[str, list[dict[str, Any]]]],
target_client: SupersetClient,
) -> list[dict[str, Any]]:
return build_risks(source_objects, target_objects, diff, target_client)
return await build_risks(source_objects, target_objects, diff, target_client)
# #endregion _build_risks
# #endregion MigrationDryRunService
# #endregion MigrationDryRunOrchestratorModule

4
backend/src/core/migration/risk_assessor.py
View File

@@ -96,7 +96,7 @@ def extract_owner_identifiers(owners: Any) -> list[str]:
# @DATA_CONTRACT Dict[str, Dict[str, List[Dict[str, Any]]]],
# @DATA_CONTRACT SupersetClient
# @DATA_CONTRACT ) -> List[Dict[str, Any]]
def build_risks(
async def build_risks(
source_objects: dict[str, list[dict[str, Any]]],
target_objects: dict[str, list[dict[str, Any]]],
diff: dict[str, dict[str, list[dict[str, Any]]]],
@@ -118,7 +118,7 @@ def build_risks(
)
target_dataset_uuids = set(index_by_uuid(target_objects["datasets"]).keys())
_, target_databases = target_client.get_databases(query={"columns": ["uuid"]})
_, target_databases = await target_client.get_databases(query={"columns": ["uuid"]})
target_database_uuids = {
str(item.get("uuid")) for item in target_databases if item.get("uuid")
}

6
backend/src/plugins/maintenance_banner.py
View File

@@ -143,13 +143,13 @@ class MaintenanceBannerPlugin(PluginBase):
if operation == "start":
if not event_id:
return {"status": "failed", "error": "event_id required for start"}
result = start_maintenance(event_id, db, superset_client)
result = await start_maintenance(event_id, db, superset_client)
elif operation == "end":
if not event_id:
return {"status": "failed", "error": "event_id required for end"}
result = end_maintenance(event_id, db, superset_client)
result = await end_maintenance(event_id, db, superset_client)
elif operation == "end_all":
result = end_all_maintenance(db, superset_client)
result = await end_all_maintenance(db, superset_client)
else:
result = {"status": "failed", "error": f"Unknown operation: {operation}"}

8
backend/src/plugins/translate/service.py
View File

@@ -64,7 +64,7 @@ class TranslateJobService:
# region create_job [TYPE Function]
# @PURPOSE: Create a new translation job with column validation.
# @SIDE_EFFECT Validates columns via SupersetClient; caches database_dialect.
def create_job(self, payload: TranslateJobCreate) -> TranslationJob:
async def create_job(self, payload: TranslateJobCreate) -> TranslationJob:
logger.info(f"[TranslateJobService] Creating job '{payload.name}'")
if payload.source_datasource_id and not payload.translation_column:
raise ValueError("A translation column is required when a datasource is selected")
@@ -80,7 +80,7 @@ class TranslateJobService:
if not dialect:
try:
env_id = payload.environment_id or payload.source_dialect
_, detected_dialect = fetch_datasource_metadata(
_, detected_dialect = await fetch_datasource_metadata(
int(payload.source_datasource_id), env_id, self.config_manager,
)
dialect = detected_dialect
@@ -132,7 +132,7 @@ class TranslateJobService:
# region update_job [TYPE Function]
# @PURPOSE: Update an existing translation job.
# @SIDE_EFFECT Re-detects database_dialect if source_datasource_id changed.
def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob:
async def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob:
logger.info(f"[TranslateJobService] Updating job '{job_id}'")
job = self.get_job(job_id)
update_data = payload.model_dump(exclude_unset=True)
@@ -156,7 +156,7 @@ class TranslateJobService:
if payload.source_datasource_id and not payload.database_dialect:
try:
env_id = (payload.environment_id or payload.source_dialect or job.environment_id or job.source_dialect)
_, detected_dialect = fetch_datasource_metadata(
_, detected_dialect = await fetch_datasource_metadata(
int(payload.source_datasource_id), env_id, self.config_manager,
)
job.database_dialect = detected_dialect

6
backend/src/plugins/translate/service_datasource.py
View File

@@ -57,7 +57,7 @@ def get_dialect_from_database(database_record: dict[str, Any]) -> str:
# #region fetch_datasource_metadata [C:3] [TYPE Function]
# @BRIEF Fetch datasource columns and database dialect from Superset.
def fetch_datasource_metadata(
async def fetch_datasource_metadata(
dataset_id: int,
env_id: str,
config_manager: ConfigManager,
@@ -69,7 +69,7 @@ def fetch_datasource_metadata(
raise ValueError(f"Superset environment '{env_id}' not found in configuration")
client = SupersetClient(env_config)
dataset_detail = client.get_dataset_detail(dataset_id)
dataset_detail = await client.get_dataset_detail(dataset_id)
raw_columns = dataset_detail.get("columns", [])
columns = []
@@ -92,7 +92,7 @@ def fetch_datasource_metadata(
try:
db_id = dataset_detail.get("database_id")
if db_id:
db_record = client.get_database(int(db_id))
db_record = await client.get_database(int(db_id))
db_result = db_record.get("result", db_record) if isinstance(db_record, dict) else db_record
dialect = get_dialect_from_database(db_result)
else:

4
backend/src/services/maintenance/_banner_renderer.py
View File

@@ -171,7 +171,7 @@ def _build_banner_text_for_dashboard(
# @SIDE_EFFECT Modifies Superset chart. Writes DB.
# @RELATION DEPENDS_ON -> [build_banner_text]
# @RELATION DEPENDS_ON -> [SupersetDashboardsWriteMixin]
def rebuild_banner(
async def rebuild_banner(
banner_id: str,
db_session: Session,
superset_client: SupersetClient,
@@ -225,7 +225,7 @@ def rebuild_banner(
# Update banner on dashboard (native MARKDOWN in layout)
if banner.chart_id:
try:
superset_client.update_banner_on_dashboard(
await superset_client.update_banner_on_dashboard(
banner.dashboard_id, banner.chart_id, banner_text
)
except Exception as e:

26
backend/src/services/maintenance/_chart_manager.py
View File

@@ -29,7 +29,7 @@ from ._dashboard_scanner import _resolve_dashboard_title
# @POST Returns MaintenanceDashboardBanner with chart_id set. Chart is placed at top of dashboard.
# @SIDE_EFFECT Creates chart in Superset, modifies dashboard layout, writes DB row.
# @RELATION DEPENDS_ON -> [SupersetDashboardsWriteMixin]
def ensure_banner_chart(
async def ensure_banner_chart(
dashboard_id: int,
environment_id: str,
superset_client: SupersetClient,
@@ -61,7 +61,7 @@ def ensure_banner_chart(
)
# Verify the chart still exists in Superset
try:
superset_client.get_chart(existing.chart_id)
await superset_client.get_chart(existing.chart_id)
app_logger.reflect(
"Existing banner chart is alive, reusing",
extra={"banner_id": existing.id, "chart_id": existing.chart_id},
@@ -81,7 +81,7 @@ def ensure_banner_chart(
# Create markdown chart in Superset
try:
chart_id = superset_client.create_markdown_chart(
chart_id = await superset_client.create_markdown_chart(
dashboard_id, "*Maintenance banner placeholder*"
)
except Exception as e:
@@ -94,7 +94,7 @@ def ensure_banner_chart(
# Insert chart at top of dashboard layout
try:
superset_client.update_dashboard_layout(dashboard_id, chart_id)
await superset_client.update_dashboard_layout(dashboard_id, chart_id)
except Exception as e:
app_logger.explore(
"Failed to update dashboard layout, deleting chart",
@@ -103,7 +103,7 @@ def ensure_banner_chart(
)
# Clean up the chart we just created
try:
superset_client.delete_chart(chart_id)
await superset_client.delete_chart(chart_id)
except Exception:
pass
raise
@@ -138,7 +138,7 @@ def ensure_banner_chart(
# @RELATION CALLS -> [ensure_banner_chart]
# @RELATION CALLS -> [_build_banner_text_for_dashboard]
# @RELATION CALLS -> [_resolve_dashboard_title]
def _process_dashboards_for_start(
async def _process_dashboards_for_start(
dashboard_ids: list[int],
event_id: str,
environment_id: str,
@@ -158,7 +158,7 @@ def _process_dashboards_for_start(
)
try:
# Ensure banner chart exists
banner = ensure_banner_chart(
banner = await ensure_banner_chart(
dash_id,
environment_id,
superset_client,
@@ -184,7 +184,7 @@ def _process_dashboards_for_start(
# Update banner content on dashboard (native MARKDOWN in layout)
if banner.chart_id:
try:
superset_client.update_banner_on_dashboard(
await superset_client.update_banner_on_dashboard(
dash_id, banner.chart_id, banner_text
)
except Exception as e:
@@ -207,7 +207,7 @@ def _process_dashboards_for_start(
db_session.flush()
# Resolve dashboard title
title = _resolve_dashboard_title(dash_id, superset_client)
title = await _resolve_dashboard_title(dash_id, superset_client)
successful_dashboards.append(
{
"id": dash_id,
@@ -239,7 +239,7 @@ def _process_dashboards_for_start(
# still use same banner → rebuild; if no other events → remove banner (chart + layout).
# @SIDE_EFFECT Modifies Superset dashboards (removes charts, updates layouts). Writes DB.
# @RELATION CALLS -> [rebuild_banner]
def _process_states_for_end(
async def _process_states_for_end(
states: list[MaintenanceDashboardState],
event_id: str,
superset_client: SupersetClient,
@@ -278,7 +278,7 @@ def _process_states_for_end(
# Other events still active — rebuild banner text
state.status = MaintenanceDashboardStateStatus.REMOVED
db_session.flush()
rebuild_banner(banner_id, db_session, superset_client)
await rebuild_banner(banner_id, db_session, superset_client)
app_logger.reflect(
"Banner rebuilt after removing event",
extra={"banner_id": banner_id, "dashboard_id": dash_id},
@@ -291,7 +291,7 @@ def _process_states_for_end(
# Remove chart from layout
if banner.chart_id:
try:
superset_client.remove_chart_from_layout(
await superset_client.remove_chart_from_layout(
dash_id, banner.chart_id
)
except Exception as e:
@@ -311,7 +311,7 @@ def _process_states_for_end(
# Delete chart
try:
superset_client.delete_chart(banner.chart_id)
await superset_client.delete_chart(banner.chart_id)
except Exception as e:
app_logger.explore(
"Failed to delete chart",

22
backend/src/services/maintenance/_dashboard_scanner.py
View File

@@ -23,7 +23,7 @@ from ..sql_table_extractor import extract_tables_from_sql
# @SIDE_EFFECT Fetches all datasets from Superset (paginated). May be slow with many datasets.
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SqlTableExtractorModule]
def find_affected_dashboards(
async def find_affected_dashboards(
tables: list[str],
superset_client: SupersetClient,
settings: MaintenanceSettings | None = None,
@@ -48,7 +48,7 @@ def find_affected_dashboards(
# Fetch all datasets from Superset
try:
_, datasets = superset_client.get_datasets()
_, datasets = await superset_client.get_datasets()
except Exception as e:
app_logger.explore(
"Failed to fetch datasets from Superset",
@@ -78,7 +78,7 @@ def find_affected_dashboards(
if not is_virtual and ds_schema and ds_table:
qualified = f"{ds_schema}.{ds_table}"
if qualified in target_tables:
linked = _get_linked_dashboards(superset_client, ds_id)
linked = await _get_linked_dashboards(superset_client, ds_id)
matched_dashboard_ids.update(linked)
continue
@@ -86,7 +86,7 @@ def find_affected_dashboards(
if is_virtual and ds_sql:
extracted = extract_tables_from_sql(ds_sql)
if extracted & target_tables:
linked = _get_linked_dashboards(superset_client, ds_id)
linked = await _get_linked_dashboards(superset_client, ds_id)
matched_dashboard_ids.update(linked)
app_logger.reflect(
@@ -95,7 +95,7 @@ def find_affected_dashboards(
)
# Apply filtering from settings
result = _apply_dashboard_filters(
result = await _apply_dashboard_filters(
list(matched_dashboard_ids), superset_client, settings
)
@@ -110,14 +110,14 @@ def find_affected_dashboards(
# #region _get_linked_dashboards [C:2] [TYPE Function]
# @BRIEF Fetch linked dashboards for a dataset from Superset.
# @SIDE_EFFECT Calls Superset API.
def _get_linked_dashboards(
async def _get_linked_dashboards(
superset_client: SupersetClient,
dataset_id: int | None,
) -> set[int]:
if not dataset_id:
return set()
try:
detail = superset_client.get_dataset_detail(dataset_id)
detail = await superset_client.get_dataset_detail(dataset_id)
linked = detail.get("linked_dashboards", [])
return {int(d["id"]) for d in linked if d.get("id")}
except Exception as e:
@@ -132,7 +132,7 @@ def _get_linked_dashboards(
# #region _apply_dashboard_filters [C:2] [TYPE Function]
# @BRIEF Apply scope (published/draft/all), excluded, and forced filters from settings.
def _apply_dashboard_filters(
async def _apply_dashboard_filters(
dashboard_ids: list[int],
superset_client: SupersetClient,
settings: MaintenanceSettings | None,
@@ -147,7 +147,7 @@ def _apply_dashboard_filters(
# Apply scope filtering
if settings.dashboard_scope != DashboardScope.ALL:
try:
_, all_dashboards = superset_client.get_dashboards()
_, all_dashboards = await superset_client.get_dashboards()
scope = settings.dashboard_scope
filtered: list[int] = []
for did in dashboard_ids:
@@ -186,12 +186,12 @@ def _apply_dashboard_filters(
# #region _resolve_dashboard_title [C:2] [TYPE Function]
# @BRIEF Resolve dashboard title from Superset by ID.
def _resolve_dashboard_title(
async def _resolve_dashboard_title(
dashboard_id: int,
superset_client: SupersetClient,
) -> str:
try:
_, dashboards = superset_client.get_dashboards()
_, dashboards = await superset_client.get_dashboards()
for d in dashboards:
if d.get("id") == dashboard_id:
return d.get("dashboard_title") or d.get("title") or str(dashboard_id)

18
backend/src/services/maintenance/_orchestrators.py
View File

@@ -56,16 +56,12 @@ def build_idempotency_key(
# @BELIEF Uses belief_scope with REASON/REFLECT/EXPLORE markers.
# @RELATION CALLS -> [find_affected_dashboards]
# @RELATION CALLS -> [_process_dashboards_for_start]
def start_maintenance(
async def start_maintenance(
event_id: str,
db_session: Session,
superset_client: SupersetClient,
) -> dict[str, Any]:
with belief_scope("start_maintenance", f"event_id={event_id}"):
app_logger.reason(
"Starting maintenance event",
extra={"event_id": event_id},
)
# Load event
event = db_session.query(MaintenanceEvent).filter(
@@ -100,7 +96,7 @@ def start_maintenance(
# Find affected dashboards
try:
dashboard_ids = find_affected_dashboards(
dashboard_ids = await find_affected_dashboards(
event.tables,
superset_client,
settings,
@@ -142,7 +138,7 @@ def start_maintenance(
}
# Process each dashboard
successful_dashboards, failed_dashboards, _ = _process_dashboards_for_start(
successful_dashboards, failed_dashboards, _ = await _process_dashboards_for_start(
dashboard_ids,
event_id,
environment_id,
@@ -193,7 +189,7 @@ def start_maintenance(
# @SIDE_EFFECT Modifies Superset dashboards (removes charts, updates layouts). Writes DB.
# @BELIEF Uses belief_scope with REASON/REFLECT/EXPLORE markers.
# @RELATION CALLS -> [_process_states_for_end]
def end_maintenance(
async def end_maintenance(
event_id: str,
db_session: Session,
superset_client: SupersetClient,
@@ -234,7 +230,7 @@ def end_maintenance(
).all()
# Delegate per-state processing
removed_count, failed_removals = _process_states_for_end(
removed_count, failed_removals = await _process_states_for_end(
states, event_id, superset_client, db_session,
)
@@ -268,7 +264,7 @@ def end_maintenance(
# @SIDE_EFFECT Modifies multiple Superset dashboards. Writes multiple DB rows.
# @BELIEF Uses belief_scope with REASON/REFLECT/EXPLORE markers.
# @RELATION CALLS -> [end_maintenance]
def end_all_maintenance(
async def end_all_maintenance(
db_session: Session,
superset_client: SupersetClient,
) -> dict[str, Any]:
@@ -302,7 +298,7 @@ def end_all_maintenance(
extra={"event_id": event.id},
)
try:
result = end_maintenance(
result = await end_maintenance(
event.id, db_session, superset_client
)
total_removed += result.get("removed_from", 0)