diff --git a/backend/src/core/superset_client/_datasets.py b/backend/src/core/superset_client/_datasets.py index c2355bcf..ad34991e 100644 --- a/backend/src/core/superset_client/_datasets.py +++ b/backend/src/core/superset_client/_datasets.py @@ -51,6 +51,36 @@ class SupersetDatasetsMixin: ) return result # #endregion SupersetClientGetDatasetsSummary + # #region SupersetClientGetDatasetLinkedDashboardCount [TYPE Function] [C:2] + # @PURPOSE: Fetch the number of dashboards linked to a dataset via related_objects endpoint. + # @RELATION: CALLS -> [APIClient] + # @RATIONALE Added to fix linked_count=0 in StatsBar. Reuses the same /dataset/{id}/related_objects call + # that get_dataset_detail() was already making, but as a lightweight count-only call. + # @REJECTED Enriching get_datasets_summary() with more columns rejected — Superset list endpoint + # doesn't include linked_dashboard_count (it's a computed field from related_objects). + def get_dataset_linked_dashboard_count(self, dataset_id: int) -> int: + with belief_scope("get_dataset_linked_dashboard_count", f"id={dataset_id}"): + try: + related_objects = self.network.request( + method="GET", endpoint=f"/dataset/{dataset_id}/related_objects" + ) + if isinstance(related_objects, dict): + if "dashboards" in related_objects: + dashboards_data = related_objects["dashboards"] + elif "result" in related_objects and isinstance( + related_objects["result"], dict + ): + dashboards_data = related_objects["result"].get("dashboards", []) + else: + dashboards_data = [] + return len(dashboards_data) + return 0 + except Exception as e: + app_logger.warning( + f"[get_dataset_linked_dashboard_count][Warning] Failed to fetch related dashboards for dataset {dataset_id}: {e}" + ) + return 0 + # #endregion SupersetClientGetDatasetLinkedDashboardCount # #region SupersetClientGetDatasetDetail [TYPE Function] [C:3] # @PURPOSE: Fetches detailed dataset information including columns and linked dashboards. # @RELATION: CALLS -> [SupersetClientGetDataset] diff --git a/backend/src/services/resource_service.py b/backend/src/services/resource_service.py index 5566afff..194a98b0 100644 --- a/backend/src/services/resource_service.py +++ b/backend/src/services/resource_service.py @@ -9,6 +9,7 @@ # @SIDE_EFFECT: Queries multiple backends for status # @DATA_CONTRACT: ResourceQuery -> ResourceStatusSummary +import asyncio from datetime import UTC, datetime from typing import Any @@ -287,9 +288,14 @@ class ResourceService: # @POST: Returns list of datasets with enhanced metadata # @PARAM: env (Environment) - The environment to fetch from # @PARAM: tasks (List[Task]) - List of tasks to check for status - # @RETURN: List[Dict] - Datasets with mapped_fields and last_task fields + # @RETURN: List[Dict] - Datasets with mapped_fields, last_task and linked_dashboard_count fields # @RELATION: CALLS -> [SupersetClientGetDatasetsSummary] + # @RELATION: CALLS -> [SupersetClientGetDatasetLinkedDashboardCount] # @RELATION: CALLS ->[_get_last_task_for_resource] + # @RATIONALE linked_dashboard_count was missing — get_datasets_summary() only returns id/table_name/schema/database + # StatsBar showed linked_count=0 because the field was never populated in list endpoint. + # Fix: fetch /dataset/{id}/related_objects for each dataset concurrently (semaphore=3, timeout=10s). + # @REJECTED N+1 blocking calls rejected — would timeout at 26×1s. Semaphore + asyncio.to_thread + timeout prevents overload. async def get_datasets_with_status( self, env: Any, @@ -299,10 +305,28 @@ class ResourceService: client = SupersetClient(env) datasets = client.get_datasets_summary() - # Enhance each dataset with task status + # Enhance each dataset with task status and linked dashboard count + sem = asyncio.Semaphore(3) # limit concurrent Superset calls + + async def fetch_linked_count(ds_id: int) -> int: + async with sem: + try: + return await asyncio.wait_for( + asyncio.to_thread( + client.get_dataset_linked_dashboard_count, ds_id + ), + timeout=10.0, + ) + except (asyncio.TimeoutError, Exception): + logger.warning( + f"[get_datasets_with_status][Warning] " + f"Failed to fetch linked dashboard count for dataset {ds_id}" + ) + return 0 + result = [] + linked_tasks = [] for dataset in datasets: - # dataset is already a dict, no need to call .dict() dataset_dict = dataset dataset_id = dataset_dict.get('id') @@ -314,8 +338,18 @@ class ResourceService: dataset_dict['last_task'] = last_task result.append(dataset_dict) + linked_tasks.append(fetch_linked_count(dataset_id)) - logger.info(f"[ResourceService][Coherence:OK] Fetched {len(result)} datasets with status") + # Fetch linked dashboard counts concurrently — FR-022 + if linked_tasks: + linked_counts = await asyncio.gather(*linked_tasks) + for ds, count in zip(result, linked_counts): + ds['linked_dashboard_count'] = count + + logger.info( + f"[ResourceService][Coherence:OK] " + f"Fetched {len(result)} datasets with status and linked counts" + ) return result # endregion get_datasets_with_status diff --git a/frontend/src/routes/datasets/DatasetList.svelte b/frontend/src/routes/datasets/DatasetList.svelte index 7ec59ac5..f590ae04 100644 --- a/frontend/src/routes/datasets/DatasetList.svelte +++ b/frontend/src/routes/datasets/DatasetList.svelte @@ -1,6 +1,7 @@ - + + @@ -76,7 +77,7 @@ {#if isLoading}
- {#each Array(3) as _} + {#each Array(3) as _, _i (_i)}
@@ -92,25 +93,25 @@
{$t.datasets?.empty}
{:else} -
- {#each datasets as dataset} +
+ {#each datasets as dataset (dataset.id)}
handleCardClick(dataset, e)} role="button" tabindex="0" > -
+
handleCheckboxChange(dataset, e)} />
-
{dataset.table_name}
+
{dataset.table_name}
{dataset.schema ?? '-'} {dataset.column_count ?? 0} {$t.datasets?.columns?.toLowerCase() ?? 'columns'} @@ -120,8 +121,8 @@
{#if dataset.mappedFields?.total} -
-
+
+
-
+
diff --git a/frontend/src/routes/datasets/StatsBar.svelte b/frontend/src/routes/datasets/StatsBar.svelte index 567ce5e2..c988e0f8 100644 --- a/frontend/src/routes/datasets/StatsBar.svelte +++ b/frontend/src/routes/datasets/StatsBar.svelte @@ -1,16 +1,16 @@ - - + + + - - - + + + - + -
+
{#each tiles as tile (tile.key)} {#if isLoading} -
-
-
-
+
{:else} {/if} {/each}