feat(030): Dataset Lifecycle Workspace — Stats Bar, split-view, inline-edit, bulk actions
Backend: - Add MetricItem, StatsCounts, ColumnDescriptionUpdate, MetricDescriptionUpdate DTOs - Add metric_count to DatasetItem and DatasetDetailResponse - Add server-side filtering via ?filter=unmapped|mapped|linked|all - Add PUT endpoints for column/metric description inline-edit - Add HTML stripping validation (max 2000 chars, plain text) - Add WebSocket dataset.updated event on task completion - RBAC: plugin:migration:WRITE for mutations, READ for reads - 14 pytest tests (stats, filter, metrics, inline-edit, validation, 502/503) Frontend: - Rewrite +page.svelte as split-view orchestrator (387 lines) - StatsBar.svelte — 4 metric tiles with aria-pressed, server-side filter dispatch - DatasetList.svelte — card grid with progress bars, checkboxes, search - DatasetPreview.svelte — presentational detail panel (container fetches) - ColumnsTable.svelte — inline-edit with type chips, bold/italic fallback - MetricsTable.svelte — inline-edit with expression hints - 52 vitest tests across 7 test files - i18n: 11 EN + 11 RU keys (with_mapping/с_маппингом) Spec: - 15 issues resolved (semantic label, filtering, endpoints, conflict, metric_count, RBAC, ID types, ports, i18n, validation, a11y, propagation, resize, perf, ownership) Status: 14/14 backend tests pass, 52/52 frontend tests pass, build succeeds Risk: Low — T051 browser validation pending (non-blocking)
This commit is contained in:
@@ -13,7 +13,9 @@
|
||||
# @INVARIANT: All dataset responses include last_task metadata
|
||||
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
import re
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from ...core.logger import belief_scope, logger
|
||||
@@ -45,6 +47,7 @@ class DatasetItem(BaseModel):
|
||||
database: str
|
||||
mapped_fields: MappedFields | None = None
|
||||
last_task: LastTask | None = None
|
||||
metric_count: int = 0
|
||||
|
||||
class Config:
|
||||
allow_population_by_field_name = True
|
||||
@@ -69,8 +72,21 @@ class DatasetColumn(BaseModel):
|
||||
description: str | None = None
|
||||
# #endregion DatasetColumn
|
||||
|
||||
# #region DatasetDetailResponse [C:1] [TYPE DataClass]
|
||||
# @BRIEF Detailed DTO for a dataset including columns and links
|
||||
# #region MetricItem [C:1] [TYPE DataClass]
|
||||
# @BRIEF Pydantic DTO for a dataset metric — carries Superset metric metadata.
|
||||
# @RELATION DEPENDS_ON -> [DatasetDetailResponse]
|
||||
class MetricItem(BaseModel):
|
||||
id: int
|
||||
metric_name: str
|
||||
expression: str | None = None
|
||||
verbose_name: str | None = None
|
||||
description: str | None = None
|
||||
metric_type: str | None = None
|
||||
# #endregion MetricItem
|
||||
|
||||
# #region DatasetDetailResponse [C:2] [TYPE DataClass]
|
||||
# @BRIEF Detailed DTO for a dataset including columns, linked dashboards, and metrics.
|
||||
# @RELATION DEPENDS_ON -> [MetricItem]
|
||||
class DatasetDetailResponse(BaseModel):
|
||||
id: int
|
||||
table_name: str | None = None
|
||||
@@ -79,6 +95,8 @@ class DatasetDetailResponse(BaseModel):
|
||||
description: str | None = None
|
||||
columns: list[DatasetColumn]
|
||||
column_count: int
|
||||
metrics: list[MetricItem] = []
|
||||
metric_count: int = 0
|
||||
sql: str | None = None
|
||||
linked_dashboards: list[LinkedDashboard]
|
||||
linked_dashboard_count: int
|
||||
@@ -90,10 +108,21 @@ class DatasetDetailResponse(BaseModel):
|
||||
allow_population_by_field_name = True
|
||||
# #endregion DatasetDetailResponse
|
||||
|
||||
# #region DatasetsResponse [C:1] [TYPE DataClass]
|
||||
# @BRIEF Paginated response DTO for dataset listings
|
||||
# #region StatsCounts [C:1] [TYPE DataClass]
|
||||
# @BRIEF Aggregate statistics for the Stats Bar — computed from full dataset list.
|
||||
class StatsCounts(BaseModel):
|
||||
total: int
|
||||
unmapped_count: int
|
||||
mapped_count: int
|
||||
linked_count: int
|
||||
# #endregion StatsCounts
|
||||
|
||||
# #region DatasetsResponse [C:2] [TYPE DataClass]
|
||||
# @BRIEF Paginated response DTO for dataset listings with stats.
|
||||
# @RELATION DEPENDS_ON -> [StatsCounts]
|
||||
class DatasetsResponse(BaseModel):
|
||||
datasets: list[DatasetItem]
|
||||
stats: StatsCounts
|
||||
total: int
|
||||
page: int
|
||||
page_size: int
|
||||
@@ -106,6 +135,18 @@ class TaskResponse(BaseModel):
|
||||
task_id: str
|
||||
# #endregion TaskResponse
|
||||
|
||||
# #region ColumnDescriptionUpdate [C:1] [TYPE DataClass]
|
||||
# @BRIEF Request DTO for inline-edit of a column description.
|
||||
class ColumnDescriptionUpdate(BaseModel):
|
||||
description: str
|
||||
# #endregion ColumnDescriptionUpdate
|
||||
|
||||
# #region MetricDescriptionUpdate [C:1] [TYPE DataClass]
|
||||
# @BRIEF Request DTO for inline-edit of a metric description.
|
||||
class MetricDescriptionUpdate(BaseModel):
|
||||
description: str
|
||||
# #endregion MetricDescriptionUpdate
|
||||
|
||||
# #region get_dataset_ids [C:4] [TYPE Function]
|
||||
# @BRIEF Fetch list of all dataset IDs from a specific environment (without pagination)
|
||||
# @PRE: env_id must be a valid environment ID
|
||||
@@ -155,17 +196,19 @@ async def get_dataset_ids(
|
||||
# #endregion get_dataset_ids
|
||||
|
||||
# #region get_datasets [C:4] [TYPE Function]
|
||||
# @BRIEF Fetch list of datasets from a specific environment with mapping progress
|
||||
# @BRIEF Fetch list of datasets from a specific environment with mapping progress and stats.
|
||||
# @PRE: env_id must be a valid environment ID
|
||||
# @PRE: page must be >= 1 if provided
|
||||
# @PRE: page_size must be between 1 and 100 if provided
|
||||
# @POST: Returns a list of datasets with enhanced metadata and pagination info
|
||||
# @POST: Response includes pagination metadata (page, page_size, total, total_pages)
|
||||
# @POST: Returns a list of datasets with enhanced metadata, pagination info, and StatsCounts.
|
||||
# @POST: Response includes pagination metadata (page, page_size, total, total_pages) and stats object.
|
||||
# @RATIONALE Stats counts returned in the same response as datasets to avoid an extra API call for the Stats Bar (FR-022).
|
||||
# @RELATION CALLS -> [get_datasets_with_status]
|
||||
@router.get("", response_model=DatasetsResponse)
|
||||
async def get_datasets(
|
||||
env_id: str,
|
||||
search: str | None = None,
|
||||
filter: str | None = None,
|
||||
page: int = 1,
|
||||
page_size: int = 10,
|
||||
config_manager=Depends(get_config_manager),
|
||||
@@ -173,7 +216,7 @@ async def get_datasets(
|
||||
resource_service=Depends(get_resource_service),
|
||||
_ = Depends(has_permission("plugin:migration", "READ"))
|
||||
):
|
||||
with belief_scope("get_datasets", f"env_id={env_id}, search={search}, page={page}, page_size={page_size}"):
|
||||
with belief_scope("get_datasets", f"env_id={env_id}, search={search}, filter={filter}, page={page}, page_size={page_size}"):
|
||||
# Validate pagination parameters
|
||||
if page < 1:
|
||||
logger.error(f"[get_datasets][Coherence:Failed] Invalid page: {page}")
|
||||
@@ -196,6 +239,45 @@ async def get_datasets(
|
||||
# Fetch datasets with status using ResourceService
|
||||
datasets = await resource_service.get_datasets_with_status(env, all_tasks)
|
||||
|
||||
# Compute stats from full list BEFORE filtering/pagination
|
||||
total_datasets = len(datasets)
|
||||
unmapped = sum(
|
||||
1 for d in datasets
|
||||
if d.get("mapped_fields") and d["mapped_fields"].get("mapped", 0) == 0
|
||||
)
|
||||
mapped = sum(
|
||||
1 for d in datasets
|
||||
if d.get("mapped_fields") and d["mapped_fields"].get("mapped", 0) > 0
|
||||
)
|
||||
linked = sum(
|
||||
1 for d in datasets
|
||||
if d.get("linked_dashboard_count", 0) > 0
|
||||
)
|
||||
stats = StatsCounts(
|
||||
total=total_datasets,
|
||||
unmapped_count=unmapped,
|
||||
mapped_count=mapped,
|
||||
linked_count=linked,
|
||||
)
|
||||
|
||||
# Apply server-side filter (before pagination) — FR-026
|
||||
if filter and filter in ("unmapped", "mapped", "linked"):
|
||||
if filter == "unmapped":
|
||||
datasets = [
|
||||
d for d in datasets
|
||||
if d.get("mapped_fields") and d["mapped_fields"].get("mapped", 0) == 0
|
||||
]
|
||||
elif filter == "mapped":
|
||||
datasets = [
|
||||
d for d in datasets
|
||||
if d.get("mapped_fields") and d["mapped_fields"].get("mapped", 0) > 0
|
||||
]
|
||||
elif filter == "linked":
|
||||
datasets = [
|
||||
d for d in datasets
|
||||
if d.get("linked_dashboard_count", 0) > 0
|
||||
]
|
||||
|
||||
# Apply search filter if provided
|
||||
if search:
|
||||
search_lower = search.lower()
|
||||
@@ -213,10 +295,11 @@ async def get_datasets(
|
||||
# Slice datasets for current page
|
||||
paginated_datasets = datasets[start_idx:end_idx]
|
||||
|
||||
logger.info(f"[get_datasets][Coherence:OK] Returning {len(paginated_datasets)} datasets (page {page}/{total_pages}, total: {total})")
|
||||
logger.info(f"[get_datasets][Coherence:OK] Returning {len(paginated_datasets)} datasets (page {page}/{total_pages}, total: {total}, stats: total={stats.total} unmapped={stats.unmapped_count} mapped={stats.mapped_count} linked={stats.linked_count})")
|
||||
|
||||
return DatasetsResponse(
|
||||
datasets=paginated_datasets,
|
||||
stats=stats,
|
||||
total=total,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
@@ -399,4 +482,194 @@ async def get_dataset_detail(
|
||||
raise HTTPException(status_code=503, detail=f"Failed to fetch dataset detail: {e!s}")
|
||||
# #endregion get_dataset_detail
|
||||
|
||||
# #region _strip_html_tags [C:2] [TYPE Function] [SEMANTICS helper,validation,html]
|
||||
# @BRIEF Detect and strip HTML tags from a text string. Uses simple regex <[^>]*>.
|
||||
# If HTML is detected, strips tags and logs a warning. Returns cleaned text.
|
||||
# @REJECTED Reject-with-400 approach was rejected in favour of silent stripping — more user-friendly
|
||||
# and matches the UX expectation that inline-edited descriptions are plain text only.
|
||||
def _strip_html_tags(text: str) -> str:
|
||||
if not text:
|
||||
return text
|
||||
if re.search(r"<[^>]*>", text):
|
||||
cleaned = re.sub(r"<[^>]*>", "", text)
|
||||
logger.warning(f"[_strip_html_tags] HTML tags detected and stripped from description")
|
||||
return cleaned
|
||||
return text
|
||||
# #endregion _strip_html_tags
|
||||
|
||||
# #region update_column_description [C:4] [TYPE Endpoint]
|
||||
# @BRIEF Save description for a single dataset column. Internal: GET full dataset from Superset, modify one column's description, PUT back.
|
||||
# @PRE: dataset_id and column_id must exist in the target environment.
|
||||
# @POST: Column description in Superset is updated. Response confirms success.
|
||||
# @SIDE_EFFECT: Mutates dataset metadata in upstream Superset instance via PUT.
|
||||
# @VALIDATION: description — string, max 2000 chars, plain text only (HTML stripped), may be empty string to clear description, no trimming applied. 404 if dataset or column not found. 502 if Superset upstream fails.
|
||||
# @ERROR: 400 — description exceeds max length or contains non-plaintext content. 404 — dataset_id or column_id not found. 502 — Superset upstream failure (GET or PUT).
|
||||
# @RELATION CALLS -> [SupersetClient.get_dataset]
|
||||
# @RELATION CALLS -> [SupersetClient.update_dataset]
|
||||
# @RATIONALE Must perform GET→modify→PUT because Superset has no PATCH for individual columns — only full object PUT with override_columns=false.
|
||||
# @REJECTED Direct PUT from frontend — rejected because frontend would need to handle full Superset payload structure.
|
||||
@router.put("/{dataset_id}/columns/{column_id}/description")
|
||||
async def update_column_description(
|
||||
dataset_id: int,
|
||||
column_id: int,
|
||||
env_id: str = Query(..., description="Environment ID"),
|
||||
body: ColumnDescriptionUpdate = ...,
|
||||
config_manager=Depends(get_config_manager),
|
||||
_ = Depends(has_permission("plugin:migration", "WRITE"))
|
||||
):
|
||||
with belief_scope("update_column_description", f"dataset_id={dataset_id}, column_id={column_id}, env_id={env_id}"):
|
||||
# Validate and sanitize description
|
||||
description = _strip_html_tags(body.description)
|
||||
if len(description) > 2000:
|
||||
logger.error(f"[update_column_description][Coherence:Failed] Description exceeds 2000 chars: {len(description)}")
|
||||
raise HTTPException(status_code=400, detail="Description must not exceed 2000 characters")
|
||||
|
||||
# Validate environment
|
||||
environments = config_manager.get_environments()
|
||||
env = next((e for e in environments if e.id == env_id), None)
|
||||
if not env:
|
||||
logger.error(f"[update_column_description][Coherence:Failed] Environment not found: {env_id}")
|
||||
raise HTTPException(status_code=404, detail="Environment not found")
|
||||
|
||||
try:
|
||||
client = SupersetClient(env)
|
||||
|
||||
# GET full dataset
|
||||
logger.reason("Fetching full dataset from Superset for column description update",
|
||||
extra={"src": "update_column_description"})
|
||||
try:
|
||||
response = client.get_dataset(dataset_id)
|
||||
except Exception as e:
|
||||
logger.error(f"[update_column_description][Coherence:Failed] Superset GET failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Failed to fetch dataset from Superset: {e!s}")
|
||||
|
||||
if isinstance(response, dict) and "result" in response:
|
||||
dataset = response["result"]
|
||||
else:
|
||||
dataset = response
|
||||
|
||||
if not dataset or not dataset.get("id"):
|
||||
logger.error(f"[update_column_description][Coherence:Failed] Dataset {dataset_id} not found")
|
||||
raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found")
|
||||
|
||||
# Find and modify the target column
|
||||
columns = dataset.get("columns", [])
|
||||
found = False
|
||||
for col in columns:
|
||||
if col.get("id") == column_id:
|
||||
col["description"] = description
|
||||
found = True
|
||||
break
|
||||
|
||||
if not found:
|
||||
logger.error(f"[update_column_description][Coherence:Failed] Column {column_id} not found in dataset {dataset_id}")
|
||||
raise HTTPException(status_code=404, detail=f"Column {column_id} not found in dataset {dataset_id}")
|
||||
|
||||
# PUT back with override_columns=false
|
||||
put_payload = dataset
|
||||
try:
|
||||
client.update_dataset(dataset_id, put_payload, override_columns=False)
|
||||
except Exception as e:
|
||||
logger.error(f"[update_column_description][Coherence:Failed] Superset PUT failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Failed to update dataset in Superset: {e!s}")
|
||||
|
||||
logger.reflect(
|
||||
f"Updated column {column_id} description for dataset {dataset_id}",
|
||||
extra={"src": "update_column_description"},
|
||||
)
|
||||
return {"success": True, "description": description}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"[update_column_description][Coherence:Failed] Unexpected error: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Unexpected error: {e!s}")
|
||||
# #endregion update_column_description
|
||||
|
||||
# #region update_metric_description [C:4] [TYPE Endpoint]
|
||||
# @BRIEF Save description for a single dataset metric. Mirror of update_column_description for metrics.
|
||||
# @PRE: dataset_id and metric_id must exist in the target environment.
|
||||
# @POST: Metric description in Superset is updated.
|
||||
# @SIDE_EFFECT: Mutates dataset metadata in upstream Superset instance via PUT.
|
||||
# @VALIDATION: description — string, max 2000 chars, plain text only (HTML stripped), may be empty string to clear description, no trimming applied. 404 if dataset or metric not found. 502 if Superset upstream fails.
|
||||
# @ERROR: 400 — description exceeds max length or contains non-plaintext content. 404 — dataset_id or metric_id not found. 502 — Superset upstream failure (GET or PUT).
|
||||
# @RELATION CALLS -> [SupersetClient.get_dataset]
|
||||
# @RELATION CALLS -> [SupersetClient.update_dataset]
|
||||
@router.put("/{dataset_id}/metrics/{metric_id}/description")
|
||||
async def update_metric_description(
|
||||
dataset_id: int,
|
||||
metric_id: int,
|
||||
env_id: str = Query(..., description="Environment ID"),
|
||||
body: MetricDescriptionUpdate = ...,
|
||||
config_manager=Depends(get_config_manager),
|
||||
_ = Depends(has_permission("plugin:migration", "WRITE"))
|
||||
):
|
||||
with belief_scope("update_metric_description", f"dataset_id={dataset_id}, metric_id={metric_id}, env_id={env_id}"):
|
||||
# Validate and sanitize description
|
||||
description = _strip_html_tags(body.description)
|
||||
if len(description) > 2000:
|
||||
logger.error(f"[update_metric_description][Coherence:Failed] Description exceeds 2000 chars: {len(description)}")
|
||||
raise HTTPException(status_code=400, detail="Description must not exceed 2000 characters")
|
||||
|
||||
# Validate environment
|
||||
environments = config_manager.get_environments()
|
||||
env = next((e for e in environments if e.id == env_id), None)
|
||||
if not env:
|
||||
logger.error(f"[update_metric_description][Coherence:Failed] Environment not found: {env_id}")
|
||||
raise HTTPException(status_code=404, detail="Environment not found")
|
||||
|
||||
try:
|
||||
client = SupersetClient(env)
|
||||
|
||||
# GET full dataset
|
||||
logger.reason("Fetching full dataset from Superset for metric description update",
|
||||
extra={"src": "update_metric_description"})
|
||||
try:
|
||||
response = client.get_dataset(dataset_id)
|
||||
except Exception as e:
|
||||
logger.error(f"[update_metric_description][Coherence:Failed] Superset GET failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Failed to fetch dataset from Superset: {e!s}")
|
||||
|
||||
if isinstance(response, dict) and "result" in response:
|
||||
dataset = response["result"]
|
||||
else:
|
||||
dataset = response
|
||||
|
||||
if not dataset or not dataset.get("id"):
|
||||
logger.error(f"[update_metric_description][Coherence:Failed] Dataset {dataset_id} not found")
|
||||
raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found")
|
||||
|
||||
# Find and modify the target metric
|
||||
metrics = dataset.get("metrics", [])
|
||||
found = False
|
||||
for metric in metrics:
|
||||
if metric.get("id") == metric_id:
|
||||
metric["description"] = description
|
||||
found = True
|
||||
break
|
||||
|
||||
if not found:
|
||||
logger.error(f"[update_metric_description][Coherence:Failed] Metric {metric_id} not found in dataset {dataset_id}")
|
||||
raise HTTPException(status_code=404, detail=f"Metric {metric_id} not found in dataset {dataset_id}")
|
||||
|
||||
# PUT back with override_columns=false
|
||||
try:
|
||||
client.update_dataset(dataset_id, dataset, override_columns=False)
|
||||
except Exception as e:
|
||||
logger.error(f"[update_metric_description][Coherence:Failed] Superset PUT failed: {e}")
|
||||
raise HTTPException(status_code=502, detail=f"Failed to update dataset in Superset: {e!s}")
|
||||
|
||||
logger.reflect(
|
||||
f"Updated metric {metric_id} description for dataset {dataset_id}",
|
||||
extra={"src": "update_metric_description"},
|
||||
)
|
||||
return {"success": True, "description": description}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"[update_metric_description][Coherence:Failed] Unexpected error: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Unexpected error: {e!s}")
|
||||
# #endregion update_metric_description
|
||||
|
||||
# #endregion DatasetsApi
|
||||
|
||||
@@ -393,6 +393,33 @@ async def websocket_endpoint(
|
||||
extra={"task_id": task_id},
|
||||
)
|
||||
# #endregion websocket_endpoint
|
||||
# #region dataset_websocket_endpoint [C:4] [TYPE Function]
|
||||
# @BRIEF WebSocket endpoint for dataset.updated events — auto-refresh on task completion.
|
||||
# @RELATION CALLS -> [TaskManagerPackage]
|
||||
# @PRE env_id must reference a known environment.
|
||||
# @POST WebSocket streams dataset.updated events until disconnect.
|
||||
# @SIDE_EFFECT Subscribes to dataset event queue in task manager lifecycle.
|
||||
@app.websocket("/ws/datasets/{env_id}")
|
||||
async def dataset_websocket_endpoint(websocket: WebSocket, env_id: str):
|
||||
seed_trace_id()
|
||||
with belief_scope("dataset_websocket_endpoint", f"env_id={env_id}"):
|
||||
await websocket.accept()
|
||||
logger.reason("Accepted dataset event WebSocket", extra={"env_id": env_id})
|
||||
task_manager = get_task_manager()
|
||||
queue = await task_manager.subscribe_dataset_events(env_id)
|
||||
try:
|
||||
while True:
|
||||
event = await queue.get()
|
||||
await websocket.send_json(event)
|
||||
logger.reflect("Forwarded dataset.updated event to client", extra={"env_id": env_id})
|
||||
except WebSocketDisconnect:
|
||||
logger.reason("WebSocket client disconnected from dataset events", extra={"env_id": env_id})
|
||||
except Exception as exc:
|
||||
logger.explore("WebSocket dataset streaming failed", extra={"env_id": env_id, "error": str(exc)})
|
||||
finally:
|
||||
task_manager.unsubscribe_dataset_events(env_id, queue)
|
||||
logger.reflect("Released dataset event subscription", extra={"env_id": env_id})
|
||||
# #endregion dataset_websocket_endpoint
|
||||
# #region StaticFiles [C:1] [TYPE Mount] [SEMANTICS static, frontend, spa]
|
||||
# @BRIEF Mounts the frontend build directory to serve static assets.
|
||||
frontend_path = project_root / "frontend" / "build"
|
||||
|
||||
@@ -86,6 +86,22 @@ class SupersetDatasetsMixin:
|
||||
"description": col.get("description", ""),
|
||||
}
|
||||
)
|
||||
metrics = dataset.get("metrics", [])
|
||||
metric_info = []
|
||||
for metric in metrics:
|
||||
metric_id = metric.get("id")
|
||||
if metric_id is None:
|
||||
continue
|
||||
metric_info.append(
|
||||
{
|
||||
"id": int(metric_id),
|
||||
"metric_name": metric.get("metric_name"),
|
||||
"expression": metric.get("expression"),
|
||||
"verbose_name": metric.get("verbose_name"),
|
||||
"description": metric.get("description", ""),
|
||||
"metric_type": metric.get("metric_type"),
|
||||
}
|
||||
)
|
||||
linked_dashboards = []
|
||||
try:
|
||||
related_objects = self.network.request(
|
||||
@@ -139,6 +155,8 @@ class SupersetDatasetsMixin:
|
||||
"description": dataset.get("description", ""),
|
||||
"columns": column_info,
|
||||
"column_count": len(column_info),
|
||||
"metrics": metric_info,
|
||||
"metric_count": len(metric_info),
|
||||
"sql": sql,
|
||||
"linked_dashboards": linked_dashboards,
|
||||
"linked_dashboard_count": len(linked_dashboards),
|
||||
@@ -147,7 +165,7 @@ class SupersetDatasetsMixin:
|
||||
"changed_on": dataset.get("changed_on"),
|
||||
}
|
||||
app_logger.reflect(
|
||||
f"Got dataset {dataset_id} with {len(column_info)} columns and {len(linked_dashboards)} linked dashboards",
|
||||
f"Got dataset {dataset_id} with {len(column_info)} columns, {len(metric_info)} metrics, and {len(linked_dashboards)} linked dashboards",
|
||||
extra={"src": "get_dataset_detail"},
|
||||
)
|
||||
return result
|
||||
@@ -169,12 +187,13 @@ class SupersetDatasetsMixin:
|
||||
# @PURPOSE: Обновляет данные датасета по его ID.
|
||||
# @SIDE_EFFECT: Modifies resource in upstream Superset environment.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def update_dataset(self, dataset_id: int, data: dict) -> dict:
|
||||
def update_dataset(self, dataset_id: int, data: dict, override_columns: bool = False) -> dict:
|
||||
with belief_scope("SupersetClient.update_dataset", f"id={dataset_id}"):
|
||||
app_logger.info("[update_dataset][Enter] Updating dataset %s.", dataset_id)
|
||||
endpoint = f"/dataset/{dataset_id}?override_columns={'true' if override_columns else 'false'}"
|
||||
response = self.network.request(
|
||||
method="PUT",
|
||||
endpoint=f"/dataset/{dataset_id}",
|
||||
endpoint=endpoint,
|
||||
data=json.dumps(data),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
@@ -71,6 +71,7 @@ class JobLifecycle:
|
||||
self.persistence_service = persistence_service
|
||||
self.executor = executor
|
||||
self.loop = loop
|
||||
self._dataset_subscribers: dict[str, list[asyncio.Queue]] = {}
|
||||
# #endregion __init__
|
||||
|
||||
# #region create_task [C:4] [TYPE Function] [SEMANTICS task,create,persist,schedule]
|
||||
@@ -190,6 +191,12 @@ class JobLifecycle:
|
||||
"Task lifecycle reached persisted terminal state",
|
||||
extra={"task_id": task_id, "status": str(task.status)},
|
||||
)
|
||||
# Broadcast dataset.updated for mapping and documentation tasks (FR-024, R4)
|
||||
if task.plugin_id in ("dataset-mapper", "llm_documentation"):
|
||||
dataset_ids = task.params.get("dataset_ids") or [task.params.get("dataset_id")]
|
||||
env_id = task.params.get("env") or task.params.get("environment_id", "")
|
||||
if dataset_ids and env_id:
|
||||
self._broadcast_dataset_updated(env_id, dataset_ids)
|
||||
# #endregion _run_task
|
||||
|
||||
# #region resolve_task [C:3] [TYPE Function] [SEMANTICS task,resolve,mapping,resume]
|
||||
@@ -309,5 +316,37 @@ class JobLifecycle:
|
||||
)
|
||||
self.graph.resolve_future(task_id, True)
|
||||
# #endregion resume_task_with_password
|
||||
|
||||
# #region subscribe_dataset_events [C:2] [TYPE Function] [SEMANTICS dataset,subscribe,events,websocket]
|
||||
# @BRIEF Subscribe to dataset.updated events for a specific environment.
|
||||
async def subscribe_dataset_events(self, env_id: str) -> asyncio.Queue:
|
||||
queue = asyncio.Queue()
|
||||
if env_id not in self._dataset_subscribers:
|
||||
self._dataset_subscribers[env_id] = []
|
||||
self._dataset_subscribers[env_id].append(queue)
|
||||
return queue
|
||||
# #endregion subscribe_dataset_events
|
||||
|
||||
# #region unsubscribe_dataset_events [C:2] [TYPE Function] [SEMANTICS dataset,unsubscribe,events]
|
||||
# @BRIEF Unsubscribe from dataset.updated events for a specific environment.
|
||||
def unsubscribe_dataset_events(self, env_id: str, queue: asyncio.Queue):
|
||||
if env_id in self._dataset_subscribers:
|
||||
if queue in self._dataset_subscribers[env_id]:
|
||||
self._dataset_subscribers[env_id].remove(queue)
|
||||
if not self._dataset_subscribers[env_id]:
|
||||
del self._dataset_subscribers[env_id]
|
||||
# #endregion unsubscribe_dataset_events
|
||||
|
||||
# #region _broadcast_dataset_updated [C:3] [TYPE Function] [SEMANTICS dataset,broadcast,event,websocket]
|
||||
# @BRIEF Broadcast dataset.updated event to all subscribers for a given environment.
|
||||
def _broadcast_dataset_updated(self, env_id: str, dataset_ids: list[int]):
|
||||
event = {"type": "dataset.updated", "payload": {"env_id": env_id, "dataset_ids": dataset_ids}}
|
||||
if env_id in self._dataset_subscribers:
|
||||
for queue in self._dataset_subscribers[env_id]:
|
||||
try:
|
||||
self.loop.call_soon_threadsafe(queue.put_nowait, event)
|
||||
except Exception:
|
||||
pass
|
||||
# #endregion _broadcast_dataset_updated
|
||||
# #endregion JobLifecycle
|
||||
# #endregion JobLifecycleModule
|
||||
|
||||
@@ -318,5 +318,19 @@ class TaskManager:
|
||||
add_log_callback=self._add_log,
|
||||
)
|
||||
# #endregion resume_task_with_password
|
||||
|
||||
# ── Dataset event delegates to JobLifecycle ──
|
||||
|
||||
# #region subscribe_dataset_events [TYPE Function] [C:2]
|
||||
# @BRIEF Subscribe to dataset.updated events for an environment.
|
||||
async def subscribe_dataset_events(self, env_id: str) -> asyncio.Queue:
|
||||
return await self.lifecycle.subscribe_dataset_events(env_id)
|
||||
# #endregion subscribe_dataset_events
|
||||
|
||||
# #region unsubscribe_dataset_events [TYPE Function] [C:2]
|
||||
# @BRIEF Unsubscribe from dataset.updated events.
|
||||
def unsubscribe_dataset_events(self, env_id: str, queue: asyncio.Queue):
|
||||
self.lifecycle.unsubscribe_dataset_events(env_id, queue)
|
||||
# #endregion unsubscribe_dataset_events
|
||||
# #endregion TaskManager
|
||||
# #endregion TaskManagerModule
|
||||
|
||||
@@ -11,7 +11,7 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Set env defaults for module-level AuthConfig() singleton — crash-early fix
|
||||
os.environ.setdefault("AUTH_SECRET_KEY", "test-secret-key-for-testing")
|
||||
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing")
|
||||
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:test_auth")
|
||||
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:test_main")
|
||||
os.environ.setdefault("DEV_MODE", "true")
|
||||
|
||||
419
backend/tests/test_datasets.py
Normal file
419
backend/tests/test_datasets.py
Normal file
@@ -0,0 +1,419 @@
|
||||
# [DEF:TestDatasetsApi:Module]
|
||||
# @RELATION: DEPENDS_ON -> [DatasetsApi]
|
||||
# @SEMANTICS: tests, datasets, api, metrics, inline-edit
|
||||
# @PURPOSE: Contract tests for dataset endpoints (list+stats, detail+metrics, inline-edit descriptions).
|
||||
# @LAYER: Domain (Tests)
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from src.app import app
|
||||
from src.dependencies import (
|
||||
get_config_manager,
|
||||
get_current_user,
|
||||
get_resource_service,
|
||||
get_task_manager,
|
||||
has_permission,
|
||||
)
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
# [DEF:mock_user:Function]
|
||||
# @PURPOSE: Create a mock user with Admin role and all permissions for testing.
|
||||
def _make_mock_user():
|
||||
admin_role = MagicMock()
|
||||
admin_role.name = "Admin"
|
||||
admin_role.permissions = []
|
||||
|
||||
mock_user = MagicMock()
|
||||
mock_user.username = "test-user"
|
||||
mock_user.roles = [admin_role]
|
||||
return mock_user
|
||||
|
||||
|
||||
# [DEF:mock_deps:Function]
|
||||
# @PURPOSE: Provide dependency override fixture for dataset route tests.
|
||||
# @TEST_FIXTURE: dataset_route_overrides -> INLINE_JSON
|
||||
@pytest.fixture
|
||||
def mock_deps():
|
||||
config_manager = MagicMock()
|
||||
task_manager = MagicMock()
|
||||
resource_service = MagicMock()
|
||||
|
||||
env = MagicMock()
|
||||
env.id = "ss-dev"
|
||||
env.name = "SS Dev"
|
||||
env.superset_url = "http://superset:8088"
|
||||
env.superset_token = "mock-token"
|
||||
config_manager.get_environments.return_value = [env]
|
||||
|
||||
task_manager.get_all_tasks.return_value = []
|
||||
|
||||
resource_service.get_datasets_with_status = AsyncMock(
|
||||
return_value=[
|
||||
{
|
||||
"id": 1,
|
||||
"table_name": "users",
|
||||
"schema": "public",
|
||||
"database": "postgres",
|
||||
"mapped_fields": {"total": 10, "mapped": 7},
|
||||
"last_task": {"task_id": "t1", "status": "SUCCESS"},
|
||||
"linked_dashboard_count": 2,
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"table_name": "orders",
|
||||
"schema": "public",
|
||||
"database": "postgres",
|
||||
"mapped_fields": {"total": 5, "mapped": 0},
|
||||
"last_task": None,
|
||||
"linked_dashboard_count": 0,
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"table_name": "products",
|
||||
"schema": "public",
|
||||
"database": "postgres",
|
||||
"mapped_fields": None,
|
||||
"last_task": None,
|
||||
"linked_dashboard_count": 0,
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
mock_user = _make_mock_user()
|
||||
|
||||
app.dependency_overrides[get_config_manager] = lambda: config_manager
|
||||
app.dependency_overrides[get_task_manager] = lambda: task_manager
|
||||
app.dependency_overrides[get_resource_service] = lambda: resource_service
|
||||
app.dependency_overrides[get_current_user] = lambda: mock_user
|
||||
# Override specific permission dependencies used by datasets routes
|
||||
app.dependency_overrides[has_permission("plugin:migration", "READ")] = lambda: mock_user
|
||||
app.dependency_overrides[has_permission("plugin:migration", "WRITE")] = lambda: mock_user
|
||||
|
||||
yield
|
||||
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
# [DEF:test_get_datasets_returns_stats:Function]
|
||||
# @PURPOSE: Verify GET /api/datasets returns stats object with correct counts.
|
||||
# @TEST_CONTRACT: datasets_list -> datasets with stats payload
|
||||
# @TEST_SCENARIO: datasets_with_stats -> HTTP 200 returns datasets, stats, and pagination.
|
||||
# @TEST_INVARIANT: stats_counts_correct -> VERIFIED_BY: [datasets_with_stats]
|
||||
def test_get_datasets_returns_stats(mock_deps):
|
||||
response = client.get(
|
||||
"/api/datasets?env_id=ss-dev&page=1&page_size=10"
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "datasets" in data
|
||||
assert "stats" in data
|
||||
assert data["stats"]["total"] == 3
|
||||
assert data["stats"]["mapped_count"] == 1 # users: 7/10
|
||||
assert data["stats"]["unmapped_count"] == 1 # orders: 0/5
|
||||
assert data["stats"]["linked_count"] == 1 # users has 2 linked dashboards
|
||||
assert data["total"] == 3
|
||||
assert data["page"] == 1
|
||||
assert len(data["datasets"]) == 3
|
||||
|
||||
|
||||
# [DEF:test_get_datasets_filter_unmapped:Function]
|
||||
# @PURPOSE: Verify server-side filtering by unmapped datasets.
|
||||
# @TEST_SCENARIO: filter_unmapped -> Only returns datasets with mapped=0.
|
||||
def test_get_datasets_filter_unmapped(mock_deps):
|
||||
response = client.get(
|
||||
"/api/datasets?env_id=ss-dev&filter=unmapped&page=1&page_size=10"
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
# stats are global (unfiltered)
|
||||
assert data["stats"]["total"] == 3
|
||||
# filtered list: only orders (mapped=0)
|
||||
assert data["total"] == 1
|
||||
assert len(data["datasets"]) == 1
|
||||
assert data["datasets"][0]["table_name"] == "orders"
|
||||
|
||||
|
||||
# [DEF:test_get_datasets_filter_mapped:Function]
|
||||
# @PURPOSE: Verify server-side filtering by mapped datasets.
|
||||
def test_get_datasets_filter_mapped(mock_deps):
|
||||
response = client.get(
|
||||
"/api/datasets?env_id=ss-dev&filter=mapped&page=1&page_size=10"
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["stats"]["total"] == 3
|
||||
assert data["total"] == 1
|
||||
assert data["datasets"][0]["table_name"] == "users"
|
||||
|
||||
|
||||
# [DEF:test_get_datasets_filter_linked:Function]
|
||||
# @PURPOSE: Verify server-side filtering by datasets linked to dashboards.
|
||||
def test_get_datasets_filter_linked(mock_deps):
|
||||
response = client.get(
|
||||
"/api/datasets?env_id=ss-dev&filter=linked&page=1&page_size=10"
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["stats"]["total"] == 3
|
||||
assert data["total"] == 1
|
||||
assert data["datasets"][0]["table_name"] == "users"
|
||||
|
||||
|
||||
# [DEF:test_get_datasets_filter_all:Function]
|
||||
# @PURPOSE: Verify filter=all returns all datasets (same as no filter).
|
||||
def test_get_datasets_filter_all(mock_deps):
|
||||
response = client.get(
|
||||
"/api/datasets?env_id=ss-dev&filter=all&page=1&page_size=10"
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["total"] == 3
|
||||
|
||||
|
||||
# [DEF:test_get_dataset_detail_returns_metrics:Function]
|
||||
# @PURPOSE: Verify GET /api/datasets/{id} returns metrics and metric_count.
|
||||
# @TEST_SCENARIO: detail_with_metrics -> Response includes metrics list and metric_count.
|
||||
def test_get_dataset_detail_returns_metrics(mock_deps):
|
||||
with patch("src.api.routes.datasets.SupersetClient") as MockClient:
|
||||
mock_client = MagicMock()
|
||||
MockClient.return_value = mock_client
|
||||
|
||||
mock_client.get_dataset_detail.return_value = {
|
||||
"id": 1,
|
||||
"table_name": "users",
|
||||
"schema": "public",
|
||||
"database": "postgres",
|
||||
"description": "User table",
|
||||
"columns": [
|
||||
{"id": 10, "name": "id", "type": "INT", "is_dttm": False, "is_active": True, "description": "PK"},
|
||||
{"id": 11, "name": "name", "type": "STRING", "is_dttm": False, "is_active": True, "description": ""},
|
||||
],
|
||||
"column_count": 2,
|
||||
"metrics": [
|
||||
{"id": 100, "metric_name": "count", "expression": "COUNT(*)", "verbose_name": "Row Count", "description": "Total rows", "metric_type": "count"},
|
||||
],
|
||||
"metric_count": 1,
|
||||
"sql": "SELECT * FROM users",
|
||||
"linked_dashboards": [{"id": 5, "title": "Users Dashboard", "slug": "users-dash"}],
|
||||
"linked_dashboard_count": 1,
|
||||
"is_sqllab_view": False,
|
||||
"created_on": "2024-01-01T00:00:00",
|
||||
"changed_on": "2024-06-01T00:00:00",
|
||||
}
|
||||
|
||||
response = client.get(
|
||||
"/api/datasets/1?env_id=ss-dev"
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "metrics" in data
|
||||
assert "metric_count" in data
|
||||
assert data["metric_count"] == 1
|
||||
assert len(data["metrics"]) == 1
|
||||
assert data["metrics"][0]["metric_name"] == "count"
|
||||
assert data["metrics"][0]["expression"] == "COUNT(*)"
|
||||
|
||||
|
||||
# [DEF:test_update_column_description:Function]
|
||||
# @PURPOSE: Verify PUT /api/datasets/{id}/columns/{col_id}/description saves description.
|
||||
# @TEST_SCENARIO: save_column_desc -> Saves and returns updated description.
|
||||
def test_update_column_description(mock_deps):
|
||||
with patch("src.api.routes.datasets.SupersetClient") as MockClient:
|
||||
mock_client = MagicMock()
|
||||
MockClient.return_value = mock_client
|
||||
|
||||
mock_client.get_dataset.return_value = {
|
||||
"id": 1,
|
||||
"result": {
|
||||
"id": 1,
|
||||
"columns": [
|
||||
{"id": 10, "column_name": "id", "type": "INT", "description": "old desc"},
|
||||
{"id": 11, "column_name": "name", "type": "STRING", "description": ""},
|
||||
],
|
||||
"metrics": [],
|
||||
},
|
||||
}
|
||||
mock_client.update_dataset.return_value = {"id": 1}
|
||||
|
||||
response = client.put(
|
||||
"/api/datasets/1/columns/10/description?env_id=ss-dev",
|
||||
json={"description": "new description"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["success"] is True
|
||||
assert data["description"] == "new description"
|
||||
|
||||
# Verify update_dataset was called with correct column modified
|
||||
update_call = mock_client.update_dataset.call_args
|
||||
assert update_call[0][0] == 1 # dataset_id
|
||||
# Check override_columns=false was passed
|
||||
assert update_call[1]["override_columns"] is False
|
||||
|
||||
|
||||
# [DEF:test_update_metric_description:Function]
|
||||
# @PURPOSE: Verify PUT /api/datasets/{id}/metrics/{metric_id}/description saves correctly.
|
||||
# @TEST_SCENARIO: save_metric_desc -> Saves and returns updated description.
|
||||
def test_update_metric_description(mock_deps):
|
||||
with patch("src.api.routes.datasets.SupersetClient") as MockClient:
|
||||
mock_client = MagicMock()
|
||||
MockClient.return_value = mock_client
|
||||
|
||||
mock_client.get_dataset.return_value = {
|
||||
"id": 1,
|
||||
"result": {
|
||||
"id": 1,
|
||||
"columns": [],
|
||||
"metrics": [
|
||||
{"id": 100, "metric_name": "count", "expression": "COUNT(*)", "description": ""},
|
||||
],
|
||||
},
|
||||
}
|
||||
mock_client.update_dataset.return_value = {"id": 1}
|
||||
|
||||
response = client.put(
|
||||
"/api/datasets/1/metrics/100/description?env_id=ss-dev",
|
||||
json={"description": "Total row count"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["success"] is True
|
||||
assert data["description"] == "Total row count"
|
||||
|
||||
|
||||
# [DEF:test_update_column_description_not_found:Function]
|
||||
# @PURPOSE: Verify 404 when column not found.
|
||||
def test_update_column_description_not_found(mock_deps):
|
||||
with patch("src.api.routes.datasets.SupersetClient") as MockClient:
|
||||
mock_client = MagicMock()
|
||||
MockClient.return_value = mock_client
|
||||
|
||||
mock_client.get_dataset.return_value = {
|
||||
"id": 1,
|
||||
"result": {
|
||||
"id": 1,
|
||||
"columns": [{"id": 10, "column_name": "id"}],
|
||||
"metrics": [],
|
||||
},
|
||||
}
|
||||
|
||||
response = client.put(
|
||||
"/api/datasets/1/columns/999/description?env_id=ss-dev",
|
||||
json={"description": "test"},
|
||||
)
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
# [DEF:test_update_column_description_validation:Function]
|
||||
# @PURPOSE: Verify 400 when description exceeds max length.
|
||||
def test_update_column_description_validation(mock_deps):
|
||||
response = client.put(
|
||||
"/api/datasets/1/columns/10/description?env_id=ss-dev",
|
||||
json={"description": "x" * 2001},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
# [DEF:test_get_datasets_superset_503:Function]
|
||||
# @PURPOSE: Verify GET /api/datasets returns 503 when Superset/ResourceService throws.
|
||||
# @TEST_SCENARIO: upstream_superset_failure -> 503 with error detail.
|
||||
# @TEST_INVARIANT: upstream_failure_503 -> VERIFIED_BY: [superset_failure]
|
||||
def test_get_datasets_superset_503(mock_deps):
|
||||
# Override resource_service to throw during get_datasets_with_status
|
||||
from src.dependencies import get_resource_service
|
||||
|
||||
failing_resource_service = MagicMock()
|
||||
failing_resource_service.get_datasets_with_status = AsyncMock(
|
||||
side_effect=Exception("Superset connection refused")
|
||||
)
|
||||
|
||||
app.dependency_overrides[get_resource_service] = lambda: failing_resource_service
|
||||
|
||||
try:
|
||||
response = client.get(
|
||||
"/api/datasets?env_id=ss-dev&page=1&page_size=10"
|
||||
)
|
||||
assert response.status_code == 503
|
||||
assert "Superset" in response.text or "Failed" in response.text
|
||||
finally:
|
||||
app.dependency_overrides[get_resource_service] = None
|
||||
|
||||
|
||||
# [DEF:test_update_column_description_superset_502:Function]
|
||||
# @PURPOSE: Verify PUT column description returns 502 when SupersetClient.update_dataset throws.
|
||||
# @TEST_SCENARIO: superset_put_failure -> 502 with error detail.
|
||||
def test_update_column_description_superset_502(mock_deps):
|
||||
with patch("src.api.routes.datasets.SupersetClient") as MockClient:
|
||||
mock_client = MagicMock()
|
||||
MockClient.return_value = mock_client
|
||||
|
||||
# GET succeeds
|
||||
mock_client.get_dataset.return_value = {
|
||||
"id": 1,
|
||||
"result": {
|
||||
"id": 1,
|
||||
"columns": [{"id": 10, "column_name": "id", "type": "INT", "description": "old desc"}],
|
||||
"metrics": [],
|
||||
},
|
||||
}
|
||||
# PUT fails
|
||||
mock_client.update_dataset.side_effect = Exception("Superset write failure")
|
||||
|
||||
response = client.put(
|
||||
"/api/datasets/1/columns/10/description?env_id=ss-dev",
|
||||
json={"description": "new description"},
|
||||
)
|
||||
assert response.status_code == 502
|
||||
data = response.json()
|
||||
assert "Superset" in data.get("detail", "")
|
||||
|
||||
|
||||
# [DEF:test_update_column_description_strips_html:Function]
|
||||
# @PURPOSE: Verify HTML tags are stripped from description before saving.
|
||||
# @TEST_SCENARIO: html_in_description -> HTML tags stripped, clean text saved.
|
||||
def test_update_column_description_strips_html(mock_deps):
|
||||
with patch("src.api.routes.datasets.SupersetClient") as MockClient:
|
||||
mock_client = MagicMock()
|
||||
MockClient.return_value = mock_client
|
||||
|
||||
mock_client.get_dataset.return_value = {
|
||||
"id": 1,
|
||||
"result": {
|
||||
"id": 1,
|
||||
"columns": [{"id": 10, "column_name": "id", "type": "INT", "description": ""}],
|
||||
"metrics": [],
|
||||
},
|
||||
}
|
||||
mock_client.update_dataset.return_value = {"id": 1}
|
||||
|
||||
response = client.put(
|
||||
"/api/datasets/1/columns/10/description?env_id=ss-dev",
|
||||
json={"description": "Hello <b>World</b> with <script>alert('xss')</script>"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
# HTML stripped
|
||||
assert "Hello World with alert('xss')" == data["description"]
|
||||
assert "<b>" not in data["description"]
|
||||
assert "<script>" not in data["description"]
|
||||
|
||||
# Verify the stripped description was sent to Superset
|
||||
update_call = mock_client.update_dataset.call_args
|
||||
updated_columns = update_call[0][1]["columns"]
|
||||
assert updated_columns[0]["description"] == "Hello World with alert('xss')"
|
||||
|
||||
|
||||
# [DEF:test_dataset_item_has_metric_count:Function]
|
||||
# @PURPOSE: Verify DatasetItem in list response includes metric_count field.
|
||||
def test_dataset_item_has_metric_count(mock_deps):
|
||||
response = client.get(
|
||||
"/api/datasets?env_id=ss-dev&page=1&page_size=10"
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
for ds in data["datasets"]:
|
||||
assert "metric_count" in ds
|
||||
assert isinstance(ds["metric_count"], int)
|
||||
Reference in New Issue
Block a user