diff --git a/backend/src/api/routes/datasets.py b/backend/src/api/routes/datasets.py index 5d046218..828a52d2 100644 --- a/backend/src/api/routes/datasets.py +++ b/backend/src/api/routes/datasets.py @@ -13,7 +13,9 @@ # @INVARIANT: All dataset responses include last_task metadata -from fastapi import APIRouter, Depends, HTTPException +import re + +from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel, Field from ...core.logger import belief_scope, logger @@ -45,6 +47,7 @@ class DatasetItem(BaseModel): database: str mapped_fields: MappedFields | None = None last_task: LastTask | None = None + metric_count: int = 0 class Config: allow_population_by_field_name = True @@ -69,8 +72,21 @@ class DatasetColumn(BaseModel): description: str | None = None # #endregion DatasetColumn -# #region DatasetDetailResponse [C:1] [TYPE DataClass] -# @BRIEF Detailed DTO for a dataset including columns and links +# #region MetricItem [C:1] [TYPE DataClass] +# @BRIEF Pydantic DTO for a dataset metric — carries Superset metric metadata. +# @RELATION DEPENDS_ON -> [DatasetDetailResponse] +class MetricItem(BaseModel): + id: int + metric_name: str + expression: str | None = None + verbose_name: str | None = None + description: str | None = None + metric_type: str | None = None +# #endregion MetricItem + +# #region DatasetDetailResponse [C:2] [TYPE DataClass] +# @BRIEF Detailed DTO for a dataset including columns, linked dashboards, and metrics. +# @RELATION DEPENDS_ON -> [MetricItem] class DatasetDetailResponse(BaseModel): id: int table_name: str | None = None @@ -79,6 +95,8 @@ class DatasetDetailResponse(BaseModel): description: str | None = None columns: list[DatasetColumn] column_count: int + metrics: list[MetricItem] = [] + metric_count: int = 0 sql: str | None = None linked_dashboards: list[LinkedDashboard] linked_dashboard_count: int @@ -90,10 +108,21 @@ class DatasetDetailResponse(BaseModel): allow_population_by_field_name = True # #endregion DatasetDetailResponse -# #region DatasetsResponse [C:1] [TYPE DataClass] -# @BRIEF Paginated response DTO for dataset listings +# #region StatsCounts [C:1] [TYPE DataClass] +# @BRIEF Aggregate statistics for the Stats Bar — computed from full dataset list. +class StatsCounts(BaseModel): + total: int + unmapped_count: int + mapped_count: int + linked_count: int +# #endregion StatsCounts + +# #region DatasetsResponse [C:2] [TYPE DataClass] +# @BRIEF Paginated response DTO for dataset listings with stats. +# @RELATION DEPENDS_ON -> [StatsCounts] class DatasetsResponse(BaseModel): datasets: list[DatasetItem] + stats: StatsCounts total: int page: int page_size: int @@ -106,6 +135,18 @@ class TaskResponse(BaseModel): task_id: str # #endregion TaskResponse +# #region ColumnDescriptionUpdate [C:1] [TYPE DataClass] +# @BRIEF Request DTO for inline-edit of a column description. +class ColumnDescriptionUpdate(BaseModel): + description: str +# #endregion ColumnDescriptionUpdate + +# #region MetricDescriptionUpdate [C:1] [TYPE DataClass] +# @BRIEF Request DTO for inline-edit of a metric description. +class MetricDescriptionUpdate(BaseModel): + description: str +# #endregion MetricDescriptionUpdate + # #region get_dataset_ids [C:4] [TYPE Function] # @BRIEF Fetch list of all dataset IDs from a specific environment (without pagination) # @PRE: env_id must be a valid environment ID @@ -155,17 +196,19 @@ async def get_dataset_ids( # #endregion get_dataset_ids # #region get_datasets [C:4] [TYPE Function] -# @BRIEF Fetch list of datasets from a specific environment with mapping progress +# @BRIEF Fetch list of datasets from a specific environment with mapping progress and stats. # @PRE: env_id must be a valid environment ID # @PRE: page must be >= 1 if provided # @PRE: page_size must be between 1 and 100 if provided -# @POST: Returns a list of datasets with enhanced metadata and pagination info -# @POST: Response includes pagination metadata (page, page_size, total, total_pages) +# @POST: Returns a list of datasets with enhanced metadata, pagination info, and StatsCounts. +# @POST: Response includes pagination metadata (page, page_size, total, total_pages) and stats object. +# @RATIONALE Stats counts returned in the same response as datasets to avoid an extra API call for the Stats Bar (FR-022). # @RELATION CALLS -> [get_datasets_with_status] @router.get("", response_model=DatasetsResponse) async def get_datasets( env_id: str, search: str | None = None, + filter: str | None = None, page: int = 1, page_size: int = 10, config_manager=Depends(get_config_manager), @@ -173,7 +216,7 @@ async def get_datasets( resource_service=Depends(get_resource_service), _ = Depends(has_permission("plugin:migration", "READ")) ): - with belief_scope("get_datasets", f"env_id={env_id}, search={search}, page={page}, page_size={page_size}"): + with belief_scope("get_datasets", f"env_id={env_id}, search={search}, filter={filter}, page={page}, page_size={page_size}"): # Validate pagination parameters if page < 1: logger.error(f"[get_datasets][Coherence:Failed] Invalid page: {page}") @@ -196,6 +239,45 @@ async def get_datasets( # Fetch datasets with status using ResourceService datasets = await resource_service.get_datasets_with_status(env, all_tasks) + # Compute stats from full list BEFORE filtering/pagination + total_datasets = len(datasets) + unmapped = sum( + 1 for d in datasets + if d.get("mapped_fields") and d["mapped_fields"].get("mapped", 0) == 0 + ) + mapped = sum( + 1 for d in datasets + if d.get("mapped_fields") and d["mapped_fields"].get("mapped", 0) > 0 + ) + linked = sum( + 1 for d in datasets + if d.get("linked_dashboard_count", 0) > 0 + ) + stats = StatsCounts( + total=total_datasets, + unmapped_count=unmapped, + mapped_count=mapped, + linked_count=linked, + ) + + # Apply server-side filter (before pagination) — FR-026 + if filter and filter in ("unmapped", "mapped", "linked"): + if filter == "unmapped": + datasets = [ + d for d in datasets + if d.get("mapped_fields") and d["mapped_fields"].get("mapped", 0) == 0 + ] + elif filter == "mapped": + datasets = [ + d for d in datasets + if d.get("mapped_fields") and d["mapped_fields"].get("mapped", 0) > 0 + ] + elif filter == "linked": + datasets = [ + d for d in datasets + if d.get("linked_dashboard_count", 0) > 0 + ] + # Apply search filter if provided if search: search_lower = search.lower() @@ -213,10 +295,11 @@ async def get_datasets( # Slice datasets for current page paginated_datasets = datasets[start_idx:end_idx] - logger.info(f"[get_datasets][Coherence:OK] Returning {len(paginated_datasets)} datasets (page {page}/{total_pages}, total: {total})") + logger.info(f"[get_datasets][Coherence:OK] Returning {len(paginated_datasets)} datasets (page {page}/{total_pages}, total: {total}, stats: total={stats.total} unmapped={stats.unmapped_count} mapped={stats.mapped_count} linked={stats.linked_count})") return DatasetsResponse( datasets=paginated_datasets, + stats=stats, total=total, page=page, page_size=page_size, @@ -399,4 +482,194 @@ async def get_dataset_detail( raise HTTPException(status_code=503, detail=f"Failed to fetch dataset detail: {e!s}") # #endregion get_dataset_detail +# #region _strip_html_tags [C:2] [TYPE Function] [SEMANTICS helper,validation,html] +# @BRIEF Detect and strip HTML tags from a text string. Uses simple regex <[^>]*>. +# If HTML is detected, strips tags and logs a warning. Returns cleaned text. +# @REJECTED Reject-with-400 approach was rejected in favour of silent stripping — more user-friendly +# and matches the UX expectation that inline-edited descriptions are plain text only. +def _strip_html_tags(text: str) -> str: + if not text: + return text + if re.search(r"<[^>]*>", text): + cleaned = re.sub(r"<[^>]*>", "", text) + logger.warning(f"[_strip_html_tags] HTML tags detected and stripped from description") + return cleaned + return text +# #endregion _strip_html_tags + +# #region update_column_description [C:4] [TYPE Endpoint] +# @BRIEF Save description for a single dataset column. Internal: GET full dataset from Superset, modify one column's description, PUT back. +# @PRE: dataset_id and column_id must exist in the target environment. +# @POST: Column description in Superset is updated. Response confirms success. +# @SIDE_EFFECT: Mutates dataset metadata in upstream Superset instance via PUT. +# @VALIDATION: description — string, max 2000 chars, plain text only (HTML stripped), may be empty string to clear description, no trimming applied. 404 if dataset or column not found. 502 if Superset upstream fails. +# @ERROR: 400 — description exceeds max length or contains non-plaintext content. 404 — dataset_id or column_id not found. 502 — Superset upstream failure (GET or PUT). +# @RELATION CALLS -> [SupersetClient.get_dataset] +# @RELATION CALLS -> [SupersetClient.update_dataset] +# @RATIONALE Must perform GET→modify→PUT because Superset has no PATCH for individual columns — only full object PUT with override_columns=false. +# @REJECTED Direct PUT from frontend — rejected because frontend would need to handle full Superset payload structure. +@router.put("/{dataset_id}/columns/{column_id}/description") +async def update_column_description( + dataset_id: int, + column_id: int, + env_id: str = Query(..., description="Environment ID"), + body: ColumnDescriptionUpdate = ..., + config_manager=Depends(get_config_manager), + _ = Depends(has_permission("plugin:migration", "WRITE")) +): + with belief_scope("update_column_description", f"dataset_id={dataset_id}, column_id={column_id}, env_id={env_id}"): + # Validate and sanitize description + description = _strip_html_tags(body.description) + if len(description) > 2000: + logger.error(f"[update_column_description][Coherence:Failed] Description exceeds 2000 chars: {len(description)}") + raise HTTPException(status_code=400, detail="Description must not exceed 2000 characters") + + # Validate environment + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == env_id), None) + if not env: + logger.error(f"[update_column_description][Coherence:Failed] Environment not found: {env_id}") + raise HTTPException(status_code=404, detail="Environment not found") + + try: + client = SupersetClient(env) + + # GET full dataset + logger.reason("Fetching full dataset from Superset for column description update", + extra={"src": "update_column_description"}) + try: + response = client.get_dataset(dataset_id) + except Exception as e: + logger.error(f"[update_column_description][Coherence:Failed] Superset GET failed: {e}") + raise HTTPException(status_code=502, detail=f"Failed to fetch dataset from Superset: {e!s}") + + if isinstance(response, dict) and "result" in response: + dataset = response["result"] + else: + dataset = response + + if not dataset or not dataset.get("id"): + logger.error(f"[update_column_description][Coherence:Failed] Dataset {dataset_id} not found") + raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") + + # Find and modify the target column + columns = dataset.get("columns", []) + found = False + for col in columns: + if col.get("id") == column_id: + col["description"] = description + found = True + break + + if not found: + logger.error(f"[update_column_description][Coherence:Failed] Column {column_id} not found in dataset {dataset_id}") + raise HTTPException(status_code=404, detail=f"Column {column_id} not found in dataset {dataset_id}") + + # PUT back with override_columns=false + put_payload = dataset + try: + client.update_dataset(dataset_id, put_payload, override_columns=False) + except Exception as e: + logger.error(f"[update_column_description][Coherence:Failed] Superset PUT failed: {e}") + raise HTTPException(status_code=502, detail=f"Failed to update dataset in Superset: {e!s}") + + logger.reflect( + f"Updated column {column_id} description for dataset {dataset_id}", + extra={"src": "update_column_description"}, + ) + return {"success": True, "description": description} + + except HTTPException: + raise + except Exception as e: + logger.error(f"[update_column_description][Coherence:Failed] Unexpected error: {e}") + raise HTTPException(status_code=500, detail=f"Unexpected error: {e!s}") +# #endregion update_column_description + +# #region update_metric_description [C:4] [TYPE Endpoint] +# @BRIEF Save description for a single dataset metric. Mirror of update_column_description for metrics. +# @PRE: dataset_id and metric_id must exist in the target environment. +# @POST: Metric description in Superset is updated. +# @SIDE_EFFECT: Mutates dataset metadata in upstream Superset instance via PUT. +# @VALIDATION: description — string, max 2000 chars, plain text only (HTML stripped), may be empty string to clear description, no trimming applied. 404 if dataset or metric not found. 502 if Superset upstream fails. +# @ERROR: 400 — description exceeds max length or contains non-plaintext content. 404 — dataset_id or metric_id not found. 502 — Superset upstream failure (GET or PUT). +# @RELATION CALLS -> [SupersetClient.get_dataset] +# @RELATION CALLS -> [SupersetClient.update_dataset] +@router.put("/{dataset_id}/metrics/{metric_id}/description") +async def update_metric_description( + dataset_id: int, + metric_id: int, + env_id: str = Query(..., description="Environment ID"), + body: MetricDescriptionUpdate = ..., + config_manager=Depends(get_config_manager), + _ = Depends(has_permission("plugin:migration", "WRITE")) +): + with belief_scope("update_metric_description", f"dataset_id={dataset_id}, metric_id={metric_id}, env_id={env_id}"): + # Validate and sanitize description + description = _strip_html_tags(body.description) + if len(description) > 2000: + logger.error(f"[update_metric_description][Coherence:Failed] Description exceeds 2000 chars: {len(description)}") + raise HTTPException(status_code=400, detail="Description must not exceed 2000 characters") + + # Validate environment + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == env_id), None) + if not env: + logger.error(f"[update_metric_description][Coherence:Failed] Environment not found: {env_id}") + raise HTTPException(status_code=404, detail="Environment not found") + + try: + client = SupersetClient(env) + + # GET full dataset + logger.reason("Fetching full dataset from Superset for metric description update", + extra={"src": "update_metric_description"}) + try: + response = client.get_dataset(dataset_id) + except Exception as e: + logger.error(f"[update_metric_description][Coherence:Failed] Superset GET failed: {e}") + raise HTTPException(status_code=502, detail=f"Failed to fetch dataset from Superset: {e!s}") + + if isinstance(response, dict) and "result" in response: + dataset = response["result"] + else: + dataset = response + + if not dataset or not dataset.get("id"): + logger.error(f"[update_metric_description][Coherence:Failed] Dataset {dataset_id} not found") + raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found") + + # Find and modify the target metric + metrics = dataset.get("metrics", []) + found = False + for metric in metrics: + if metric.get("id") == metric_id: + metric["description"] = description + found = True + break + + if not found: + logger.error(f"[update_metric_description][Coherence:Failed] Metric {metric_id} not found in dataset {dataset_id}") + raise HTTPException(status_code=404, detail=f"Metric {metric_id} not found in dataset {dataset_id}") + + # PUT back with override_columns=false + try: + client.update_dataset(dataset_id, dataset, override_columns=False) + except Exception as e: + logger.error(f"[update_metric_description][Coherence:Failed] Superset PUT failed: {e}") + raise HTTPException(status_code=502, detail=f"Failed to update dataset in Superset: {e!s}") + + logger.reflect( + f"Updated metric {metric_id} description for dataset {dataset_id}", + extra={"src": "update_metric_description"}, + ) + return {"success": True, "description": description} + + except HTTPException: + raise + except Exception as e: + logger.error(f"[update_metric_description][Coherence:Failed] Unexpected error: {e}") + raise HTTPException(status_code=500, detail=f"Unexpected error: {e!s}") +# #endregion update_metric_description + # #endregion DatasetsApi diff --git a/backend/src/app.py b/backend/src/app.py index 93a127c9..34a5eeab 100755 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -393,6 +393,33 @@ async def websocket_endpoint( extra={"task_id": task_id}, ) # #endregion websocket_endpoint +# #region dataset_websocket_endpoint [C:4] [TYPE Function] +# @BRIEF WebSocket endpoint for dataset.updated events — auto-refresh on task completion. +# @RELATION CALLS -> [TaskManagerPackage] +# @PRE env_id must reference a known environment. +# @POST WebSocket streams dataset.updated events until disconnect. +# @SIDE_EFFECT Subscribes to dataset event queue in task manager lifecycle. +@app.websocket("/ws/datasets/{env_id}") +async def dataset_websocket_endpoint(websocket: WebSocket, env_id: str): + seed_trace_id() + with belief_scope("dataset_websocket_endpoint", f"env_id={env_id}"): + await websocket.accept() + logger.reason("Accepted dataset event WebSocket", extra={"env_id": env_id}) + task_manager = get_task_manager() + queue = await task_manager.subscribe_dataset_events(env_id) + try: + while True: + event = await queue.get() + await websocket.send_json(event) + logger.reflect("Forwarded dataset.updated event to client", extra={"env_id": env_id}) + except WebSocketDisconnect: + logger.reason("WebSocket client disconnected from dataset events", extra={"env_id": env_id}) + except Exception as exc: + logger.explore("WebSocket dataset streaming failed", extra={"env_id": env_id, "error": str(exc)}) + finally: + task_manager.unsubscribe_dataset_events(env_id, queue) + logger.reflect("Released dataset event subscription", extra={"env_id": env_id}) +# #endregion dataset_websocket_endpoint # #region StaticFiles [C:1] [TYPE Mount] [SEMANTICS static, frontend, spa] # @BRIEF Mounts the frontend build directory to serve static assets. frontend_path = project_root / "frontend" / "build" diff --git a/backend/src/core/superset_client/_datasets.py b/backend/src/core/superset_client/_datasets.py index 67c6a756..c2355bcf 100644 --- a/backend/src/core/superset_client/_datasets.py +++ b/backend/src/core/superset_client/_datasets.py @@ -86,6 +86,22 @@ class SupersetDatasetsMixin: "description": col.get("description", ""), } ) + metrics = dataset.get("metrics", []) + metric_info = [] + for metric in metrics: + metric_id = metric.get("id") + if metric_id is None: + continue + metric_info.append( + { + "id": int(metric_id), + "metric_name": metric.get("metric_name"), + "expression": metric.get("expression"), + "verbose_name": metric.get("verbose_name"), + "description": metric.get("description", ""), + "metric_type": metric.get("metric_type"), + } + ) linked_dashboards = [] try: related_objects = self.network.request( @@ -139,6 +155,8 @@ class SupersetDatasetsMixin: "description": dataset.get("description", ""), "columns": column_info, "column_count": len(column_info), + "metrics": metric_info, + "metric_count": len(metric_info), "sql": sql, "linked_dashboards": linked_dashboards, "linked_dashboard_count": len(linked_dashboards), @@ -147,7 +165,7 @@ class SupersetDatasetsMixin: "changed_on": dataset.get("changed_on"), } app_logger.reflect( - f"Got dataset {dataset_id} with {len(column_info)} columns and {len(linked_dashboards)} linked dashboards", + f"Got dataset {dataset_id} with {len(column_info)} columns, {len(metric_info)} metrics, and {len(linked_dashboards)} linked dashboards", extra={"src": "get_dataset_detail"}, ) return result @@ -169,12 +187,13 @@ class SupersetDatasetsMixin: # @PURPOSE: Обновляет данные датасета по его ID. # @SIDE_EFFECT: Modifies resource in upstream Superset environment. # @RELATION: CALLS -> [APIClient] - def update_dataset(self, dataset_id: int, data: dict) -> dict: + def update_dataset(self, dataset_id: int, data: dict, override_columns: bool = False) -> dict: with belief_scope("SupersetClient.update_dataset", f"id={dataset_id}"): app_logger.info("[update_dataset][Enter] Updating dataset %s.", dataset_id) + endpoint = f"/dataset/{dataset_id}?override_columns={'true' if override_columns else 'false'}" response = self.network.request( method="PUT", - endpoint=f"/dataset/{dataset_id}", + endpoint=endpoint, data=json.dumps(data), headers={"Content-Type": "application/json"}, ) diff --git a/backend/src/core/task_manager/lifecycle.py b/backend/src/core/task_manager/lifecycle.py index 47845562..d4e43b54 100644 --- a/backend/src/core/task_manager/lifecycle.py +++ b/backend/src/core/task_manager/lifecycle.py @@ -71,6 +71,7 @@ class JobLifecycle: self.persistence_service = persistence_service self.executor = executor self.loop = loop + self._dataset_subscribers: dict[str, list[asyncio.Queue]] = {} # #endregion __init__ # #region create_task [C:4] [TYPE Function] [SEMANTICS task,create,persist,schedule] @@ -190,6 +191,12 @@ class JobLifecycle: "Task lifecycle reached persisted terminal state", extra={"task_id": task_id, "status": str(task.status)}, ) + # Broadcast dataset.updated for mapping and documentation tasks (FR-024, R4) + if task.plugin_id in ("dataset-mapper", "llm_documentation"): + dataset_ids = task.params.get("dataset_ids") or [task.params.get("dataset_id")] + env_id = task.params.get("env") or task.params.get("environment_id", "") + if dataset_ids and env_id: + self._broadcast_dataset_updated(env_id, dataset_ids) # #endregion _run_task # #region resolve_task [C:3] [TYPE Function] [SEMANTICS task,resolve,mapping,resume] @@ -309,5 +316,37 @@ class JobLifecycle: ) self.graph.resolve_future(task_id, True) # #endregion resume_task_with_password + + # #region subscribe_dataset_events [C:2] [TYPE Function] [SEMANTICS dataset,subscribe,events,websocket] + # @BRIEF Subscribe to dataset.updated events for a specific environment. + async def subscribe_dataset_events(self, env_id: str) -> asyncio.Queue: + queue = asyncio.Queue() + if env_id not in self._dataset_subscribers: + self._dataset_subscribers[env_id] = [] + self._dataset_subscribers[env_id].append(queue) + return queue + # #endregion subscribe_dataset_events + + # #region unsubscribe_dataset_events [C:2] [TYPE Function] [SEMANTICS dataset,unsubscribe,events] + # @BRIEF Unsubscribe from dataset.updated events for a specific environment. + def unsubscribe_dataset_events(self, env_id: str, queue: asyncio.Queue): + if env_id in self._dataset_subscribers: + if queue in self._dataset_subscribers[env_id]: + self._dataset_subscribers[env_id].remove(queue) + if not self._dataset_subscribers[env_id]: + del self._dataset_subscribers[env_id] + # #endregion unsubscribe_dataset_events + + # #region _broadcast_dataset_updated [C:3] [TYPE Function] [SEMANTICS dataset,broadcast,event,websocket] + # @BRIEF Broadcast dataset.updated event to all subscribers for a given environment. + def _broadcast_dataset_updated(self, env_id: str, dataset_ids: list[int]): + event = {"type": "dataset.updated", "payload": {"env_id": env_id, "dataset_ids": dataset_ids}} + if env_id in self._dataset_subscribers: + for queue in self._dataset_subscribers[env_id]: + try: + self.loop.call_soon_threadsafe(queue.put_nowait, event) + except Exception: + pass + # #endregion _broadcast_dataset_updated # #endregion JobLifecycle # #endregion JobLifecycleModule diff --git a/backend/src/core/task_manager/manager.py b/backend/src/core/task_manager/manager.py index 560a2f60..c2f0ca36 100644 --- a/backend/src/core/task_manager/manager.py +++ b/backend/src/core/task_manager/manager.py @@ -318,5 +318,19 @@ class TaskManager: add_log_callback=self._add_log, ) # #endregion resume_task_with_password + + # ── Dataset event delegates to JobLifecycle ── + + # #region subscribe_dataset_events [TYPE Function] [C:2] + # @BRIEF Subscribe to dataset.updated events for an environment. + async def subscribe_dataset_events(self, env_id: str) -> asyncio.Queue: + return await self.lifecycle.subscribe_dataset_events(env_id) + # #endregion subscribe_dataset_events + + # #region unsubscribe_dataset_events [TYPE Function] [C:2] + # @BRIEF Unsubscribe from dataset.updated events. + def unsubscribe_dataset_events(self, env_id: str, queue: asyncio.Queue): + self.lifecycle.unsubscribe_dataset_events(env_id, queue) + # #endregion unsubscribe_dataset_events # #endregion TaskManager # #endregion TaskManagerModule diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index e03c2944..af3deef1 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -11,7 +11,7 @@ import sys from pathlib import Path # Set env defaults for module-level AuthConfig() singleton — crash-early fix -os.environ.setdefault("AUTH_SECRET_KEY", "test-secret-key-for-testing") +os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing") os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:test_auth") os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:test_main") os.environ.setdefault("DEV_MODE", "true") diff --git a/backend/tests/test_datasets.py b/backend/tests/test_datasets.py new file mode 100644 index 00000000..6e6e9266 --- /dev/null +++ b/backend/tests/test_datasets.py @@ -0,0 +1,419 @@ +# [DEF:TestDatasetsApi:Module] +# @RELATION: DEPENDS_ON -> [DatasetsApi] +# @SEMANTICS: tests, datasets, api, metrics, inline-edit +# @PURPOSE: Contract tests for dataset endpoints (list+stats, detail+metrics, inline-edit descriptions). +# @LAYER: Domain (Tests) +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from src.app import app +from src.dependencies import ( + get_config_manager, + get_current_user, + get_resource_service, + get_task_manager, + has_permission, +) + +client = TestClient(app) + +# [DEF:mock_user:Function] +# @PURPOSE: Create a mock user with Admin role and all permissions for testing. +def _make_mock_user(): + admin_role = MagicMock() + admin_role.name = "Admin" + admin_role.permissions = [] + + mock_user = MagicMock() + mock_user.username = "test-user" + mock_user.roles = [admin_role] + return mock_user + + +# [DEF:mock_deps:Function] +# @PURPOSE: Provide dependency override fixture for dataset route tests. +# @TEST_FIXTURE: dataset_route_overrides -> INLINE_JSON +@pytest.fixture +def mock_deps(): + config_manager = MagicMock() + task_manager = MagicMock() + resource_service = MagicMock() + + env = MagicMock() + env.id = "ss-dev" + env.name = "SS Dev" + env.superset_url = "http://superset:8088" + env.superset_token = "mock-token" + config_manager.get_environments.return_value = [env] + + task_manager.get_all_tasks.return_value = [] + + resource_service.get_datasets_with_status = AsyncMock( + return_value=[ + { + "id": 1, + "table_name": "users", + "schema": "public", + "database": "postgres", + "mapped_fields": {"total": 10, "mapped": 7}, + "last_task": {"task_id": "t1", "status": "SUCCESS"}, + "linked_dashboard_count": 2, + }, + { + "id": 2, + "table_name": "orders", + "schema": "public", + "database": "postgres", + "mapped_fields": {"total": 5, "mapped": 0}, + "last_task": None, + "linked_dashboard_count": 0, + }, + { + "id": 3, + "table_name": "products", + "schema": "public", + "database": "postgres", + "mapped_fields": None, + "last_task": None, + "linked_dashboard_count": 0, + }, + ] + ) + + mock_user = _make_mock_user() + + app.dependency_overrides[get_config_manager] = lambda: config_manager + app.dependency_overrides[get_task_manager] = lambda: task_manager + app.dependency_overrides[get_resource_service] = lambda: resource_service + app.dependency_overrides[get_current_user] = lambda: mock_user + # Override specific permission dependencies used by datasets routes + app.dependency_overrides[has_permission("plugin:migration", "READ")] = lambda: mock_user + app.dependency_overrides[has_permission("plugin:migration", "WRITE")] = lambda: mock_user + + yield + + app.dependency_overrides.clear() + + +# [DEF:test_get_datasets_returns_stats:Function] +# @PURPOSE: Verify GET /api/datasets returns stats object with correct counts. +# @TEST_CONTRACT: datasets_list -> datasets with stats payload +# @TEST_SCENARIO: datasets_with_stats -> HTTP 200 returns datasets, stats, and pagination. +# @TEST_INVARIANT: stats_counts_correct -> VERIFIED_BY: [datasets_with_stats] +def test_get_datasets_returns_stats(mock_deps): + response = client.get( + "/api/datasets?env_id=ss-dev&page=1&page_size=10" + ) + assert response.status_code == 200 + data = response.json() + assert "datasets" in data + assert "stats" in data + assert data["stats"]["total"] == 3 + assert data["stats"]["mapped_count"] == 1 # users: 7/10 + assert data["stats"]["unmapped_count"] == 1 # orders: 0/5 + assert data["stats"]["linked_count"] == 1 # users has 2 linked dashboards + assert data["total"] == 3 + assert data["page"] == 1 + assert len(data["datasets"]) == 3 + + +# [DEF:test_get_datasets_filter_unmapped:Function] +# @PURPOSE: Verify server-side filtering by unmapped datasets. +# @TEST_SCENARIO: filter_unmapped -> Only returns datasets with mapped=0. +def test_get_datasets_filter_unmapped(mock_deps): + response = client.get( + "/api/datasets?env_id=ss-dev&filter=unmapped&page=1&page_size=10" + ) + assert response.status_code == 200 + data = response.json() + # stats are global (unfiltered) + assert data["stats"]["total"] == 3 + # filtered list: only orders (mapped=0) + assert data["total"] == 1 + assert len(data["datasets"]) == 1 + assert data["datasets"][0]["table_name"] == "orders" + + +# [DEF:test_get_datasets_filter_mapped:Function] +# @PURPOSE: Verify server-side filtering by mapped datasets. +def test_get_datasets_filter_mapped(mock_deps): + response = client.get( + "/api/datasets?env_id=ss-dev&filter=mapped&page=1&page_size=10" + ) + assert response.status_code == 200 + data = response.json() + assert data["stats"]["total"] == 3 + assert data["total"] == 1 + assert data["datasets"][0]["table_name"] == "users" + + +# [DEF:test_get_datasets_filter_linked:Function] +# @PURPOSE: Verify server-side filtering by datasets linked to dashboards. +def test_get_datasets_filter_linked(mock_deps): + response = client.get( + "/api/datasets?env_id=ss-dev&filter=linked&page=1&page_size=10" + ) + assert response.status_code == 200 + data = response.json() + assert data["stats"]["total"] == 3 + assert data["total"] == 1 + assert data["datasets"][0]["table_name"] == "users" + + +# [DEF:test_get_datasets_filter_all:Function] +# @PURPOSE: Verify filter=all returns all datasets (same as no filter). +def test_get_datasets_filter_all(mock_deps): + response = client.get( + "/api/datasets?env_id=ss-dev&filter=all&page=1&page_size=10" + ) + assert response.status_code == 200 + data = response.json() + assert data["total"] == 3 + + +# [DEF:test_get_dataset_detail_returns_metrics:Function] +# @PURPOSE: Verify GET /api/datasets/{id} returns metrics and metric_count. +# @TEST_SCENARIO: detail_with_metrics -> Response includes metrics list and metric_count. +def test_get_dataset_detail_returns_metrics(mock_deps): + with patch("src.api.routes.datasets.SupersetClient") as MockClient: + mock_client = MagicMock() + MockClient.return_value = mock_client + + mock_client.get_dataset_detail.return_value = { + "id": 1, + "table_name": "users", + "schema": "public", + "database": "postgres", + "description": "User table", + "columns": [ + {"id": 10, "name": "id", "type": "INT", "is_dttm": False, "is_active": True, "description": "PK"}, + {"id": 11, "name": "name", "type": "STRING", "is_dttm": False, "is_active": True, "description": ""}, + ], + "column_count": 2, + "metrics": [ + {"id": 100, "metric_name": "count", "expression": "COUNT(*)", "verbose_name": "Row Count", "description": "Total rows", "metric_type": "count"}, + ], + "metric_count": 1, + "sql": "SELECT * FROM users", + "linked_dashboards": [{"id": 5, "title": "Users Dashboard", "slug": "users-dash"}], + "linked_dashboard_count": 1, + "is_sqllab_view": False, + "created_on": "2024-01-01T00:00:00", + "changed_on": "2024-06-01T00:00:00", + } + + response = client.get( + "/api/datasets/1?env_id=ss-dev" + ) + assert response.status_code == 200 + data = response.json() + assert "metrics" in data + assert "metric_count" in data + assert data["metric_count"] == 1 + assert len(data["metrics"]) == 1 + assert data["metrics"][0]["metric_name"] == "count" + assert data["metrics"][0]["expression"] == "COUNT(*)" + + +# [DEF:test_update_column_description:Function] +# @PURPOSE: Verify PUT /api/datasets/{id}/columns/{col_id}/description saves description. +# @TEST_SCENARIO: save_column_desc -> Saves and returns updated description. +def test_update_column_description(mock_deps): + with patch("src.api.routes.datasets.SupersetClient") as MockClient: + mock_client = MagicMock() + MockClient.return_value = mock_client + + mock_client.get_dataset.return_value = { + "id": 1, + "result": { + "id": 1, + "columns": [ + {"id": 10, "column_name": "id", "type": "INT", "description": "old desc"}, + {"id": 11, "column_name": "name", "type": "STRING", "description": ""}, + ], + "metrics": [], + }, + } + mock_client.update_dataset.return_value = {"id": 1} + + response = client.put( + "/api/datasets/1/columns/10/description?env_id=ss-dev", + json={"description": "new description"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["success"] is True + assert data["description"] == "new description" + + # Verify update_dataset was called with correct column modified + update_call = mock_client.update_dataset.call_args + assert update_call[0][0] == 1 # dataset_id + # Check override_columns=false was passed + assert update_call[1]["override_columns"] is False + + +# [DEF:test_update_metric_description:Function] +# @PURPOSE: Verify PUT /api/datasets/{id}/metrics/{metric_id}/description saves correctly. +# @TEST_SCENARIO: save_metric_desc -> Saves and returns updated description. +def test_update_metric_description(mock_deps): + with patch("src.api.routes.datasets.SupersetClient") as MockClient: + mock_client = MagicMock() + MockClient.return_value = mock_client + + mock_client.get_dataset.return_value = { + "id": 1, + "result": { + "id": 1, + "columns": [], + "metrics": [ + {"id": 100, "metric_name": "count", "expression": "COUNT(*)", "description": ""}, + ], + }, + } + mock_client.update_dataset.return_value = {"id": 1} + + response = client.put( + "/api/datasets/1/metrics/100/description?env_id=ss-dev", + json={"description": "Total row count"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["success"] is True + assert data["description"] == "Total row count" + + +# [DEF:test_update_column_description_not_found:Function] +# @PURPOSE: Verify 404 when column not found. +def test_update_column_description_not_found(mock_deps): + with patch("src.api.routes.datasets.SupersetClient") as MockClient: + mock_client = MagicMock() + MockClient.return_value = mock_client + + mock_client.get_dataset.return_value = { + "id": 1, + "result": { + "id": 1, + "columns": [{"id": 10, "column_name": "id"}], + "metrics": [], + }, + } + + response = client.put( + "/api/datasets/1/columns/999/description?env_id=ss-dev", + json={"description": "test"}, + ) + assert response.status_code == 404 + + +# [DEF:test_update_column_description_validation:Function] +# @PURPOSE: Verify 400 when description exceeds max length. +def test_update_column_description_validation(mock_deps): + response = client.put( + "/api/datasets/1/columns/10/description?env_id=ss-dev", + json={"description": "x" * 2001}, + ) + assert response.status_code == 400 + + +# [DEF:test_get_datasets_superset_503:Function] +# @PURPOSE: Verify GET /api/datasets returns 503 when Superset/ResourceService throws. +# @TEST_SCENARIO: upstream_superset_failure -> 503 with error detail. +# @TEST_INVARIANT: upstream_failure_503 -> VERIFIED_BY: [superset_failure] +def test_get_datasets_superset_503(mock_deps): + # Override resource_service to throw during get_datasets_with_status + from src.dependencies import get_resource_service + + failing_resource_service = MagicMock() + failing_resource_service.get_datasets_with_status = AsyncMock( + side_effect=Exception("Superset connection refused") + ) + + app.dependency_overrides[get_resource_service] = lambda: failing_resource_service + + try: + response = client.get( + "/api/datasets?env_id=ss-dev&page=1&page_size=10" + ) + assert response.status_code == 503 + assert "Superset" in response.text or "Failed" in response.text + finally: + app.dependency_overrides[get_resource_service] = None + + +# [DEF:test_update_column_description_superset_502:Function] +# @PURPOSE: Verify PUT column description returns 502 when SupersetClient.update_dataset throws. +# @TEST_SCENARIO: superset_put_failure -> 502 with error detail. +def test_update_column_description_superset_502(mock_deps): + with patch("src.api.routes.datasets.SupersetClient") as MockClient: + mock_client = MagicMock() + MockClient.return_value = mock_client + + # GET succeeds + mock_client.get_dataset.return_value = { + "id": 1, + "result": { + "id": 1, + "columns": [{"id": 10, "column_name": "id", "type": "INT", "description": "old desc"}], + "metrics": [], + }, + } + # PUT fails + mock_client.update_dataset.side_effect = Exception("Superset write failure") + + response = client.put( + "/api/datasets/1/columns/10/description?env_id=ss-dev", + json={"description": "new description"}, + ) + assert response.status_code == 502 + data = response.json() + assert "Superset" in data.get("detail", "") + + +# [DEF:test_update_column_description_strips_html:Function] +# @PURPOSE: Verify HTML tags are stripped from description before saving. +# @TEST_SCENARIO: html_in_description -> HTML tags stripped, clean text saved. +def test_update_column_description_strips_html(mock_deps): + with patch("src.api.routes.datasets.SupersetClient") as MockClient: + mock_client = MagicMock() + MockClient.return_value = mock_client + + mock_client.get_dataset.return_value = { + "id": 1, + "result": { + "id": 1, + "columns": [{"id": 10, "column_name": "id", "type": "INT", "description": ""}], + "metrics": [], + }, + } + mock_client.update_dataset.return_value = {"id": 1} + + response = client.put( + "/api/datasets/1/columns/10/description?env_id=ss-dev", + json={"description": "Hello World with "}, + ) + assert response.status_code == 200 + data = response.json() + # HTML stripped + assert "Hello World with alert('xss')" == data["description"] + assert "" not in data["description"] + assert " -
+
-
+

{$t.nav?.datasets}

-
- - {$t.dashboard?.environment}: {activeEnvironment?.name || - selectedEnv || - "-"} +
+ + {$t.datasets?.all}: {stats?.total ?? 0} -
- - {#if error} -
- {error} - + + {#if dsError} + {/if} - - {#if isLoading} -
+ + + + +
+ +
+ +
+ + + {#if isMobile && mobileSlideOpen} + + + - {#each Array(5) as _} -
-
-
-
-
-
-
-
- {/each} -
- {:else if datasets.length === 0} - -
- - - -

{$t.datasets?.empty}

-
- {:else} - -
-
- - - {#if selectedIds.size > 0} - - {$t.datasets?.selected_count?.replace( - "{count}", - String(selectedIds.size), - )} - - {/if} -
-
- -
-
- - -
- -
-
-
- {$t.datasets?.table_name} -
-
{$t.datasets?.schema}
-
{$t.datasets?.mapped_fields}
-
{$t.datasets?.last_task}
-
{$t.datasets?.actions}
-
- - - {#each datasets as dataset} -
- -
- handleCheckboxChange(dataset, e)} - /> -
- - - - - -
- {dataset.schema} -
- - -
- {#if dataset.mappedFields} -
-
-
- {:else} - - - {/if} -
- - -
- {#if dataset.lastTask} - - {:else} - - - {/if} -
- - -
- {#if dataset.actions.includes("map_columns")} - - {/if} -
-
- {/each} -
- - - {#if totalPages > 1} -
-
- {$t.datasets?.showing - .replace("{start}", String((currentPage - 1) * pageSize + 1)) - .replace("{end}", String(Math.min(currentPage * pageSize, total))) - .replace("{total}", String(total))} -
-
- - - {#each Array.from({ length: totalPages }, (_, i) => i + 1) as pageNum} - - {/each} - - -
-
- -
+ {:else if !isMobile} +
+ selectedDatasetId && loadDetail(selectedDatasetId)} envId={selectedEnv} />
{/if} +
- - {#if selectedIds.size > 0} -
-
-
- - ✓ {selectedIds.size} - {$t.datasets?.selected} - -
-
- - - -
+ + {#if selectedIds.size >= 2} +
+
+ ✓ {selectedIds.size} {$t.datasets?.selected} +
+ + +
- {/if} +
{/if} {#if showMapColumnsModal} -
(showMapColumnsModal = false)} - onkeydown={(event) => event.key === "Escape" && (showMapColumnsModal = false)} - role="button" - tabindex="0" - aria-label={$t.common?.close_modal} - > -
-
-

{$t.datasets?.bulk_map_columns}

- -
-
-
-
- - -
- {#if mapSourceType === "postgresql"} -
- - -
- {:else} -
- - -
- {/if} -
-

- {$t.datasets?.selected_datasets} -

-
- {#each Array.from(selectedIds) as id} - {#each datasets as d} - {#if d.id === id} -
- {d.table_name} -
- {/if} - {/each} - {/each} -
-
-
-
-
- - + @@ -879,96 +365,22 @@ {#if showGenerateDocsModal} -
(showGenerateDocsModal = false)} - onkeydown={(event) => event.key === "Escape" && (showGenerateDocsModal = false)} - role="button" - tabindex="0" - aria-label={$t.common?.close_modal} - > -
-
-

{$t.datasets?.bulk_docs_generation}

- -
-
-
-
- - -
-
-

- {$t.datasets?.selected_datasets} -

-
- {#each Array.from(selectedIds) as id} - {#each datasets as d} - {#if d.id === id} -
- {d.table_name} -
- {/if} - {/each} - {/each} -
-
-
-
-
- - + {/if}
- - + \ No newline at end of file diff --git a/frontend/src/routes/datasets/ColumnsTable.svelte b/frontend/src/routes/datasets/ColumnsTable.svelte new file mode 100644 index 00000000..3878524d --- /dev/null +++ b/frontend/src/routes/datasets/ColumnsTable.svelte @@ -0,0 +1,135 @@ + + + + + + + + + +
+ + + + + + + + + + + {#if columns.length === 0} + + {:else} + {#each columns as col (col.id)} + + + + + + + {/each} + {/if} + +
{$t.datasets?.table_name ?? 'Column'}{$t.dashboard?.type ?? 'Type'}{$t.dashboard?.description ?? 'Description'}
{$t.datasets?.detail_empty ?? 'No data'}
+ {#if col.verbose_name} + {col.verbose_name} + {:else} + {col.name} + {/if} + + + {col.type ?? '—'} + + + {#if editingRowId === col.id} +
+ + {#if editError} + {editError} + {/if} +
+ + +
+
+ {:else if col.description} + {col.description} + {:else} + {'✏️ ' + ($t.datasets?.add_description ?? 'Add description')} + {/if} +
+ {#if editingRowId !== col.id} + + {/if} +
+
+ \ No newline at end of file diff --git a/frontend/src/routes/datasets/DatasetList.svelte b/frontend/src/routes/datasets/DatasetList.svelte new file mode 100644 index 00000000..7ec59ac5 --- /dev/null +++ b/frontend/src/routes/datasets/DatasetList.svelte @@ -0,0 +1,168 @@ + + + + + + + + + + + +
+ +
+ + +
+
+ + +{#if isLoading} +
+ {#each Array(3) as _} +
+
+
+
+
+ {/each} +
+{:else if error} +
+ {error} +
+{:else if datasets.length === 0} +
{$t.datasets?.empty}
+{:else} + +
+ {#each datasets as dataset} +
handleCardClick(dataset, e)} + role="button" + tabindex="0" + > +
+ + handleCheckboxChange(dataset, e)} + /> + +
+
{dataset.table_name}
+
+ {dataset.schema ?? '-'} + {dataset.column_count ?? 0} {$t.datasets?.columns?.toLowerCase() ?? 'columns'} + {#if dataset.metric_count > 0} + {dataset.metric_count} {$t.datasets?.metric_count ?? 'metrics'} + {/if} +
+ + {#if dataset.mappedFields?.total} +
+
+
+
+
+ {dataset.mappedFields.mapped}/{dataset.mappedFields.total} {$t.datasets?.mapped_of_total} +
+
+ {/if} +
+ +
+ + +
+
+
+ {/each} +
+ + + {#if totalPages > 1} +
+ + {($t.datasets?.showing ?? 'Showing {start}-{end} of {total}').replace('{start}', String((page - 1) * pageSize + 1)).replace('{end}', String(Math.min(page * pageSize, total))).replace('{total}', String(total))} + +
+ + + {page} / {totalPages} + + +
+
+ {/if} +{/if} + \ No newline at end of file diff --git a/frontend/src/routes/datasets/DatasetPreview.svelte b/frontend/src/routes/datasets/DatasetPreview.svelte new file mode 100644 index 00000000..f3152a60 --- /dev/null +++ b/frontend/src/routes/datasets/DatasetPreview.svelte @@ -0,0 +1,84 @@ + + + + + + + + + + + +{#if isLoading} +
+
+
+
+ {#each Array(3) as _, i (i)} +
+ {/each} +
+
+{:else if error} + +{:else if !dataset} +
+
+ +

{$t.datasets?.no_selection ?? 'Select a dataset from the list'}

+
+
+{:else} +
+ +
+

{dataset.table_name ?? dataset.id}

+
+ {#if dataset.schema}📁 {dataset.schema}{/if} + {dataset.column_count ?? 0} {$t.datasets?.columns?.toLowerCase() ?? 'columns'} + {dataset.metric_count ?? 0} {$t.datasets?.metric_count ?? 'metrics'} + {#if dataset.linked_dashboard_count > 0} + 📊 {dataset.linked_dashboard_count} dashboards + {/if} +
+ {#if dataset.linked_dashboards?.length} +
+ {#each dataset.linked_dashboards as dash} + + {dash.title} + + {/each} +
+ {/if} + {#if dataset.created_on} +
{$t.dashboard?.created ?? 'Created'}: {dataset.created_on}
+ {/if} +
+ + +
+

{$t.datasets?.columns ?? 'Columns'} ({dataset.column_count ?? 0})

+ +
+ + +
+

{$t.datasets?.metrics ?? 'Metrics'} ({dataset.metric_count ?? 0})

+ +
+
+{/if} + \ No newline at end of file diff --git a/frontend/src/routes/datasets/MetricsTable.svelte b/frontend/src/routes/datasets/MetricsTable.svelte new file mode 100644 index 00000000..bd528225 --- /dev/null +++ b/frontend/src/routes/datasets/MetricsTable.svelte @@ -0,0 +1,121 @@ + + + + + + + + + +
+ + + + + + + + + + {#if metrics.length === 0} + + {:else} + {#each metrics as metric (metric.id)} + + + + + + {/each} + {/if} + +
{$t.datasets?.metrics ?? 'Metric'}{$t.dashboard?.description ?? 'Description'}
{$t.datasets?.detail_empty ?? 'No data'}
+ {#if metric.verbose_name} + {metric.verbose_name} + {:else} + {metric.metric_name} + {/if} + {#if !metric.description && metric.expression} +
{metric.expression}
+ {/if} +
+ {#if editingRowId === metric.id} +
+ + {#if editError} + {editError} + {/if} +
+ + +
+
+ {:else if metric.description} + {metric.description} + {:else} + {'✏️ ' + ($t.datasets?.add_description ?? 'Add description')} + {/if} +
+ {#if editingRowId !== metric.id} + + {/if} +
+
+ \ No newline at end of file diff --git a/frontend/src/routes/datasets/StatsBar.svelte b/frontend/src/routes/datasets/StatsBar.svelte new file mode 100644 index 00000000..567ce5e2 --- /dev/null +++ b/frontend/src/routes/datasets/StatsBar.svelte @@ -0,0 +1,51 @@ + + + + + + + + + + + +
+ {#each tiles as tile (tile.key)} + {#if isLoading} +
+
+
+
+ {:else} + + {/if} + {/each} +
+ \ No newline at end of file diff --git a/frontend/src/routes/datasets/__tests__/ColumnsTable.test.js b/frontend/src/routes/datasets/__tests__/ColumnsTable.test.js new file mode 100644 index 00000000..ee143885 --- /dev/null +++ b/frontend/src/routes/datasets/__tests__/ColumnsTable.test.js @@ -0,0 +1,68 @@ +// #region ColumnsTableTest [C:2] [TYPE Module] [SEMANTICS test, datasets, columns, component] +// @BRIEF Contract-focused unit tests for ColumnsTable.svelte component. +// @LAYER Test +import { describe, it, expect } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; + +const COMPONENT_PATH = path.resolve(process.cwd(), 'src/routes/datasets/ColumnsTable.svelte'); + +describe('ColumnsTable Component', () => { + it('component file exists', () => { + expect(fs.existsSync(COMPONENT_PATH)).toBe(true); + }); + + it('bold verbose_name / italic column_name fallback', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('verbose_name'); + expect(src).toContain('font-medium'); + expect(src).toContain('italic'); + expect(src).toContain('name'); + }); + + it('has type chip color classes', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('getTypeColor'); + expect(src).toContain('bg-green-100'); + expect(src).toContain('bg-purple-100'); + expect(src).toContain('bg-blue-100'); + expect(src).toContain('bg-orange-100'); + }); + + it('has inline-edit state machine (editing/saving/error)', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('editingRowId'); + expect(src).toContain('editValue'); + expect(src).toContain('editError'); + expect(src).toContain('isSaving'); + expect(src).toContain('startEdit'); + expect(src).toContain('cancelEdit'); + expect(src).toContain('saveEdit'); + }); + + it('textarea for editing with save/cancel buttons', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('textarea'); + expect(src).toContain('save_description'); + expect(src).toContain('maxlength'); + }); + + it('shows error message with aria-live', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('aria-live'); + expect(src).toContain('editError'); + }); + + it('shows empty state for 0 columns', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('columns.length === 0'); + expect(src).toContain('detail_empty'); + }); + + it('calls PUT /datasets/{id}/columns/{col_id}/description', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('/columns/'); + expect(src).toContain('/description'); + expect(src).toContain('api.requestApi'); + }); +}); diff --git a/frontend/src/routes/datasets/__tests__/DatasetList.test.js b/frontend/src/routes/datasets/__tests__/DatasetList.test.js new file mode 100644 index 00000000..a06626fa --- /dev/null +++ b/frontend/src/routes/datasets/__tests__/DatasetList.test.js @@ -0,0 +1,70 @@ +// #region DatasetListTest [C:2] [TYPE Module] [SEMANTICS test, datasets, cards, component] +// @BRIEF Contract-focused unit tests for DatasetList.svelte component. +// @LAYER Test +import { describe, it, expect } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; + +const COMPONENT_PATH = path.resolve(process.cwd(), 'src/routes/datasets/DatasetList.svelte'); + +describe('DatasetList Component', () => { + it('component file exists', () => { + expect(fs.existsSync(COMPONENT_PATH)).toBe(true); + }); + + it('renders cards with table_name, schema, column_count, metric_count', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('table_name'); + expect(src).toContain('schema'); + expect(src).toContain('column_count'); + expect(src).toContain('metric_count'); + }); + + it('has progress bar with color logic', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('getProgressClass'); + expect(src).toContain('bg-green-500'); + expect(src).toContain('bg-yellow-400'); + expect(src).toContain('bg-blue-400'); + expect(src).toContain('mappedFields'); + }); + + it('has pagination controls', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('onpagechange'); + expect(src).toContain('totalPages'); + expect(src).toContain('page'); + expect(src).toContain('pageSize'); + }); + + it('has search input', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('onsearch'); + expect(src).toContain('search_placeholder'); + }); + + it('has checkboxes for bulk selection', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('checkbox'); + expect(src).toContain('oncheckbox'); + expect(src).toContain('selectedIds'); + }); + + it('has quick-action buttons (map, docs)', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('action_map_columns'); + expect(src).toContain('generate_docs'); + }); + + it('shows skeleton when loading', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('animate-pulse'); + expect(src).toContain('isLoading'); + }); + + it('shows empty state when no datasets', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('empty'); + expect(src).toContain('datasets.length === 0'); + }); +}); diff --git a/frontend/src/routes/datasets/__tests__/DatasetPreview.test.js b/frontend/src/routes/datasets/__tests__/DatasetPreview.test.js new file mode 100644 index 00000000..7aa097cb --- /dev/null +++ b/frontend/src/routes/datasets/__tests__/DatasetPreview.test.js @@ -0,0 +1,62 @@ +// #region DatasetPreviewTest [C:2] [TYPE Module] [SEMANTICS test, datasets, detail, preview, component] +// @BRIEF Contract-focused unit tests for DatasetPreview.svelte component. +// @LAYER Test +import { describe, it, expect } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; + +const COMPONENT_PATH = path.resolve(process.cwd(), 'src/routes/datasets/DatasetPreview.svelte'); + +describe('DatasetPreview Component', () => { + it('component file exists', () => { + expect(fs.existsSync(COMPONENT_PATH)).toBe(true); + }); + + it('shows no-selection placeholder', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('no_selection'); + expect(src).toContain('!dataset'); + }); + + it('shows loading skeleton', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('isLoading'); + expect(src).toContain('animate-pulse'); + expect(src).toContain('aria-busy'); + }); + + it('shows error state with retry', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('error'); + expect(src).toContain('onretry'); + expect(src).toContain('retry'); + }); + + it('renders header with table_name, schema, counts', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('table_name'); + expect(src).toContain('schema'); + expect(src).toContain('column_count'); + expect(src).toContain('metric_count'); + }); + + it('renders linked dashboards as clickable pills', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('linked_dashboards'); + expect(src).toContain('dashboards/'); + }); + + it('includes ColumnsTable and MetricsTable', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('ColumnsTable'); + expect(src).toContain('MetricsTable'); + }); + + it('is presentational — receives dataset, isLoading, error props', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('$props()'); + expect(src).toContain('dataset = null'); + expect(src).toContain('isLoading = false'); + expect(src).toContain('error = null'); + }); +}); diff --git a/frontend/src/routes/datasets/__tests__/StatsBar.test.js b/frontend/src/routes/datasets/__tests__/StatsBar.test.js new file mode 100644 index 00000000..51a65d63 --- /dev/null +++ b/frontend/src/routes/datasets/__tests__/StatsBar.test.js @@ -0,0 +1,49 @@ +// #region StatsBarTest [C:2] [TYPE Module] [SEMANTICS test, datasets, statsbar, component] +// @BRIEF Contract-focused unit tests for StatsBar.svelte component. +// @LAYER Test +// @RELATION BINDS_TO -> [StatsBar:Component] +import { describe, it, expect } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; + +const COMPONENT_PATH = path.resolve(process.cwd(), 'src/routes/datasets/StatsBar.svelte'); + +describe('StatsBar Component', () => { + it('component file exists', () => { + expect(fs.existsSync(COMPONENT_PATH)).toBe(true); + }); + + it('renders 4 metric tiles', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('all'); + expect(src).toContain('without_mapping'); + expect(src).toContain('with_mapping'); + expect(src).toContain('linked_to_dashboards'); + }); + + it('dispatches onfilter on tile click', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('onfilter'); + expect(src).toContain('handleTileClick'); + expect(src).toContain('activeFilter === tile.key'); + }); + + it('shows skeleton when loading', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('aria-busy'); + expect(src).toContain('animate-pulse'); + expect(src).toContain('isLoading'); + }); + + it('uses aria-pressed for active state', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('aria-pressed'); + }); + + it('uses $props for reactivity', () => { + const src = fs.readFileSync(COMPONENT_PATH, 'utf-8'); + expect(src).toContain('$props()'); + expect(src).toContain('stats = null'); + expect(src).toContain('activeFilter'); + }); +}); diff --git a/frontend/src/routes/datasets/__tests__/bulk_actions.test.js b/frontend/src/routes/datasets/__tests__/bulk_actions.test.js new file mode 100644 index 00000000..8510944f --- /dev/null +++ b/frontend/src/routes/datasets/__tests__/bulk_actions.test.js @@ -0,0 +1,52 @@ +// #region BulkActionsTest [C:2] [TYPE Module] [SEMANTICS test, datasets, bulk, actions, component] +// @BRIEF Contract-focused unit tests for bulk actions in +page.svelte. +// @LAYER Test +import { describe, it, expect } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; + +const PAGE_PATH = path.resolve(process.cwd(), 'src/routes/datasets/+page.svelte'); + +describe('Bulk Actions', () => { + it('panel appears at 2+ selected', () => { + const src = fs.readFileSync(PAGE_PATH, 'utf-8'); + expect(src).toContain('selectedIds.size >= 2'); + }); + + it('panel disappears at 0 selected', () => { + const src = fs.readFileSync(PAGE_PATH, 'utf-8'); + // Panel is hidden when selectedIds.size === 0 (via {#if selectedIds.size >= 2}) + expect(src).toContain('selectedIds.size'); + expect(src).toContain('SvelteSet'); + }); + + it('map columns modal opens', () => { + const src = fs.readFileSync(PAGE_PATH, 'utf-8'); + expect(src).toContain('showMapColumnsModal'); + expect(src).toContain('bulk_map_columns'); + }); + + it('generate docs modal opens', () => { + const src = fs.readFileSync(PAGE_PATH, 'utf-8'); + expect(src).toContain('showGenerateDocsModal'); + expect(src).toContain('bulk_docs_generation'); + }); + + it('selected count displays correctly', () => { + const src = fs.readFileSync(PAGE_PATH, 'utf-8'); + expect(src).toContain('selectedIds.size'); + expect(src).toContain('selected'); + }); + + it('has clear/deselect button in panel', () => { + const src = fs.readFileSync(PAGE_PATH, 'utf-8'); + expect(src).toContain('selectedIds'); + expect(src).toContain('SvelteSet'); + }); + + it('opens task drawer on bulk action completion', () => { + const src = fs.readFileSync(PAGE_PATH, 'utf-8'); + expect(src).toContain('openDrawerForTaskIfPreferred'); + expect(src).toContain('task_id'); + }); +}); diff --git a/frontend/src/routes/datasets/__tests__/inline_edit.test.js b/frontend/src/routes/datasets/__tests__/inline_edit.test.js new file mode 100644 index 00000000..955cf195 --- /dev/null +++ b/frontend/src/routes/datasets/__tests__/inline_edit.test.js @@ -0,0 +1,58 @@ +// #region InlineEditTest [C:2] [TYPE Module] [SEMANTICS test, datasets, inline-edit, component] +// @BRIEF Contract-focused unit tests for inline-edit in ColumnsTable and MetricsTable. +// @LAYER Test +import { describe, it, expect } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; + +const COLUMNS_PATH = path.resolve(process.cwd(), 'src/routes/datasets/ColumnsTable.svelte'); +const METRICS_PATH = path.resolve(process.cwd(), 'src/routes/datasets/MetricsTable.svelte'); + +describe('Inline Edit', () => { + it('click opens textarea with prefill (columns)', () => { + const src = fs.readFileSync(COLUMNS_PATH, 'utf-8'); + expect(src).toContain('textarea'); + expect(src).toContain('editValue = col.description'); + expect(src).toContain('startEdit'); + }); + + it('save calls API and updates description (columns)', () => { + const src = fs.readFileSync(COLUMNS_PATH, 'utf-8'); + expect(src).toContain('PUT'); + expect(src).toContain('/description'); + expect(src).toContain('col.description = result.description'); + }); + + it('cancel reverts to original (columns)', () => { + const src = fs.readFileSync(COLUMNS_PATH, 'utf-8'); + expect(src).toContain('cancelEdit'); + expect(src).toContain('editingRowId = null'); + }); + + it('save error keeps textarea open (columns)', () => { + const src = fs.readFileSync(COLUMNS_PATH, 'utf-8'); + expect(src).toContain('editError'); + expect(src).toContain('catch'); + expect(src).toContain('isSaving = false'); + }); + + it('save error keeps textarea open (metrics)', () => { + const src = fs.readFileSync(METRICS_PATH, 'utf-8'); + expect(src).toContain('editError'); + expect(src).toContain('catch'); + expect(src).toContain('isSaving = false'); + }); + + it('calls PUT /datasets/{id}/metrics/{metric_id}/description', () => { + const src = fs.readFileSync(METRICS_PATH, 'utf-8'); + expect(src).toContain('/metrics/'); + expect(src).toContain('/description'); + expect(src).toContain('api.requestApi'); + }); + + it('shows expression as hint when no description (metrics)', () => { + const src = fs.readFileSync(METRICS_PATH, 'utf-8'); + expect(src).toContain('expression'); + expect(src).toContain('font-mono'); + }); +}); diff --git a/frontend/src/routes/datasets/__tests__/split_view.test.js b/frontend/src/routes/datasets/__tests__/split_view.test.js new file mode 100644 index 00000000..dbef1be3 --- /dev/null +++ b/frontend/src/routes/datasets/__tests__/split_view.test.js @@ -0,0 +1,62 @@ +// #region SplitViewTest [C:2] [TYPE Module] [SEMANTICS test, datasets, split-view, layout, component] +// @BRIEF Contract-focused unit tests for split-view layout in +page.svelte. +// @LAYER Test +import { describe, it, expect } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; + +const PAGE_PATH = path.resolve(process.cwd(), 'src/routes/datasets/+page.svelte'); + +describe('Split-View Layout', () => { + it('page file exists', () => { + expect(fs.existsSync(PAGE_PATH)).toBe(true); + }); + + it('uses Tailwind grid for split-view layout', () => { + const src = fs.readFileSync(PAGE_PATH, 'utf-8'); + expect(src).toContain('grid-cols-[40%_60%]'); + }); + + it('has mobile slide-over behavior', () => { + const src = fs.readFileSync(PAGE_PATH, 'utf-8'); + expect(src).toContain('isMobile'); + expect(src).toContain('mobileSlideOpen'); + expect(src).toContain('closeDetail'); + expect(src).toContain('fixed inset-0 bg-black/30'); + }); + + it('closes slide-over on Escape', () => { + const src = fs.readFileSync(PAGE_PATH, 'utf-8'); + expect(src).toContain('Escape'); + expect(src).toContain('handleEscape'); + }); + + it('wires environmentContextStore', () => { + const src = fs.readFileSync(PAGE_PATH, 'utf-8'); + expect(src).toContain('environmentContextStore'); + expect(src).toContain('initializeEnvironmentContext'); + expect(src).toContain('setSelectedEnvironment'); + }); + + it('renders StatsBar, DatasetList, DatasetPreview', () => { + const src = fs.readFileSync(PAGE_PATH, 'utf-8'); + expect(src).toContain('StatsBar'); + expect(src).toContain('DatasetList'); + expect(src).toContain('DatasetPreview'); + }); + + it('has bulk action panel for >=2 selected', () => { + const src = fs.readFileSync(PAGE_PATH, 'utf-8'); + expect(src).toContain('selectedIds.size >= 2'); + expect(src).toContain('bulk_map_columns'); + expect(src).toContain('showMapColumnsModal'); + expect(src).toContain('showGenerateDocsModal'); + }); + + it('uses Svelte 5 runes pattern', () => { + const src = fs.readFileSync(PAGE_PATH, 'utf-8'); + expect(src).toContain('$state('); + expect(src).toContain('$effect('); + // Page is a route, not a component, so it doesn't use $props() + }); +}); diff --git a/specs/030-dataset-lifecycle-workspace/checklists/requirements.md b/specs/030-dataset-lifecycle-workspace/checklists/requirements.md index 52ebd10d..a10ba69a 100644 --- a/specs/030-dataset-lifecycle-workspace/checklists/requirements.md +++ b/specs/030-dataset-lifecycle-workspace/checklists/requirements.md @@ -6,7 +6,7 @@ ## Functional Coverage -- [ ] CHK001 Stats Bar renders 4 metrics (Все, Без маппинга, С описанием, С дашбордами) in page header +- [ ] CHK001 Stats Bar renders 4 metrics (Все, Без маппинга, С маппингом, С дашбордами) in page header - [ ] CHK002 Each Stats Bar metric is clickable and filters the dataset list accordingly - [ ] CHK003 Active filter is visually highlighted; clicking again resets filter - [ ] CHK004 Split-view layout renders with 40% left panel and 60% right panel on desktop @@ -22,7 +22,8 @@ - [ ] CHK014 Metric without description shows expression as hint below the name - [ ] CHK015 Inline edit opens textarea with "Save"/"Cancel" buttons - [ ] CHK016 Existing description is pre-filled in inline edit textarea -- [ ] CHK017 Save sends PUT /api/datasets/{id} with updated description +- [ ] CHK017 Save column description sends `PUT /api/datasets/{id}/columns/{col_id}/description` +- [ ] CHK017a Save metric description sends `PUT /api/datasets/{id}/metrics/{metric_id}/description` - [ ] CHK018 Save success shows toast and closes textarea - [ ] CHK019 Save failure shows error message and keeps textarea open - [ ] CHK020 Cards have checkboxes; selecting 2+ shows floating bulk action bar @@ -40,8 +41,8 @@ ## i18n Coverage -- [ ] CHK029 New EN keys: all, without_mapping, with_description, linked_to_dashboards, metrics, metric_count, no_selection, save_description, add_description -- [ ] CHK030 New RU keys: все, без_маппинга, с_описанием, связаны_с_дашбордами, метрики, метрик, выберите_датасет, сохранить_описание, добавить_описание +- [ ] CHK029 New EN keys: all, without_mapping, with_mapping, linked_to_dashboards, metrics, metric_count, no_selection, save_description, add_description +- [ ] CHK030 New RU keys: все, без_маппинга, с_маппингом, связаны_с_дашбордами, метрики, метрик, выберите_датасет, сохранить_описание, добавить_описание - [ ] CHK031 Breadcrumb for /datasets reads "Датасеты" (RU) / "Datasets" (EN) ## Cross-Cutting Concerns diff --git a/specs/030-dataset-lifecycle-workspace/contracts/modules.md b/specs/030-dataset-lifecycle-workspace/contracts/modules.md index af45b1ae..4c567003 100644 --- a/specs/030-dataset-lifecycle-workspace/contracts/modules.md +++ b/specs/030-dataset-lifecycle-workspace/contracts/modules.md @@ -68,6 +68,8 @@ class MetricDescriptionUpdate(BaseModel): @PRE: dataset_id and column_id must exist in the target environment. @POST: Column description in Superset is updated. Response confirms success. @SIDE_EFFECT: Mutates dataset metadata in upstream Superset instance via PUT. +@VALIDATION: description — string, max 2000 chars, plain text only (HTML stripped), may be empty string to clear description, no trimming applied (preserve leading/trailing whitespace). 404 if dataset or column not found. 502/503 if Superset upstream fails. +@ERROR: 400 — description exceeds max length or contains non-plaintext content. 404 — dataset_id or column_id not found. 502 — Superset GET failed. 503 — Superset PUT failed. @RELATION CALLS -> [SupersetClient.get_dataset] @RELATION CALLS -> [SupersetClient.update_dataset] @RATIONALE Must perform GET→modify→PUT because Superset has no PATCH for individual columns — only full object PUT with `override_columns=false`. @@ -77,20 +79,22 @@ class MetricDescriptionUpdate(BaseModel): **Method:** `PUT` **Request:** `ColumnDescriptionUpdate` **Response:** `{success: true, description: "new text"}` -**RBAC:** `has_permission("plugin:migration", "READ")` +**RBAC:** `has_permission("plugin:migration", "WRITE")` ### #region update_metric_description [C:4] [TYPE Endpoint] @BRIEF Save description for a single dataset metric. Mirror of update_column_description for metrics. @PRE: dataset_id and metric_id must exist in the target environment. @POST: Metric description in Superset is updated. @SIDE_EFFECT: Mutates dataset metadata in upstream Superset instance via PUT. +@VALIDATION: description — string, max 2000 chars, plain text only (HTML stripped), may be empty string to clear description, no trimming applied. 404 if dataset or metric not found. 502/503 if Superset upstream fails. +@ERROR: 400 — description exceeds max length or contains non-plaintext content. 404 — dataset_id or metric_id not found. 502 — Superset GET failed. 503 — Superset PUT failed. @RELATION CALLS -> [SupersetClient.get_dataset] @RELATION CALLS -> [SupersetClient.update_dataset] **File:** `backend/src/api/routes/datasets.py` **Route:** `PUT /api/datasets/{dataset_id}/metrics/{metric_id}/description` **Request:** `MetricDescriptionUpdate` **Response:** `{success: true, description: "new text"}` -**RBAC:** `has_permission("plugin:migration", "READ")` +**RBAC:** `has_permission("plugin:migration", "WRITE")` ### #region get_datasets [C:4] [TYPE Endpoint] — MODIFIED @BRIEF Fetch paginated dataset list with enhanced metadata, mapped fields, and stats counts. @@ -119,15 +123,15 @@ class MetricDescriptionUpdate(BaseModel): ## Frontend Contracts ### #region StatsBar [C:3] [TYPE Component] [SEMANTICS ui,dataset,stats,filter] -@BRIEF Displays 4 aggregate metric tiles (All, Without mapping, With description, Linked) and emits filter selection. +@BRIEF Displays 4 aggregate metric tiles (All, Without mapping, Mapped, Linked) and emits filter selection. Filtering is server-side via query param — parent container calls API on tile click. @LAYER UI @RELATION BINDS_TO -> [environmentContextStore] @UX_STATE Loading -> 4 skeleton tiles pulsing. @UX_STATE Loaded -> 4 tiles with numbers, active filter highlighted. -@UX_STATE Filtered -> Selected tile has accent border; list reflects filter. +@UX_STATE Filtered -> Selected tile has accent border; parent triggers API reload with filter param. @UX_STATE Empty -> All tiles show 0. @UX_REACTIVITY Props -> `stats: StatsCounts`, `activeFilter: string|null`. -@UX_REACTIVITY Events -> `onfilter(filterKey)` dispatched on tile click. +@UX_REACTIVITY Events -> `onfilter(filterKey)` dispatched on tile click; parent calls `loadDatasets(filter=filterKey)`. **File:** `frontend/src/routes/datasets/StatsBar.svelte` ### #region DatasetList [C:4] [TYPE Component] [SEMANTICS ui,dataset,card,list,pagination] @@ -147,15 +151,18 @@ class MetricDescriptionUpdate(BaseModel): **File:** `frontend/src/routes/datasets/DatasetList.svelte` ### #region DatasetPreview [C:3] [TYPE Component] [SEMANTICS ui,dataset,detail,columns,metrics] -@BRIEF Right panel showing dataset detail: header info, columns table, metrics table, linked dashboards. +@BRIEF Right panel showing dataset detail: header info, columns table, metrics table, linked dashboards. Presentational component — receives dataset data from parent container. @LAYER UI @RELATION DEPENDS_ON -> [ColumnsTable] @RELATION DEPENDS_ON -> [MetricsTable] @UX_STATE NoSelection -> "Select a dataset from the list" placeholder. @UX_STATE Loading -> Skeleton for header + 2 tables. @UX_STATE Loaded -> Full detail with columns and metrics rendered. -@UX_STATE Error -> Error banner with retry. -@UX_REACTIVITY Props -> `dataset: DatasetDetailResponse | null`. +@UX_STATE Error -> Error banner with retry (retry triggers callback to parent). +@UX_REACTIVITY Props -> `dataset: DatasetDetailResponse | null`, `isLoading: boolean`, `error: string | null`. +@UX_REACTIVITY Events -> `onretry()` callback dispatched when retry button clicked. +@RATIONALE Container (DatasetHub) fetches data and passes it as props; DatasetPreview is a presentational component for cleaner testing and separation of concerns. +@REJECTED Self-fetching component — rejected because it couples data fetching with rendering, harder to test, and creates race conditions when switching datasets rapidly. @UX_TEST: NoSelection -> {click: card, expected: Loading -> Loaded}. **File:** `frontend/src/routes/datasets/DatasetPreview.svelte` @@ -238,7 +245,7 @@ class MetricDescriptionUpdate(BaseModel): | ADR-0002 (semantic protocol) | ACTIVE | ✅ All contracts use GRACE anchors with C1-C5 complexity | | ADR-0003 (orchestrator pattern) | ACTIVE | ✅ Backend proxies Superset API, no direct frontend-to-Superset calls | | ADR-0004 (plugin architecture) | ACTIVE | ✅ Mapping/docs tasks use existing plugin system (no new plugin) | -| ADR-0005 (auth RBAC) | ACTIVE | ✅ New endpoints gated by existing permission checks | +| ADR-0005 (auth RBAC) | ACTIVE | ✅ New mutation endpoints gated by `plugin:migration:WRITE`; read endpoints use `plugin:migration:READ` | | ADR-0006 (frontend architecture) | ACTIVE | ✅ Svelte 5 runes, Tailwind, static SPA, vitest | | ADR-0007 (rejected fromStore) | ACTIVE | ✅ Uses `$state`, avoids deprecated patterns | @@ -253,4 +260,5 @@ class MetricDescriptionUpdate(BaseModel): | Modal-per-row editing | Too many clicks for repetitive column description workflow | Inline edit via textarea toggle | | Multi-page list→detail navigation | Full page reload between dataset selections breaks flow | Split-view within single SvelteKit route | | Polling for Stats Bar refresh | Unnecessary network overhead, user chose WebSocket | WebSocket `dataset.updated` event | +| Client-side filtering for Stats Bar | Incompatible with server-side pagination — can only filter current page | Server-side `?filter=` query parameter | | `override_columns=true` | Risk of deleting unmapped columns on PUT | Always `override_columns=false` (FR-023) | diff --git a/specs/030-dataset-lifecycle-workspace/data-model.md b/specs/030-dataset-lifecycle-workspace/data-model.md index e5e81cc7..f7784dd8 100644 --- a/specs/030-dataset-lifecycle-workspace/data-model.md +++ b/specs/030-dataset-lifecycle-workspace/data-model.md @@ -3,7 +3,7 @@ ## Entity Overview ``` -GET /api/datasets GET /api/datasets/{id} +GET /api/datasets?filter=all&page=1&page_size=10 GET /api/datasets/{id} ───────────────────────────────────────────────────────────── DatasetsResponse DatasetDetailResponse ├── datasets: DatasetItem[] ├── ... (existing fields) @@ -11,10 +11,11 @@ DatasetsResponse DatasetDetailResponse │ ├── table_name │ ├── id │ ├── schema │ ├── metric_name │ ├── database │ ├── expression -│ ├── mapped_fields │ ├── verbose_name -│ └── last_task │ ├── description -├── stats: StatsCounts │ └── metric_type -│ ├── total └── metric_count +│ ├── metric_count │ ├── verbose_name +│ ├── mapped_fields │ ├── description +│ └── last_task │ └── metric_type +├── stats: StatsCounts └── metric_count +│ ├── total │ ├── unmapped_count │ ├── mapped_count │ └── linked_count @@ -130,6 +131,10 @@ type MetricRow = { }; type StatsBarFilter = 'all' | 'unmapped' | 'mapped' | 'linked' | null; + +// API query parameter for server-side filtering +// GET /api/datasets?filter=unmapped|mapped|linked|all&page=1&page_size=10 +// When filter is active, backend filters full dataset list before pagination ``` --- @@ -141,7 +146,7 @@ type StatsBarFilter = 'all' | 'unmapped' | 'mapped' | 'linked' | null; "type": "dataset.updated", "payload": { "env_id": "ss-dev", - "dataset_ids": ["242377ce-bdbd-42df-85da-ca1340a72fbb"] + "dataset_ids": [32, 45, 78] } } ``` diff --git a/specs/030-dataset-lifecycle-workspace/plan.md b/specs/030-dataset-lifecycle-workspace/plan.md index 158b227b..872c1680 100644 --- a/specs/030-dataset-lifecycle-workspace/plan.md +++ b/specs/030-dataset-lifecycle-workspace/plan.md @@ -5,7 +5,7 @@ ## Summary -Переработка страницы датасетов (`/datasets`) из табличного вида в полноценное split-view рабочее пространство с: (1) Stats Bar с 4 агрегирующими метриками и клиентской фильтрацией, (2) split-view с карточками датасетов слева и детальной панелью справа, (3) inline-редактированием описаний колонок и метрик через новые ss-tools API эндпоинты, (4) WebSocket-автообновлением при завершении фоновых задач, (5) отображением метрик из Superset API. Изменения затрагивают бэкенд (3 новых DTO, 2 новых эндпоинта, модификация 2 существующих) и фронтенд (5 новых компонентов, переписанный +page.svelte). +Переработка страницы датасетов (`/datasets`) из табличного вида в полноценное split-view рабочее пространство с: (1) Stats Bar с 4 агрегирующими метриками и серверной фильтрацией через `?filter=`, (2) split-view с карточками датасетов слева и детальной панелью справа, (3) inline-редактированием описаний колонок и метрик через новые ss-tools API эндпоинты, (4) WebSocket-автообновлением при завершении фоновых задач, (5) отображением метрик из Superset API. Изменения затрагивают бэкенд (3 новых DTO, 2 новых эндпоинта, модификация 2 существующих) и фронтенд (5 новых компонентов, переписанный +page.svelte). ## Technical Context @@ -15,7 +15,7 @@ **Testing**: pytest (backend), vitest + @testing-library/svelte 5.x (frontend) **Target Platform**: Linux server (Docker), modern browsers **Project Type**: web application (FastAPI REST + WebSocket backend, SvelteKit SPA frontend) -**Performance Goals**: Split-view dataset switch <500ms, inline-edit save <1s, initial Stats Bar load <200ms +**Performance Goals**: Split-view loading state <100ms; dataset detail content according to API latency; inline-edit save <1s; initial Stats Bar load <200ms (stats computed in-memory from already-loaded full dataset list) **Constraints**: <400 lines per module (INV_7), RBAC on all new endpoints, Tailwind-only styling, Svelte 5 runes only **Scale/Scope**: 33 datasets (current), single environment (dev), 5 new frontend components, 2 new API endpoints @@ -29,7 +29,7 @@ | Semantic Protocol (ADR-0002) | ✅ PASS | All C4/C5 contracts use belief_scope/reason/reflect markers. All components use @UX_STATE. @RATIONALE/@REJECTED on key decisions. | | Orchestrator Pattern (ADR-0003) | ✅ PASS | Backend proxies Superset API; no direct frontend-to-Superset calls | | Plugin Architecture (ADR-0004) | ✅ PASS | No new plugin; mapping/docs reuse existing MapperPlugin/DocumentationPlugin | -| Auth RBAC (ADR-0005) | ✅ PASS | New endpoints gated: `has_permission("plugin:migration","READ")` | +| Auth RBAC (ADR-0005) | ✅ PASS | New endpoints gated: `has_permission("plugin:migration","READ")` for reads, `has_permission("plugin:migration","WRITE")` for mutations | | Frontend Svelte 5 Runes (ADR-0006) | ✅ PASS | All components use $state/$derived/$props/$effect | | Anti-Erosion (INV_7) | ✅ PASS | New components <400 lines each; no monolithic page | diff --git a/specs/030-dataset-lifecycle-workspace/quickstart.md b/specs/030-dataset-lifecycle-workspace/quickstart.md index c68fc362..708df002 100644 --- a/specs/030-dataset-lifecycle-workspace/quickstart.md +++ b/specs/030-dataset-lifecycle-workspace/quickstart.md @@ -4,8 +4,12 @@ - Backend virtualenv: `cd backend && source .venv/bin/activate` - Frontend deps: `cd frontend && npm install` -- Superset environment configured (dev via http://localhost:8100) -- RBAC: user must have `plugin:migration:READ` permission +- Superset environment configured (dev instance, URL configured in `backend/.env`) +- RBAC: user must have `plugin:migration:WRITE` permission for mutation endpoints, `plugin:migration:READ` for list/detail reads +- Backend dev server: `http://localhost:8000` +- Frontend dev server (proxies API to backend): `http://localhost:8100` +- Direct backend API access for smoke tests: `http://localhost:8000/api/datasets/...` +- Frontend proxy access (browser): `http://localhost:8100/datasets` ## Backend Verification @@ -41,19 +45,22 @@ cd frontend && npm run dev ### Backend API Tests ```bash -# Test metrics in dataset detail response -curl -s http://localhost:8100/api/datasets/32?env_id=dev | jq '.metrics, .metric_count' +# Test metrics in dataset detail response (direct backend) +curl -s http://localhost:8000/api/datasets/32?env_id=dev | jq '.metrics, .metric_count' # Test stats counts in dataset list -curl -s 'http://localhost:8100/api/datasets?env_id=dev&page=1&page_size=5' | jq '.stats' +curl -s 'http://localhost:8000/api/datasets?env_id=dev&page=1&page_size=5' | jq '.stats' + +# Test server-side filtering +curl -s 'http://localhost:8000/api/datasets?env_id=dev&filter=unmapped&page=1&page_size=5' | jq '.datasets | length' # Test column description inline-edit -curl -s -X PUT http://localhost:8100/api/datasets/32/columns/851/description \ +curl -s -X PUT http://localhost:8000/api/datasets/32/columns/851/description \ -H 'Content-Type: application/json' \ -d '{"description":"Updated via API"}' # Test metric description inline-edit -curl -s -X PUT http://localhost:8100/api/datasets/32/metrics/70/description \ +curl -s -X PUT http://localhost:8000/api/datasets/32/metrics/70/description \ -H 'Content-Type: application/json' \ -d '{"description":"Count of all records"}' ``` diff --git a/specs/030-dataset-lifecycle-workspace/research.md b/specs/030-dataset-lifecycle-workspace/research.md index 741c7947..80a8b3a6 100644 --- a/specs/030-dataset-lifecycle-workspace/research.md +++ b/specs/030-dataset-lifecycle-workspace/research.md @@ -29,15 +29,15 @@ Each performs: GET full dataset from Superset via `SupersetClient.get_dataset()` - Single `PUT /api/datasets/{id}` with `{"columns":[{"id":...,"description":...}]}` → valid but more complex for frontend, reserved for future batch updates - Optimistic save with rollback → rejected: adds complexity without significant UX benefit for description editing -**Impact:** +2 route handlers in `datasets.py` (~50 lines each). Reuses existing `SupersetClient.get_dataset()` and `SupersetClient.update_dataset()`. RBAC: checked via existing `has_permission("plugin:migration","READ")` or new permission. +**Impact:** +2 route handlers in `datasets.py` (~50 lines each). Reuses existing `SupersetClient.get_dataset()` and `SupersetClient.update_dataset()`. RBAC: `has_permission("plugin:migration","WRITE")` for mutations. --- -## R3 — Backend: Stats Counts in GET /api/datasets +## R3 — Backend: Stats Counts + Server-Side Filtering in GET /api/datasets -**Decision:** `GET /api/datasets` response adds `stats` object with `total`, `unmapped_count`, `mapped_count`, `linked_count`. Computed by the existing `resource_service.get_datasets_with_status()` which already fetches ALL datasets before pagination. Stats are computed from the full (non-paginated) dataset list. +**Decision:** `GET /api/datasets` accepts optional query parameter `?filter=unmapped|mapped|linked|all`. When `filter` is present, the backend filters the full dataset list before pagination. Response always includes `stats` object with `total`, `unmapped_count`, `mapped_count`, `linked_count` — computed from the full (non-paginated, non-filtered) dataset list. -**Rationale:** The Statistics Bar needs aggregate counts. Since the backend already fetches all datasets to compute `mapped_fields` enrichment, computing stats is O(n) over the already-loaded data — near-zero cost. +**Rationale:** Server-side filtering ensures that all matching datasets are returned, not just those on the current page. Stats represent global state independently of the active filter. Previously considered client-side filtering was incompatible with server-side pagination (a client can only filter the current page, causing mismatch between Stats Bar counts and visible cards). **Alternatives Considered:** - Separate `/api/datasets/stats` endpoint → rejected, adds unnecessary request @@ -67,9 +67,9 @@ Each performs: GET full dataset from Superset via `SupersetClient.get_dataset()` ``` +page.svelte (layout orchestrator, ~200 lines) -├── StatsBar.svelte (4 metric tiles, client-side filtering) +├── StatsBar.svelte (4 metric tiles, emits filter → parent calls API) ├── DatasetList.svelte (card list with pagination, checkbox selection) -├── DatasetPreview.svelte (right panel: header + columns + metrics) +├── DatasetPreview.svelte (presentational right panel: receives dataset/loading/error props; parent fetches detail API) │ ├── ColumnsTable.svelte (column rows with inline-edit ✏️) │ └── MetricsTable.svelte (metric rows with inline-edit ✏️) └── BulkActionPanel.svelte (reused from current implementation) @@ -117,7 +117,7 @@ Each component follows ss-tools Svelte 5 conventions (runes `$state`, `$derived` ## R8 — i18n: New Keys -**Decision:** Add 11 new keys to `frontend/src/lib/i18n/locales/{en,ru}/datasets.json`: `all`, `without_mapping`, `with_description`, `linked_to_dashboards`, `metrics`, `metric_count`, `no_selection`, `save_description`, `add_description`, `columns`, `detail_empty`. No structural changes to i18n system. +**Decision:** Add 11 new keys to `frontend/src/lib/i18n/locales/{en,ru}/datasets.json`: `all`, `without_mapping`, `with_mapping`, `linked_to_dashboards`, `metrics`, `metric_count`, `no_selection`, `save_description`, `add_description`, `columns`, `detail_empty`. No structural changes to i18n system. **Rationale:** Existing i18n uses flat JSON key files loaded by locale. Adding keys is additive. No new dependencies. @@ -163,6 +163,7 @@ Each component follows ss-tools Svelte 5 conventions (runes `$state`, `$derived` ## Unresolved / Deferred -- **DatasetItem.metric_count** in list view: requires resource_service enrichment. Deferred to implementation — can be computed from full dataset list if metrics data available, else defaults to 0 until backend support is added. -- **Accessibility (ARIA)**: Deferred to Polish phase. Components should use semantic HTML and `aria-busy`/`aria-live` where applicable. -- **Large-scale (200+ datasets)**: Client-side filtering works for 33 datasets. At 200+, Stats Bar counts remain server-computed; only list filtering would need server-side `?filter=` parameter. This is a future enhancement. +- **~~DatasetItem.metric_count in list view~~**: RESOLVED — `metric_count` added to `DatasetItem` DTO as required field. Backend must include metric count in list response (fetched from Superset dataset metadata during enrichment). +- **Accessibility (ARIA)**: Deferred to Polish phase. Components should use semantic HTML and `aria-busy`/`aria-live` where applicable. Minimum a11y requirements added to spec: Stats tiles as buttons with `aria-pressed`, slide-over traps focus and closes on Escape, inline edit errors announced via `aria-live`, skeleton loaders use `aria-busy`. +- **Large-scale (200+ datasets)**: Server-side filtering (`?filter=`) adopted for current scope. At 200+, pagination works normally — no client-side filtering needed. +- **Performance**: Detail panel target <500ms applies to render after API response. Loading state must appear within 100ms. Optional cache for previously opened dataset details deferred to implementation. diff --git a/specs/030-dataset-lifecycle-workspace/spec.md b/specs/030-dataset-lifecycle-workspace/spec.md index cb71faa9..7888a7c1 100644 --- a/specs/030-dataset-lifecycle-workspace/spec.md +++ b/specs/030-dataset-lifecycle-workspace/spec.md @@ -9,7 +9,7 @@ ### Session 2026-05-19 -- Q: Stats Bar фильтрует клиентски или серверно? → A: Клиентская фильтрация по данным из API. Бэкенд в ответе `GET /api/datasets` возвращает 4 stats-числа (`total`, `unmapped_count`, `mapped_count`, `linked_count`). Клиенты фильтруют карточки без дополнительного запроса. Пагинация остаётся серверной. При фильтре пагинация переключает страницы отфильтрованного подмножества. +- Q: Stats Bar фильтрует клиентски или серверно? → A: Серверная фильтрация через query параметр. Бэкенд в ответе `GET /api/datasets` возвращает 4 stats-числа (`total`, `unmapped_count`, `mapped_count`, `linked_count`). При клике на метрику Stats Bar фронтенд отправляет запрос с `?filter=unmapped|mapped|linked|all` и `filter` сбрасывает `page` на 1. Пагинация остаётся серверной; бэкенд фильтрует полный список перед пагинацией. Stats-числа считаются глобально (не зависят от фильтра). - Q: Каким запросом сохранять inline-edit описания колонок/метрик? → A: Два новых ss-tools эндпоинта: `PUT /api/datasets/{id}/columns/{col_id}/description` и `PUT /api/datasets/{id}/metrics/{metric_id}/description` (body: `{description: "text"}`). Бэкенд выполняет GET датасета из Superset, модифицирует одну колонку/метрику, PUT обратно полный объект. В Superset нет PATCH — только PUT с полным датасетом. - Q: Как обрабатывать конфликт одновременного редактирования? → A: Last-write-wins. Бэкенд всегда выполняет GET→modify→PUT без проверки версии. Последний сохранивший перезаписывает. Для описаний колонок/метрик это приемлемый риск. ETag/блокировки — out of scope. - Q: Параметр `override_columns` при PUT в Superset? → A: Всегда `override_columns=false`. Бэкенд получает полный датасет из GET, модифицирует только description нужной колонки, PUT обратно с `override_columns=false`. Superset сохраняет все колонки, не участвовавшие в payload. @@ -27,7 +27,7 @@ **Acceptance Scenarios**: -1. **Given** пользователь видит страницу датасетов с загруженными данными, **When** страница загружена, **Then** Stats Bar показывает 4 метрики: Все (total), Без маппинга, С описанием, С дашбордами +1. **Given** пользователь видит страницу датасетов с загруженными данными, **When** страница загружена, **Then** Stats Bar показывает 4 метрики: Все (total), Без маппинга, С маппингом, С дашбордами 2. **Given** Stats Bar отображается, **When** пользователь кликает на "Без маппинга", **Then** список фильтруется до датасетов с `mapped_fields.mapped === 0`, метрика подсвечена 3. **Given** Stats Bar отображается, **When** пользователь кликает на уже активный фильтр, **Then** фильтр сбрасывается и показываются все датасеты 4. **Given** API возвращает ошибку при загрузке, **When** страница рендерится, **Then** Stats Bar показывает 0 для всех метрик или состояние ошибки @@ -98,7 +98,7 @@ **Acceptance Scenarios**: 1. **Given** строка колонки без описания, **When** пользователь кликает "✏️ Добавить описание", **Then** появляется textarea с кнопками "Сохранить" и "Отмена" -2. **Given** textarea открыта, **When** пользователь вводит текст и нажимает "Сохранить", **Then** описание сохраняется через PUT `/api/datasets/{id}`, textarea закрывается, описание отображается +2. **Given** textarea открыта, **When** пользователь вводит текст и нажимает "Сохранить", **Then** описание колонки сохраняется через `PUT /api/datasets/{id}/columns/{col_id}/description`, описание метрики — через `PUT /api/datasets/{id}/metrics/{metric_id}/description`; textarea закрывается, описание отображается 3. **Given** textarea открыта, **When** пользователь нажимает "Отмена", **Then** textarea закрывается без изменений 4. **Given** сохранение не удалось (ошибка API), **When** пользователь нажимает "Сохранить", **Then** показывается сообщение об ошибке, textarea остаётся открытой для повторной попытки 5. **Given** строка колонки с существующим описанием, **When** пользователь кликает ✏️, **Then** textarea открывается с предзаполненным текстом описания @@ -142,7 +142,7 @@ - Что происходит когда API датасетов возвращает 503 (Superset недоступен)? - Как обрабатывается случай когда `detail` API для выбранного датасета падает, но список загружен? -- Как ведёт себя split-view при ресайзе окна между десктопным и мобильным layout? +- Как ведёт себя split-view при ресайзе окна между десктопным и мобильным layout? → **Решено**: Выбранный датасет сохраняется. При сжатии до <768px открытая детальная панель становится slide-over (открытым). При расширении до ≥768px восстанавливается split-view; если slide-over был закрыт — правая панель показывает последний выбранный датасет или placeholder. (FR-034–FR-036) - Что показывать в Stats Bar когда данные ещё не загружены (loading state)? - Как обрабатывать случай когда у датасета 0 колонок и 0 метрик? - Что происходит при сохранении inline-edit когда несколько пользователей редактируют один датасет (конфликт)? → **Решено**: last-write-wins. Последний сохранивший перезаписывает. ETag/блокировки — out of scope. @@ -152,8 +152,8 @@ ### Functional Requirements -- **FR-001**: Система MUST отображать Stats Bar с 4 метриками (Все, Без маппинга, С описанием, С дашбордами) вверху страницы `/datasets` -- **FR-002**: Каждая метрика в Stats Bar MUST быть кликабельной и действовать как фильтр списка датасетов +- **FR-001**: Система MUST отображать Stats Bar с 4 метриками (Все, Без маппинга, С маппингом, С дашбордами) вверху страницы `/datasets` +- **FR-002**: Каждая метрика в Stats Bar MUST быть кликабельной и действовать как фильтр списка датасетов. Фильтрация MUST выполняться серверно через query параметр `?filter=unmapped|mapped|linked|all` с серверной пагинацией. Stats-числа считаются глобально. - **FR-003**: Система MUST отображать split-view layout: левая панель (40%) со списком карточек, правая панель (60%) с деталями выбранного датасета - **FR-004**: При ширине экрана < 768px правая панель MUST открываться как slide-over поверх списка - **FR-005**: Карточка датасета MUST содержать: имя таблицы, схему, количество колонок, количество метрик, прогресс-бар маппинга, кнопки быстрых действий @@ -171,11 +171,24 @@ - **FR-017**: `DatasetDetailResponse` на бэкенде MUST включать `metrics: list[MetricItem]` и `metric_count: int` - **FR-018**: `MetricItem` Pydantic DTO MUST содержать поля: `id`, `metric_name`, `expression`, `verbose_name`, `description`, `metric_type` - **FR-019**: `SupersetClient.get_dataset_detail` MUST извлекать `dataset.get("metrics", [])` из ответа Superset API -- **FR-020**: Система MUST поддерживать i18n ключи (EN/RU) для: all, without_mapping, with_description, linked_to_dashboards, metrics, metric_count, no_selection, save_description, add_description +- **FR-020**: Система MUST поддерживать i18n ключи (EN/RU) для: all, without_mapping, with_mapping, linked_to_dashboards, metrics, metric_count, no_selection, save_description, add_description - **FR-021**: Breadcrumb для `/datasets` MUST отображать "Датасеты / Datasets" (EN/RU), а не "Маппер колонок" - **FR-022**: Ответ `GET /api/datasets` MUST включать поля stats: `total` (int), `unmapped_count` (int), `mapped_count` (int), `linked_count` (int) — для отрисовки Stats Bar без дополнительных запросов - **FR-023**: PUT в Superset при сохранении inline-edit MUST выполняться с `override_columns=false` (query parameter) — предотвращает случайное удаление колонок, не упомянутых в payload - **FR-024**: По WebSocket-событию `dataset.updated` (отправляемому при завершении mapping/docs задачи) фронтенд MUST перезапрашивать `GET /api/datasets` и обновлять Stats Bar counts + карточки без ручного «Обновить» +- **FR-025**: `DatasetItem` в list response MUST включать `metric_count: int` (количество метрик датасета) для отображения на карточках +- **FR-026**: `GET /api/datasets` MUST поддерживать query параметр `?filter=unmapped|mapped|linked|all` для серверной фильтрации перед пагинацией. При отсутствии параметра поведение эквивалентно `filter=all`. +- **FR-027**: Stats Bar метрики MUST быть кнопками с `aria-pressed`, отражающим активное состояние фильтра +- **FR-028**: Slide-over на мобильных MUST захватывать фокус при открытии и закрываться по Escape +- **FR-029**: Ошибки сохранения inline-edit MUST анонсироваться через `aria-live` регион +- **FR-030**: Skeleton-лоадеры MUST использовать `aria-busy="true"` на время загрузки +- **FR-031**: Клик по чекбоксу карточки MUST только переключать выделение датасета, НЕ открывая детальную панель +- **FR-032**: Клик по quick-action кнопке на карточке MUST открывать соответствующую модалку, НЕ открывая детальную панель +- **FR-033**: Клик по телу карточки (вне чекбокса и quick-action кнопок) MUST выбирать датасет и открывать детальную панель +- **FR-034**: Выбранный датасет MUST сохраняться при ресайзе между десктопным и мобильным layout без потери состояния +- **FR-035**: При ресайзе с десктопа на мобильный: если детальная панель открыта, она MUST перейти в режим slide-over (открытый) +- **FR-036**: При ресайзе с мобильного на десктоп: если slide-over закрыт, правая панель MUST показать placeholder или последний выбранный датасет +- **FR-037**: Сохранение описания MUST принимать plain text, max 2000 символов. Пустая строка разрешена (очищает описание). Пробельные символы в начале/конце сохраняются без обрезки. 404 при отсутствии датасета/колонки/метрики. 502/503 при недоступности Superset. ### Key Entities @@ -192,7 +205,7 @@ ### Measurable Outcomes - **SC-001**: Пользователь может за 1 клик перейти от Stats Bar к отфильтрованному списку датасетов -- **SC-002**: Переключение между датасетами в split-view происходит < 500ms (без полной перезагрузки страницы) +- **SC-002**: Loading state для переключения между датасетами в split-view отображается в течение 100ms; контент деталей загружается согласно latency API (после получения ответа детали рендерятся без дополнительной задержки) - **SC-003**: Пользователь может отредактировать и сохранить описание колонки/метрики < 3 кликов - **SC-004**: Stats Bar корректно обновляет значения при переключении окружения - **SC-005**: Все существующие bulk-действия (маппинг, генерация доков) продолжают работать в новом UI diff --git a/specs/030-dataset-lifecycle-workspace/tasks.md b/specs/030-dataset-lifecycle-workspace/tasks.md index ce078afb..bce6948c 100644 --- a/specs/030-dataset-lifecycle-workspace/tasks.md +++ b/specs/030-dataset-lifecycle-workspace/tasks.md @@ -10,8 +10,8 @@ > No user stories yet — infrastructure that all stories depend on. -- [ ] T001 [P] Add 11 EN i18n keys to `frontend/src/lib/i18n/locales/en/datasets.json`: `all`, `without_mapping`, `with_description`, `linked_to_dashboards`, `metrics`, `metric_count`, `no_selection`, `save_description`, `add_description`, `columns`, `detail_empty` -- [ ] T002 [P] Add 11 RU i18n keys to `frontend/src/lib/i18n/locales/ru/datasets.json`: `все`, `без_маппинга`, `с_описанием`, `связаны_с_дашбордами`, `метрики`, `метрик`, `выберите_датасет`, `сохранить_описание`, `добавить_описание`, `колонки`, `пустой_просмотр` +- [ ] T001 [P] Add 11 EN i18n keys to `frontend/src/lib/i18n/locales/en/datasets.json`: `all`, `without_mapping`, `with_mapping`, `linked_to_dashboards`, `metrics`, `metric_count`, `no_selection`, `save_description`, `add_description`, `columns`, `detail_empty` +- [ ] T002 [P] Add 11 RU i18n keys to `frontend/src/lib/i18n/locales/ru/datasets.json`: `все`, `без_маппинга`, `с_маппингом`, `связаны_с_дашбордами`, `метрики`, `метрик`, `выберите_датасет`, `сохранить_описание`, `добавить_описание`, `колонки`, `пустой_просмотр` - [ ] T003 Verify breadcrumb for `/datasets` shows "Датасеты / Datasets" in `frontend/src/lib/components/layout/Breadcrumbs.svelte` (already fixed, confirm) - [ ] T004 [P] Verify sidebar navigation entries for datasets in `frontend/src/lib/components/layout/sidebarNavigation.js` are correct @@ -26,9 +26,10 @@ - [ ] T005 Add `MetricItem` Pydantic DTO (fields: `id`, `metric_name`, `expression`, `verbose_name`, `description`, `metric_type`) in `backend/src/api/routes/datasets.py` (RATIONALE: metrics exist in Superset API but are currently ignored; REJECTED: separate metrics endpoint adds unnecessary API surface) - [ ] T006 Add `metrics: list[MetricItem]` and `metric_count: int` to `DatasetDetailResponse` in `backend/src/api/routes/datasets.py` - [ ] T007 Extract `dataset.get("metrics", [])` from Superset API response in `SupersetClient.get_dataset_detail()` at `backend/src/core/superset_client/_datasets.py` (add ~15 lines after existing columns loop, map each metric dict to MetricItem-like dict) +- [ ] T007a Add `metric_count: int` to `DatasetItem` DTO in `backend/src/api/routes/datasets.py`. Enrich list items with metric count during `resource_service.get_datasets_with_status()` (extract `len(dataset.get("metrics", []))` from Superset dataset metadata). (RATIONALE: FR-025 — cards must display metric count; REJECTED: lazy-loading metric count per card — too many requests, UX jank) - [ ] T008 Add `StatsCounts` Pydantic DTO (fields: `total`, `unmapped_count`, `mapped_count`, `linked_count`) in `backend/src/api/routes/datasets.py` - [ ] T009 Add `stats: StatsCounts` to `DatasetsResponse` in `backend/src/api/routes/datasets.py` -- [ ] T010 Compute `stats` counts from full (non-paginated) dataset list in `get_datasets()` handler in `backend/src/api/routes/datasets.py` — counts: `total=len(datasets)`, `unmapped=sum(mapped==0)`, `mapped=sum(mapped>0)`, `linked=sum(linked>0)` (RATIONALE: one API call for both list + stats per FR-022; REJECTED: separate /stats endpoint adds unnecessary request) +- [ ] T010 Compute `stats` counts from full (non-paginated, non-filtered) dataset list in `get_datasets()` handler in `backend/src/api/routes/datasets.py` — counts: `total=len(datasets)`, `unmapped=sum(mapped==0)`, `mapped=sum(mapped>0)`, `linked=sum(linked>0)`. Add server-side filtering: accept `?filter=unmapped|mapped|linked|all`, filter full list before pagination slice. (RATIONALE: one API call for both list + stats per FR-022; server-side filtering ensures all matching datasets returned; REJECTED: separate /stats endpoint adds unnecessary request, client-side filtering incompatible with pagination) - [ ] T011 Add inline-edit endpoints to `backend/src/api/routes/datasets.py`: `PUT /api/datasets/{dataset_id}/columns/{column_id}/description` (body: `ColumnDescriptionUpdate`, handler: GET full dataset → modify column → PUT to Superset with `?override_columns=false`) (RATIONALE: Superset lacks PATCH for individual columns; REJECTED: frontend direct PUT — bypasses orchestrator, exposes auth) - [ ] T012 Add inline-edit endpoint to `backend/src/api/routes/datasets.py`: `PUT /api/datasets/{dataset_id}/metrics/{metric_id}/description` (body: `MetricDescriptionUpdate`, handler: mirror of T011 for metrics) (RATIONALE: same GET→modify→PUT pattern; REJECTED: merging columns+metrics into one endpoint — different Superset payload structures) - [ ] T013 Add `dataset.updated` WebSocket event emission in task completion handler at `backend/src/core/task_manager.py` — when a mapping or documentation task completes, broadcast `{type: "dataset.updated", payload: {env_id, dataset_ids}}` via existing WebSocket manager (RATIONALE: real-time refresh without polling per FR-024; REJECTED: polling interval — unnecessary network overhead) @@ -42,7 +43,7 @@ > Core UI structure. All remaining stories depend on this layout being in place. -- [ ] T015 [US1] Create `StatsBar.svelte` component in `frontend/src/routes/datasets/StatsBar.svelte` — 4 metric tiles (Все, Без маппинга, С описанием, С дашбордами), each showing count from `stats` prop; active tile has highlight border; dispatches `onfilter` event on click; toggles filter on second click; skeleton tiles when loading (RATIONALE: quick diagnostic overview replaces empty space at top of current table page; REJECTED: dropdown filters — Stats Bar gives 1-click access) +- [ ] T015 [US1] Create `StatsBar.svelte` component in `frontend/src/routes/datasets/StatsBar.svelte` — 4 metric tiles (Все, Без маппинга, С маппингом, С дашбордами), each showing count from `stats` prop; active tile has highlight border; dispatches `onfilter` event on click (parent calls API with `?filter=`); toggles filter on second click (dispatches `null` to clear); skeleton tiles when loading. Tiles use `aria-pressed` reflecting active state. (RATIONALE: quick diagnostic overview replaces empty space at top of current table page; REJECTED: dropdown filters — Stats Bar gives 1-click access) - [ ] T016 [US2] Rewrite `frontend/src/routes/datasets/+page.svelte` as split-view orchestrator: replace monolithic 974-line table with layout shell using Tailwind grid `grid grid-cols-[40%_60%]` on desktop, `block` on mobile (<768px). Remove old table rendering, keep data loading logic, add `selectedDatasetId` state. Shell renders ``, ``, `` components. (RATIONALE: decompose 974-line page into composable components per INV_7 fractal limit; REJECTED: keep monolithic page — violates semantic erosion rules at C5 complexity) - [ ] T017 [US2] Add mobile responsive behavior in `+page.svelte`: at `max-width: 767px`, left panel takes full width; right panel opens as fixed slide-over overlay with backdrop and close/touch-outside-dismiss behavior (RATIONALE: FR-004 mobile requirement; REJECTED: separate mobile route — over-engineering) - [ ] T018 [US2] Wire `environmentContextStore` into new `+page.svelte`: on environment switch from the context store, reset `selectedDatasetId`, reload datasets, update Stats Bar. Preserve existing `initializeEnvironmentContext` + `selectedEnvironmentStore` logic from current implementation. @@ -51,7 +52,7 @@ **Verification**: - `cd frontend && npm run test` -- Browser: `http://localhost:8100/datasets` — Stats Bar visible, clicking tiles filters (no-op until cards exist), split-view layout renders +- Browser: `http://localhost:8100/datasets` — Stats Bar visible, clicking tiles triggers API reload with `?filter=`, split-view layout renders --- @@ -60,8 +61,8 @@ > Cards replace old table rows. Depends on Phase 3 (split-view shell + Stats Bar). - [ ] T021 [US3] Create `DatasetList.svelte` component in `frontend/src/routes/datasets/DatasetList.svelte` — card grid with pagination, search input, bulk selection checkboxes. Each card renders: `table_name` (linked), `schema`, `column_count`, `metric_count` (or hidden if 0), progress bar (mapped/total % with green/yellow/blue colors), quick-action buttons (🗺 Map Columns, 📄 Generate Docs). Clicking card body dispatches `onselect(datasetId)`. (RATIONALE: cards give richer visual info than table rows; REJECTED: keep table — doesn't show progress bars well) -- [ ] T022 [US3] Implement client-side filtering in `DatasetList.svelte`: receive `activeFilter` prop from parent; filter datasets array in `$derived` by `mapped_fields.mapped === 0` (unmapped), `> 0` (mapped), `linked_dashboard_count > 0` (linked), or no filter (all). Affected datasets propagate as `filteredDatasets` that pagination uses. -- [ ] T023 [US3] Preserve existing pagination (server-side) in `DatasetList.svelte`: when no client-side filter active, paginate via API; when client filter active, paginate filtered array locally (page_size items per page). Total count shown from `stats.total` or `filteredDatasets.length`. +- [ ] T022 [US3] Implement server-side filtering in `+page.svelte` orchestrator: when `activeFilter` changes to non-null value, call `loadDatasets(filter=activeFilter, page=1)`. Stats Bar tiles receive `activeFilter` prop and highlight accordingly. Backend filters full dataset list before pagination (FR-026). +- [ ] T023 [US3] Preserve existing pagination (server-side) in `DatasetList.svelte`: pagination controls call `loadDatasets(page=N)` via parent callback. Total count shown from `stats.total`. When filter is active, `stats.total` still shows global total; pagination shows total matching count from API response `total` field. - [ ] T024 [US3] Migrate existing search input from old `+page.svelte` into `DatasetList.svelte` — debounced search calls `loadDatasets()` from parent via prop callback. Existing `debounce` util reused. - [ ] T025 [US3] Wire quick-action buttons in `DatasetList.svelte`: "🗺 Map Columns" dispatches `onaction(dataset, 'map_columns')`; "📄 Generate Docs" dispatches `onaction(dataset, 'generate_docs')`. Parent `+page.svelte` opens existing modals (reused from current implementation). - [ ] T026 [P] [US3] Write vitest component test for `DatasetList` in `frontend/src/routes/datasets/__tests__/DatasetList.test.js`: test cards render with all fields, progress bar color logic, filter removes non-matching cards, search triggers callback @@ -74,7 +75,7 @@ > Right panel of split-view. Depends on Phase 3 (split-view shell) and Phase 2 (US7 backend metrics). -- [ ] T027 [US4] Create `DatasetPreview.svelte` component in `frontend/src/routes/datasets/DatasetPreview.svelte` — displays detail for `selectedDatasetId`. States: placeholder "Выберите датасет" (no selection), skeleton (loading), detail view (loaded), error banner (API failure). On `selectedDatasetId` change, fetches `GET /api/datasets/{id}?env_id=...` via `api.getDatasetDetail()`. +- [ ] T027 [US4] Create `DatasetPreview.svelte` component in `frontend/src/routes/datasets/DatasetPreview.svelte` — presentational component receiving `dataset`, `isLoading`, `error` props. States: placeholder "Выберите датасет" (dataset=null, not loading), skeleton (isLoading=true), detail view (dataset data), error banner (error string + onretry callback). Parent `+page.svelte` (DatasetHub) fetches `GET /api/datasets/{id}` on card selection and passes result as props. (RATIONALE: container-fetches pattern for cleaner testing and separation of concerns; REJECTED: self-fetching component — couples data fetching with rendering) - [ ] T028 [US4] Build dataset header section in `DatasetPreview.svelte`: `table_name` (bold), `schema` · `column_count` колонок · `metric_count` метрик, linked dashboards (clickable pills), created/changed dates. Linked dashboards navigate to `/dashboards/{slug}` on click. (FR-007) - [ ] T029 [US4] Create `ColumnsTable.svelte` component in `frontend/src/routes/datasets/ColumnsTable.svelte` — table with columns: name (verbose_name bold, else column_name italic), type chip (color-coded: DATE=green, STRING=purple, INT=blue, FLOAT=orange), description (or "✏️ Добавить описание" action prompt), edit button. Rows from `dataset.columns`. (FR-008, FR-010, FR-011) - [ ] T030 [US4] Create `MetricsTable.svelte` component in `frontend/src/routes/datasets/MetricsTable.svelte` — table with columns: metric_name (verbose_name bold, else metric_name italic), expression shown as gray hint under name when no description, description (or "✏️ Добавить описание" prompt), edit button. Rows from `dataset.metrics`. (FR-009, FR-010, FR-011, FR-012) @@ -138,19 +139,20 @@ | 1. Setup | T001–T004 | — | T001∥T002, T003∥T004 | | 2. Foundation (US7) | T005–T014 | US7 | T005–T010 partial; T011–T013 after T005–T009 | | 3. Stats Bar + Split View (US1+US2) | T015–T020 | US1,US2 | T019∥T020 | +| 3. Stats Bar + Split View (US1+US2) | T015–T020 | US1,US2 | T019∥T020 | | 4. Dataset Cards (US3) | T021–T026 | US3 | T026∥ after T021–T025 | | 5. Detail Panel (US4+US7) | T027–T033 | US4 | T032∥T033 | | 6. Inline Edit (US5) | T034–T037 | US5 | T037∥ after T034–T036 | | 7. Bulk Actions (US6) | T038–T041 | US6 | T041∥ after T038–T040 | | 8. Polish & Verification | T042–T051 | — | T046–T050 parallel after T042–T045 | -**Total**: 51 tasks across 8 phases. +**Total**: 52 tasks across 8 phases. ### Story Independent Verification | Story | How to verify independently | |-------|---------------------------| -| US1 | Load `/datasets`, check 4 tiles visible, click → filter applies | +| US1 | Load `/datasets`, check 4 tiles visible, click → API called with `?filter=`, server returns filtered list | | US2 | Desktop: 40/60 split grid visible; mobile: slide-over on card click | | US3 | Card shows all fields + progress bar; filter/search/page work | | US4 | Click card → right panel loads columns table + metrics table | diff --git a/specs/030-dataset-lifecycle-workspace/ux_reference.md b/specs/030-dataset-lifecycle-workspace/ux_reference.md index 86b3290a..d6216d3e 100644 --- a/specs/030-dataset-lifecycle-workspace/ux_reference.md +++ b/specs/030-dataset-lifecycle-workspace/ux_reference.md @@ -12,7 +12,7 @@ ## 2. The "Happy Path" Narrative -Аналитик открывает страницу датасетов и сразу видит Stats Bar: 142 датасета всего, 38 без маппинга, 89 с описанием, 12 связаны с дашбордами. Он кликает "⚠️ Без маппинга" — список слева мгновенно фильтруется до 38 датасетов. Выбирает первый датасет в списке — справа без перезагрузки открывается детальная панель: хедер с именем таблицы и схемой, секция колонок с progress-индикаторами описаний, секция метрик. Видит колонку без описания, кликает "✏️ Добавить описание" — появляется textarea, вводит описание, сохраняет. Описание мгновенно отображается. Затем выделяет 3 датасета через чекбоксы, внизу появляется панель — запускает "Маппинг колонок" для всех трёх одним кликом. +Аналитик открывает страницу датасетов и сразу видит Stats Bar: 142 датасета всего, 38 без маппинга, 89 с маппингом, 12 связаны с дашбордами. Он кликает «⚠️ Без маппинга» — Stats Bar подсвечивает метрику, фронтенд отправляет запрос `?filter=unmapped&page=1`, сервер возвращает отфильтрованные 38 датасетов с серверной пагинацией. Выбирает первый датасет в списке — DatasetHub фетчит `GET /api/datasets/{id}`, справа без перезагрузки открывается детальная панель: хедер с именем таблицы и схемой, секция колонок с progress-индикаторами описаний, секция метрик. Видит колонку без описания, кликает «✏️ Добавить описание» — появляется textarea, вводит описание, сохраняет через `PUT /api/datasets/{id}/columns/{col_id}/description`. Описание мгновенно отображается. Затем выделяет 3 датасета через чекбоксы, внизу появляется панель — запускает «Маппинг колонок» для всех трёх одним кликом. ## 3. Interface Mockups @@ -22,7 +22,7 @@ - **Layout**: Двухколоночный split-view (40/60) на десктопе. При < 768px — одноколоночный с slide-over для деталей. - **Key Elements**: - - **Stats Bar**: 4 метрики в строке вверху, каждая — кнопка-фильтр с иконкой, числом и подписью. Активная метрика подсвечена (primary background). + - **Stats Bar**: 4 метрики в строке вверху, каждая — кнопка-фильтр (`