diff --git a/backend/src/api/routes/dashboards.py b/backend/src/api/routes/dashboards.py index 49836488..79705c36 100644 --- a/backend/src/api/routes/dashboards.py +++ b/backend/src/api/routes/dashboards.py @@ -12,9 +12,9 @@ # [SECTION: IMPORTS] from fastapi import APIRouter, Depends, HTTPException -from typing import List, Optional +from typing import List, Optional, Dict from pydantic import BaseModel, Field -from ...dependencies import get_config_manager, get_task_manager, get_resource_service, has_permission +from ...dependencies import get_config_manager, get_task_manager, get_resource_service, get_mapping_service, has_permission from ...core.logger import logger, belief_scope # [/SECTION] @@ -47,26 +47,44 @@ class DashboardItem(BaseModel): class DashboardsResponse(BaseModel): dashboards: List[DashboardItem] total: int + page: int + page_size: int + total_pages: int # [/DEF:DashboardsResponse:DataClass] # [DEF:get_dashboards:Function] # @PURPOSE: Fetch list of dashboards from a specific environment with Git status and last task status # @PRE: env_id must be a valid environment ID -# @POST: Returns a list of dashboards with enhanced metadata +# @PRE: page must be >= 1 if provided +# @PRE: page_size must be between 1 and 100 if provided +# @POST: Returns a list of dashboards with enhanced metadata and pagination info +# @POST: Response includes pagination metadata (page, page_size, total, total_pages) # @PARAM: env_id (str) - The environment ID to fetch dashboards from # @PARAM: search (Optional[str]) - Filter by title/slug +# @PARAM: page (Optional[int]) - Page number (default: 1) +# @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100) # @RETURN: DashboardsResponse - List of dashboards with status metadata # @RELATION: CALLS -> ResourceService.get_dashboards_with_status @router.get("/api/dashboards", response_model=DashboardsResponse) async def get_dashboards( env_id: str, search: Optional[str] = None, + page: int = 1, + page_size: int = 10, config_manager=Depends(get_config_manager), task_manager=Depends(get_task_manager), resource_service=Depends(get_resource_service), _ = Depends(has_permission("plugin:migration", "READ")) ): - with belief_scope("get_dashboards", f"env_id={env_id}, search={search}"): + with belief_scope("get_dashboards", f"env_id={env_id}, search={search}, page={page}, page_size={page_size}"): + # Validate pagination parameters + if page < 1: + logger.error(f"[get_dashboards][Coherence:Failed] Invalid page: {page}") + raise HTTPException(status_code=400, detail="Page must be >= 1") + if page_size < 1 or page_size > 100: + logger.error(f"[get_dashboards][Coherence:Failed] Invalid page_size: {page_size}") + raise HTTPException(status_code=400, detail="Page size must be between 1 and 100") + # Validate environment exists environments = config_manager.get_environments() env = next((e for e in environments if e.id == env_id), None) @@ -90,11 +108,23 @@ async def get_dashboards( or search_lower in d.get('slug', '').lower() ] - logger.info(f"[get_dashboards][Coherence:OK] Returning {len(dashboards)} dashboards") + # Calculate pagination + total = len(dashboards) + total_pages = (total + page_size - 1) // page_size if total > 0 else 1 + start_idx = (page - 1) * page_size + end_idx = start_idx + page_size + + # Slice dashboards for current page + paginated_dashboards = dashboards[start_idx:end_idx] + + logger.info(f"[get_dashboards][Coherence:OK] Returning {len(paginated_dashboards)} dashboards (page {page}/{total_pages}, total: {total})") return DashboardsResponse( - dashboards=dashboards, - total=len(dashboards) + dashboards=paginated_dashboards, + total=total, + page=page, + page_size=page_size, + total_pages=total_pages ) except Exception as e: @@ -102,4 +132,192 @@ async def get_dashboards( raise HTTPException(status_code=503, detail=f"Failed to fetch dashboards: {str(e)}") # [/DEF:get_dashboards:Function] +# [DEF:MigrateRequest:DataClass] +class MigrateRequest(BaseModel): + source_env_id: str = Field(..., description="Source environment ID") + target_env_id: str = Field(..., description="Target environment ID") + dashboard_ids: List[int] = Field(..., description="List of dashboard IDs to migrate") + db_mappings: Optional[Dict[str, str]] = Field(None, description="Database mappings for migration") + replace_db_config: bool = Field(False, description="Replace database configuration") +# [/DEF:MigrateRequest:DataClass] + +# [DEF:TaskResponse:DataClass] +class TaskResponse(BaseModel): + task_id: str +# [/DEF:TaskResponse:DataClass] + +# [DEF:migrate_dashboards:Function] +# @PURPOSE: Trigger bulk migration of dashboards from source to target environment +# @PRE: User has permission plugin:migration:execute +# @PRE: source_env_id and target_env_id are valid environment IDs +# @PRE: dashboard_ids is a non-empty list +# @POST: Returns task_id for tracking migration progress +# @POST: Task is created and queued for execution +# @PARAM: request (MigrateRequest) - Migration request with source, target, and dashboard IDs +# @RETURN: TaskResponse - Task ID for tracking +# @RELATION: DISPATCHES -> MigrationPlugin +# @RELATION: CALLS -> task_manager.create_task +@router.post("/api/dashboards/migrate", response_model=TaskResponse) +async def migrate_dashboards( + request: MigrateRequest, + config_manager=Depends(get_config_manager), + task_manager=Depends(get_task_manager), + _ = Depends(has_permission("plugin:migration", "EXECUTE")) +): + with belief_scope("migrate_dashboards", f"source={request.source_env_id}, target={request.target_env_id}, count={len(request.dashboard_ids)}"): + # Validate request + if not request.dashboard_ids: + logger.error("[migrate_dashboards][Coherence:Failed] No dashboard IDs provided") + raise HTTPException(status_code=400, detail="At least one dashboard ID must be provided") + + # Validate environments exist + environments = config_manager.get_environments() + source_env = next((e for e in environments if e.id == request.source_env_id), None) + target_env = next((e for e in environments if e.id == request.target_env_id), None) + + if not source_env: + logger.error(f"[migrate_dashboards][Coherence:Failed] Source environment not found: {request.source_env_id}") + raise HTTPException(status_code=404, detail="Source environment not found") + if not target_env: + logger.error(f"[migrate_dashboards][Coherence:Failed] Target environment not found: {request.target_env_id}") + raise HTTPException(status_code=404, detail="Target environment not found") + + try: + # Create migration task + task_params = { + 'source_env_id': request.source_env_id, + 'target_env_id': request.target_env_id, + 'dashboards': request.dashboard_ids, + 'replace_db_config': request.replace_db_config, + 'db_mappings': request.db_mappings or {} + } + + task_id = await task_manager.create_task( + plugin_id='superset-migration', + params=task_params + ) + + logger.info(f"[migrate_dashboards][Coherence:OK] Migration task created: {task_id} for {len(request.dashboard_ids)} dashboards") + + return TaskResponse(task_id=str(task_id)) + + except Exception as e: + logger.error(f"[migrate_dashboards][Coherence:Failed] Failed to create migration task: {e}") + raise HTTPException(status_code=503, detail=f"Failed to create migration task: {str(e)}") +# [/DEF:migrate_dashboards:Function] + +# [DEF:BackupRequest:DataClass] +class BackupRequest(BaseModel): + env_id: str = Field(..., description="Environment ID") + dashboard_ids: List[int] = Field(..., description="List of dashboard IDs to backup") + schedule: Optional[str] = Field(None, description="Cron schedule for recurring backups (e.g., '0 0 * * *')") +# [/DEF:BackupRequest:DataClass] + +# [DEF:backup_dashboards:Function] +# @PURPOSE: Trigger bulk backup of dashboards with optional cron schedule +# @PRE: User has permission plugin:backup:execute +# @PRE: env_id is a valid environment ID +# @PRE: dashboard_ids is a non-empty list +# @POST: Returns task_id for tracking backup progress +# @POST: Task is created and queued for execution +# @POST: If schedule is provided, a scheduled task is created +# @PARAM: request (BackupRequest) - Backup request with environment and dashboard IDs +# @RETURN: TaskResponse - Task ID for tracking +# @RELATION: DISPATCHES -> BackupPlugin +# @RELATION: CALLS -> task_manager.create_task +@router.post("/api/dashboards/backup", response_model=TaskResponse) +async def backup_dashboards( + request: BackupRequest, + config_manager=Depends(get_config_manager), + task_manager=Depends(get_task_manager), + _ = Depends(has_permission("plugin:backup", "EXECUTE")) +): + with belief_scope("backup_dashboards", f"env={request.env_id}, count={len(request.dashboard_ids)}, schedule={request.schedule}"): + # Validate request + if not request.dashboard_ids: + logger.error("[backup_dashboards][Coherence:Failed] No dashboard IDs provided") + raise HTTPException(status_code=400, detail="At least one dashboard ID must be provided") + + # Validate environment exists + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == request.env_id), None) + + if not env: + logger.error(f"[backup_dashboards][Coherence:Failed] Environment not found: {request.env_id}") + raise HTTPException(status_code=404, detail="Environment not found") + + try: + # Create backup task + task_params = { + 'env': request.env_id, + 'dashboards': request.dashboard_ids, + 'schedule': request.schedule + } + + task_id = await task_manager.create_task( + plugin_id='superset-backup', + params=task_params + ) + + logger.info(f"[backup_dashboards][Coherence:OK] Backup task created: {task_id} for {len(request.dashboard_ids)} dashboards") + + return TaskResponse(task_id=str(task_id)) + + except Exception as e: + logger.error(f"[backup_dashboards][Coherence:Failed] Failed to create backup task: {e}") + raise HTTPException(status_code=503, detail=f"Failed to create backup task: {str(e)}") +# [/DEF:backup_dashboards:Function] + +# [DEF:DatabaseMapping:DataClass] +class DatabaseMapping(BaseModel): + source_db: str + target_db: str + confidence: float +# [/DEF:DatabaseMapping:DataClass] + +# [DEF:DatabaseMappingsResponse:DataClass] +class DatabaseMappingsResponse(BaseModel): + mappings: List[DatabaseMapping] +# [/DEF:DatabaseMappingsResponse:DataClass] + +# [DEF:get_database_mappings:Function] +# @PURPOSE: Get database mapping suggestions between source and target environments +# @PRE: User has permission plugin:migration:read +# @PRE: source_env_id and target_env_id are valid environment IDs +# @POST: Returns list of suggested database mappings with confidence scores +# @PARAM: source_env_id (str) - Source environment ID +# @PARAM: target_env_id (str) - Target environment ID +# @RETURN: DatabaseMappingsResponse - List of suggested mappings +# @RELATION: CALLS -> MappingService.get_suggestions +@router.get("/api/dashboards/db-mappings", response_model=DatabaseMappingsResponse) +async def get_database_mappings( + source_env_id: str, + target_env_id: str, + mapping_service=Depends(get_mapping_service), + _ = Depends(has_permission("plugin:migration", "READ")) +): + with belief_scope("get_database_mappings", f"source={source_env_id}, target={target_env_id}"): + try: + # Get mapping suggestions using MappingService + suggestions = await mapping_service.get_suggestions(source_env_id, target_env_id) + + # Format suggestions as DatabaseMapping objects + mappings = [ + DatabaseMapping( + source_db=s.get('source_db', ''), + target_db=s.get('target_db', ''), + confidence=s.get('confidence', 0.0) + ) + for s in suggestions + ] + + logger.info(f"[get_database_mappings][Coherence:OK] Returning {len(mappings)} database mapping suggestions") + + return DatabaseMappingsResponse(mappings=mappings) + + except Exception as e: + logger.error(f"[get_database_mappings][Coherence:Failed] Failed to get database mappings: {e}") + raise HTTPException(status_code=503, detail=f"Failed to get database mappings: {str(e)}") +# [/DEF:get_database_mappings:Function] + # [/DEF:backend.src.api.routes.dashboards:Module] diff --git a/backend/src/api/routes/datasets.py b/backend/src/api/routes/datasets.py index 025c5d6c..89a0de43 100644 --- a/backend/src/api/routes/datasets.py +++ b/backend/src/api/routes/datasets.py @@ -16,6 +16,7 @@ from typing import List, Optional from pydantic import BaseModel, Field from ...dependencies import get_config_manager, get_task_manager, get_resource_service, has_permission from ...core.logger import logger, belief_scope +from ...core.superset_client import SupersetClient # [/SECTION] router = APIRouter() @@ -42,22 +43,64 @@ class DatasetItem(BaseModel): last_task: Optional[LastTask] = None # [/DEF:DatasetItem:DataClass] +# [DEF:LinkedDashboard:DataClass] +class LinkedDashboard(BaseModel): + id: int + title: str + slug: Optional[str] = None +# [/DEF:LinkedDashboard:DataClass] + +# [DEF:DatasetColumn:DataClass] +class DatasetColumn(BaseModel): + id: int + name: str + type: Optional[str] = None + is_dttm: bool = False + is_active: bool = True + description: Optional[str] = None +# [/DEF:DatasetColumn:DataClass] + +# [DEF:DatasetDetailResponse:DataClass] +class DatasetDetailResponse(BaseModel): + id: int + table_name: str + schema: str + database: str + description: Optional[str] = None + columns: List[DatasetColumn] + column_count: int + sql: Optional[str] = None + linked_dashboards: List[LinkedDashboard] + linked_dashboard_count: int + is_sqllab_view: bool = False + created_on: Optional[str] = None + changed_on: Optional[str] = None +# [/DEF:DatasetDetailResponse:DataClass] + # [DEF:DatasetsResponse:DataClass] class DatasetsResponse(BaseModel): datasets: List[DatasetItem] total: int + page: int + page_size: int + total_pages: int # [/DEF:DatasetsResponse:DataClass] -# [DEF:get_datasets:Function] -# @PURPOSE: Fetch list of datasets from a specific environment with mapping progress -# @PRE: env_id must be a valid environment ID -# @POST: Returns a list of datasets with enhanced metadata -# @PARAM: env_id (str) - The environment ID to fetch datasets from -# @PARAM: search (Optional[str]) - Filter by table name -# @RETURN: DatasetsResponse - List of datasets with status metadata +# [DEF:TaskResponse:DataClass] +class TaskResponse(BaseModel): + task_id: str +# [/DEF:TaskResponse:DataClass] + +# [DEF:get_dataset_ids:Function] +# @PURPOSE: Fetch list of all dataset IDs from a specific environment (without pagination) +# @PRE: env_id must be a valid environment ID +# @POST: Returns a list of all dataset IDs +# @PARAM: env_id (str) - The environment ID to fetch datasets from +# @PARAM: search (Optional[str]) - Filter by table name +# @RETURN: List[int] - List of dataset IDs # @RELATION: CALLS -> ResourceService.get_datasets_with_status -@router.get("/api/datasets", response_model=DatasetsResponse) -async def get_datasets( +@router.get("/api/datasets/ids") +async def get_dataset_ids( env_id: str, search: Optional[str] = None, config_manager=Depends(get_config_manager), @@ -65,7 +108,73 @@ async def get_datasets( resource_service=Depends(get_resource_service), _ = Depends(has_permission("plugin:migration", "READ")) ): - with belief_scope("get_datasets", f"env_id={env_id}, search={search}"): + with belief_scope("get_dataset_ids", f"env_id={env_id}, search={search}"): + # Validate environment exists + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == env_id), None) + if not env: + logger.error(f"[get_dataset_ids][Coherence:Failed] Environment not found: {env_id}") + raise HTTPException(status_code=404, detail="Environment not found") + + try: + # Get all tasks for status lookup + all_tasks = task_manager.get_all_tasks() + + # Fetch datasets with status using ResourceService + datasets = await resource_service.get_datasets_with_status(env, all_tasks) + + # Apply search filter if provided + if search: + search_lower = search.lower() + datasets = [ + d for d in datasets + if search_lower in d.get('table_name', '').lower() + ] + + # Extract and return just the IDs + dataset_ids = [d['id'] for d in datasets] + logger.info(f"[get_dataset_ids][Coherence:OK] Returning {len(dataset_ids)} dataset IDs") + + return {"dataset_ids": dataset_ids} + + except Exception as e: + logger.error(f"[get_dataset_ids][Coherence:Failed] Failed to fetch dataset IDs: {e}") + raise HTTPException(status_code=503, detail=f"Failed to fetch dataset IDs: {str(e)}") +# [/DEF:get_dataset_ids:Function] + +# [DEF:get_datasets:Function] +# @PURPOSE: Fetch list of datasets from a specific environment with mapping progress +# @PRE: env_id must be a valid environment ID +# @PRE: page must be >= 1 if provided +# @PRE: page_size must be between 1 and 100 if provided +# @POST: Returns a list of datasets with enhanced metadata and pagination info +# @POST: Response includes pagination metadata (page, page_size, total, total_pages) +# @PARAM: env_id (str) - The environment ID to fetch datasets from +# @PARAM: search (Optional[str]) - Filter by table name +# @PARAM: page (Optional[int]) - Page number (default: 1) +# @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100) +# @RETURN: DatasetsResponse - List of datasets with status metadata +# @RELATION: CALLS -> ResourceService.get_datasets_with_status +@router.get("/api/datasets", response_model=DatasetsResponse) +async def get_datasets( + env_id: str, + search: Optional[str] = None, + page: int = 1, + page_size: int = 10, + config_manager=Depends(get_config_manager), + task_manager=Depends(get_task_manager), + resource_service=Depends(get_resource_service), + _ = Depends(has_permission("plugin:migration", "READ")) +): + with belief_scope("get_datasets", f"env_id={env_id}, search={search}, page={page}, page_size={page_size}"): + # Validate pagination parameters + if page < 1: + logger.error(f"[get_datasets][Coherence:Failed] Invalid page: {page}") + raise HTTPException(status_code=400, detail="Page must be >= 1") + if page_size < 1 or page_size > 100: + logger.error(f"[get_datasets][Coherence:Failed] Invalid page_size: {page_size}") + raise HTTPException(status_code=400, detail="Page size must be between 1 and 100") + # Validate environment exists environments = config_manager.get_environments() env = next((e for e in environments if e.id == env_id), None) @@ -88,11 +197,23 @@ async def get_datasets( if search_lower in d.get('table_name', '').lower() ] - logger.info(f"[get_datasets][Coherence:OK] Returning {len(datasets)} datasets") + # Calculate pagination + total = len(datasets) + total_pages = (total + page_size - 1) // page_size if total > 0 else 1 + start_idx = (page - 1) * page_size + end_idx = start_idx + page_size + + # Slice datasets for current page + paginated_datasets = datasets[start_idx:end_idx] + + logger.info(f"[get_datasets][Coherence:OK] Returning {len(paginated_datasets)} datasets (page {page}/{total_pages}, total: {total})") return DatasetsResponse( - datasets=datasets, - total=len(datasets) + datasets=paginated_datasets, + total=total, + page=page, + page_size=page_size, + total_pages=total_pages ) except Exception as e: @@ -100,4 +221,175 @@ async def get_datasets( raise HTTPException(status_code=503, detail=f"Failed to fetch datasets: {str(e)}") # [/DEF:get_datasets:Function] +# [DEF:MapColumnsRequest:DataClass] +class MapColumnsRequest(BaseModel): + env_id: str = Field(..., description="Environment ID") + dataset_ids: List[int] = Field(..., description="List of dataset IDs to map") + source_type: str = Field(..., description="Source type: 'postgresql' or 'xlsx'") + connection_id: Optional[str] = Field(None, description="Connection ID for PostgreSQL source") + file_data: Optional[str] = Field(None, description="File path or data for XLSX source") +# [/DEF:MapColumnsRequest:DataClass] + +# [DEF:map_columns:Function] +# @PURPOSE: Trigger bulk column mapping for datasets +# @PRE: User has permission plugin:mapper:execute +# @PRE: env_id is a valid environment ID +# @PRE: dataset_ids is a non-empty list +# @POST: Returns task_id for tracking mapping progress +# @POST: Task is created and queued for execution +# @PARAM: request (MapColumnsRequest) - Mapping request with environment and dataset IDs +# @RETURN: TaskResponse - Task ID for tracking +# @RELATION: DISPATCHES -> MapperPlugin +# @RELATION: CALLS -> task_manager.create_task +@router.post("/api/datasets/map-columns", response_model=TaskResponse) +async def map_columns( + request: MapColumnsRequest, + config_manager=Depends(get_config_manager), + task_manager=Depends(get_task_manager), + _ = Depends(has_permission("plugin:mapper", "EXECUTE")) +): + with belief_scope("map_columns", f"env={request.env_id}, count={len(request.dataset_ids)}, source={request.source_type}"): + # Validate request + if not request.dataset_ids: + logger.error("[map_columns][Coherence:Failed] No dataset IDs provided") + raise HTTPException(status_code=400, detail="At least one dataset ID must be provided") + + # Validate source type + if request.source_type not in ['postgresql', 'xlsx']: + logger.error(f"[map_columns][Coherence:Failed] Invalid source type: {request.source_type}") + raise HTTPException(status_code=400, detail="Source type must be 'postgresql' or 'xlsx'") + + # Validate environment exists + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == request.env_id), None) + + if not env: + logger.error(f"[map_columns][Coherence:Failed] Environment not found: {request.env_id}") + raise HTTPException(status_code=404, detail="Environment not found") + + try: + # Create mapping task + task_params = { + 'env_id': request.env_id, + 'datasets': request.dataset_ids, + 'source_type': request.source_type, + 'connection_id': request.connection_id, + 'file_data': request.file_data + } + + task_id = await task_manager.create_task( + plugin_id='dataset-mapper', + params=task_params + ) + + logger.info(f"[map_columns][Coherence:OK] Mapping task created: {task_id} for {len(request.dataset_ids)} datasets") + + return TaskResponse(task_id=str(task_id)) + + except Exception as e: + logger.error(f"[map_columns][Coherence:Failed] Failed to create mapping task: {e}") + raise HTTPException(status_code=503, detail=f"Failed to create mapping task: {str(e)}") +# [/DEF:map_columns:Function] + +# [DEF:GenerateDocsRequest:DataClass] +class GenerateDocsRequest(BaseModel): + env_id: str = Field(..., description="Environment ID") + dataset_ids: List[int] = Field(..., description="List of dataset IDs to generate docs for") + llm_provider: str = Field(..., description="LLM provider to use") + options: Optional[dict] = Field(None, description="Additional options for documentation generation") +# [/DEF:GenerateDocsRequest:DataClass] + +# [DEF:generate_docs:Function] +# @PURPOSE: Trigger bulk documentation generation for datasets +# @PRE: User has permission plugin:llm_analysis:execute +# @PRE: env_id is a valid environment ID +# @PRE: dataset_ids is a non-empty list +# @POST: Returns task_id for tracking documentation generation progress +# @POST: Task is created and queued for execution +# @PARAM: request (GenerateDocsRequest) - Documentation generation request +# @RETURN: TaskResponse - Task ID for tracking +# @RELATION: DISPATCHES -> LLMAnalysisPlugin +# @RELATION: CALLS -> task_manager.create_task +@router.post("/api/datasets/generate-docs", response_model=TaskResponse) +async def generate_docs( + request: GenerateDocsRequest, + config_manager=Depends(get_config_manager), + task_manager=Depends(get_task_manager), + _ = Depends(has_permission("plugin:llm_analysis", "EXECUTE")) +): + with belief_scope("generate_docs", f"env={request.env_id}, count={len(request.dataset_ids)}, provider={request.llm_provider}"): + # Validate request + if not request.dataset_ids: + logger.error("[generate_docs][Coherence:Failed] No dataset IDs provided") + raise HTTPException(status_code=400, detail="At least one dataset ID must be provided") + + # Validate environment exists + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == request.env_id), None) + + if not env: + logger.error(f"[generate_docs][Coherence:Failed] Environment not found: {request.env_id}") + raise HTTPException(status_code=404, detail="Environment not found") + + try: + # Create documentation generation task + task_params = { + 'env_id': request.env_id, + 'datasets': request.dataset_ids, + 'llm_provider': request.llm_provider, + 'options': request.options or {} + } + + task_id = await task_manager.create_task( + plugin_id='llm_documentation', + params=task_params + ) + + logger.info(f"[generate_docs][Coherence:OK] Documentation generation task created: {task_id} for {len(request.dataset_ids)} datasets") + + return TaskResponse(task_id=str(task_id)) + + except Exception as e: + logger.error(f"[generate_docs][Coherence:Failed] Failed to create documentation generation task: {e}") + raise HTTPException(status_code=503, detail=f"Failed to create documentation generation task: {str(e)}") +# [/DEF:generate_docs:Function] + +# [DEF:get_dataset_detail:Function] +# @PURPOSE: Get detailed dataset information including columns and linked dashboards +# @PRE: env_id is a valid environment ID +# @PRE: dataset_id is a valid dataset ID +# @POST: Returns detailed dataset info with columns and linked dashboards +# @PARAM: env_id (str) - The environment ID +# @PARAM: dataset_id (int) - The dataset ID +# @RETURN: DatasetDetailResponse - Detailed dataset information +# @RELATION: CALLS -> SupersetClient.get_dataset_detail +@router.get("/api/datasets/{dataset_id}", response_model=DatasetDetailResponse) +async def get_dataset_detail( + env_id: str, + dataset_id: int, + config_manager=Depends(get_config_manager), + _ = Depends(has_permission("plugin:migration", "READ")) +): + with belief_scope("get_dataset_detail", f"env_id={env_id}, dataset_id={dataset_id}"): + # Validate environment exists + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == env_id), None) + if not env: + logger.error(f"[get_dataset_detail][Coherence:Failed] Environment not found: {env_id}") + raise HTTPException(status_code=404, detail="Environment not found") + + try: + # Fetch detailed dataset info using SupersetClient + client = SupersetClient(env) + dataset_detail = client.get_dataset_detail(dataset_id) + + logger.info(f"[get_dataset_detail][Coherence:OK] Retrieved dataset {dataset_id} with {dataset_detail['column_count']} columns and {dataset_detail['linked_dashboard_count']} linked dashboards") + + return DatasetDetailResponse(**dataset_detail) + + except Exception as e: + logger.error(f"[get_dataset_detail][Coherence:Failed] Failed to fetch dataset detail: {e}") + raise HTTPException(status_code=503, detail=f"Failed to fetch dataset detail: {str(e)}") +# [/DEF:get_dataset_detail:Function] + # [/DEF:backend.src.api.routes.datasets:Module] diff --git a/backend/src/api/routes/settings.py b/backend/src/api/routes/settings.py index b60f6ac9..0f400d5e 100755 --- a/backend/src/api/routes/settings.py +++ b/backend/src/api/routes/settings.py @@ -303,11 +303,11 @@ async def get_consolidated_settings( config = config_manager.get_config() return ConsolidatedSettingsResponse( - environments=config.environments, + environments=[env.dict() for env in config.environments], connections=config.settings.connections, llm=config.settings.llm, - logging=config.settings.logging, - storage=config.settings.storage + logging=config.settings.logging.dict(), + storage=config.settings.storage.dict() ) # [/DEF:get_consolidated_settings:Function] diff --git a/backend/src/app.py b/backend/src/app.py index 59dd139d..0793a369 100755 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -115,7 +115,7 @@ app.include_router(plugins.router, prefix="/api/plugins", tags=["Plugins"]) app.include_router(tasks.router, prefix="/api/tasks", tags=["Tasks"]) app.include_router(settings.router, prefix="/api/settings", tags=["Settings"]) app.include_router(connections.router, prefix="/api/settings/connections", tags=["Connections"]) -app.include_router(environments.router, prefix="/api/environments", tags=["Environments"]) +app.include_router(environments.router, prefix="/api/settings/environments", tags=["Environments"]) app.include_router(mappings.router) app.include_router(migration.router) app.include_router(git.router) diff --git a/backend/src/core/config_models.py b/backend/src/core/config_models.py index 4413ab00..5ac659eb 100755 --- a/backend/src/core/config_models.py +++ b/backend/src/core/config_models.py @@ -48,6 +48,8 @@ class GlobalSettings(BaseModel): storage: StorageConfig = Field(default_factory=StorageConfig) default_environment_id: Optional[str] = None logging: LoggingConfig = Field(default_factory=LoggingConfig) + connections: List[dict] = [] + llm: dict = Field(default_factory=lambda: {"providers": [], "default_provider": ""}) # Task retention settings task_retention_days: int = 30 diff --git a/backend/src/core/superset_client.py b/backend/src/core/superset_client.py index c529d60d..fe084334 100644 --- a/backend/src/core/superset_client.py +++ b/backend/src/core/superset_client.py @@ -236,6 +236,82 @@ class SupersetClient: return result # [/DEF:get_datasets_summary:Function] + # [DEF:get_dataset_detail:Function] + # @PURPOSE: Fetches detailed dataset information including columns and linked dashboards + # @PRE: Client is authenticated and dataset_id exists. + # @POST: Returns detailed dataset info with columns and linked dashboards. + # @PARAM: dataset_id (int) - The dataset ID to fetch details for. + # @RETURN: Dict - Dataset details with columns and linked_dashboards. + # @RELATION: CALLS -> self.get_dataset + # @RELATION: CALLS -> self.network.request (for related_objects) + def get_dataset_detail(self, dataset_id: int) -> Dict: + with belief_scope("SupersetClient.get_dataset_detail", f"id={dataset_id}"): + # Get base dataset info + dataset = self.get_dataset(dataset_id) + + # Extract columns information + columns = dataset.get("columns", []) + column_info = [] + for col in columns: + column_info.append({ + "id": col.get("id"), + "name": col.get("column_name"), + "type": col.get("type"), + "is_dttm": col.get("is_dttm", False), + "is_active": col.get("is_active", True), + "description": col.get("description", "") + }) + + # Get linked dashboards using related_objects endpoint + linked_dashboards = [] + try: + related_objects = self.network.request( + method="GET", + endpoint=f"/dataset/{dataset_id}/related_objects" + ) + + # Handle different response formats + if isinstance(related_objects, dict): + if "dashboards" in related_objects: + dashboards_data = related_objects["dashboards"] + elif "result" in related_objects and isinstance(related_objects["result"], dict): + dashboards_data = related_objects["result"].get("dashboards", []) + else: + dashboards_data = [] + + for dash in dashboards_data: + linked_dashboards.append({ + "id": dash.get("id"), + "title": dash.get("dashboard_title") or dash.get("title", "Unknown"), + "slug": dash.get("slug") + }) + except Exception as e: + app_logger.warning(f"[get_dataset_detail][Warning] Failed to fetch related dashboards: {e}") + linked_dashboards = [] + + # Extract SQL table information + sql = dataset.get("sql", "") + + result = { + "id": dataset.get("id"), + "table_name": dataset.get("table_name"), + "schema": dataset.get("schema"), + "database": dataset.get("database", {}).get("database_name", "Unknown"), + "description": dataset.get("description", ""), + "columns": column_info, + "column_count": len(column_info), + "sql": sql, + "linked_dashboards": linked_dashboards, + "linked_dashboard_count": len(linked_dashboards), + "is_sqllab_view": dataset.get("is_sqllab_view", False), + "created_on": dataset.get("created_on"), + "changed_on": dataset.get("changed_on") + } + + app_logger.info(f"[get_dataset_detail][Exit] Got dataset {dataset_id} with {len(column_info)} columns and {len(linked_dashboards)} linked dashboards") + return result + # [/DEF:get_dataset_detail:Function] + # [DEF:get_dataset:Function] # @PURPOSE: Получает информацию о конкретном датасете по его ID. # @PARAM: dataset_id (int) - ID датасета. diff --git a/backend/src/core/utils/matching.py b/backend/src/core/utils/matching.py index 100c7809..4797935a 100644 --- a/backend/src/core/utils/matching.py +++ b/backend/src/core/utils/matching.py @@ -42,6 +42,8 @@ def suggest_mappings(source_databases: List[Dict], target_databases: List[Dict], name, score, index = match if score >= threshold: suggestions.append({ + "source_db": s_db['database_name'], + "target_db": target_databases[index]['database_name'], "source_db_uuid": s_db['uuid'], "target_db_uuid": target_databases[index]['uuid'], "confidence": score / 100.0 diff --git a/backend/src/core/utils/network.py b/backend/src/core/utils/network.py index 88b9f240..48620bed 100644 --- a/backend/src/core/utils/network.py +++ b/backend/src/core/utils/network.py @@ -118,14 +118,41 @@ class APIClient: def _init_session(self) -> requests.Session: with belief_scope("_init_session"): session = requests.Session() + + # Create a custom adapter that handles TLS issues + class TLSAdapter(HTTPAdapter): + def init_poolmanager(self, connections, maxsize, block=False): + from urllib3.poolmanager import PoolManager + import ssl + + # Create an SSL context that ignores TLSv1 unrecognized name errors + ctx = ssl.create_default_context() + ctx.set_ciphers('HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!SRP:!CAMELLIA') + + # Ignore TLSV1_UNRECOGNIZED_NAME errors by disabling hostname verification + # This is safe when verify_ssl is false (we're already not verifying the certificate) + ctx.check_hostname = False + + self.poolmanager = PoolManager( + num_pools=connections, + maxsize=maxsize, + block=block, + ssl_context=ctx + ) + retries = Retry(total=3, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504]) - adapter = HTTPAdapter(max_retries=retries) + adapter = TLSAdapter(max_retries=retries) session.mount('http://', adapter) session.mount('https://', adapter) + if not self.request_settings["verify_ssl"]: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) app_logger.warning("[_init_session][State] SSL verification disabled.") - session.verify = self.request_settings["verify_ssl"] + # When verify_ssl is false, we should also disable hostname verification + session.verify = False + else: + session.verify = True + return session # [/DEF:_init_session:Function] diff --git a/backend/src/dependencies.py b/backend/src/dependencies.py index 8707390f..c25db490 100755 --- a/backend/src/dependencies.py +++ b/backend/src/dependencies.py @@ -1,8 +1,8 @@ # [DEF:Dependencies:Module] # @SEMANTICS: dependency, injection, singleton, factory, auth, jwt -# @PURPOSE: Manages the creation and provision of shared application dependencies, such as the PluginLoader and TaskManager, to avoid circular imports. +# @PURPOSE: Manages creation and provision of shared application dependencies, such as PluginLoader and TaskManager, to avoid circular imports. # @LAYER: Core -# @RELATION: Used by the main app and API routers to get access to shared instances. +# @RELATION: Used by main app and API routers to get access to shared instances. from pathlib import Path from fastapi import Depends, HTTPException, status @@ -13,6 +13,7 @@ from .core.task_manager import TaskManager from .core.config_manager import ConfigManager from .core.scheduler import SchedulerService from .services.resource_service import ResourceService +from .services.mapping_service import MappingService from .core.database import init_db, get_auth_db from .core.logger import logger from .core.auth.jwt import decode_token @@ -29,12 +30,12 @@ config_manager = ConfigManager(config_path=str(config_path)) init_db() # [DEF:get_config_manager:Function] -# @PURPOSE: Dependency injector for the ConfigManager. +# @PURPOSE: Dependency injector for ConfigManager. # @PRE: Global config_manager must be initialized. # @POST: Returns shared ConfigManager instance. # @RETURN: ConfigManager - The shared config manager instance. def get_config_manager() -> ConfigManager: - """Dependency injector for the ConfigManager.""" + """Dependency injector for ConfigManager.""" return config_manager # [/DEF:get_config_manager:Function] @@ -54,54 +55,64 @@ resource_service = ResourceService() logger.info("ResourceService initialized") # [DEF:get_plugin_loader:Function] -# @PURPOSE: Dependency injector for the PluginLoader. +# @PURPOSE: Dependency injector for PluginLoader. # @PRE: Global plugin_loader must be initialized. # @POST: Returns shared PluginLoader instance. # @RETURN: PluginLoader - The shared plugin loader instance. def get_plugin_loader() -> PluginLoader: - """Dependency injector for the PluginLoader.""" + """Dependency injector for PluginLoader.""" return plugin_loader # [/DEF:get_plugin_loader:Function] # [DEF:get_task_manager:Function] -# @PURPOSE: Dependency injector for the TaskManager. +# @PURPOSE: Dependency injector for TaskManager. # @PRE: Global task_manager must be initialized. # @POST: Returns shared TaskManager instance. # @RETURN: TaskManager - The shared task manager instance. def get_task_manager() -> TaskManager: - """Dependency injector for the TaskManager.""" + """Dependency injector for TaskManager.""" return task_manager # [/DEF:get_task_manager:Function] # [DEF:get_scheduler_service:Function] -# @PURPOSE: Dependency injector for the SchedulerService. +# @PURPOSE: Dependency injector for SchedulerService. # @PRE: Global scheduler_service must be initialized. # @POST: Returns shared SchedulerService instance. # @RETURN: SchedulerService - The shared scheduler service instance. def get_scheduler_service() -> SchedulerService: - """Dependency injector for the SchedulerService.""" + """Dependency injector for SchedulerService.""" return scheduler_service # [/DEF:get_scheduler_service:Function] # [DEF:get_resource_service:Function] -# @PURPOSE: Dependency injector for the ResourceService. +# @PURPOSE: Dependency injector for ResourceService. # @PRE: Global resource_service must be initialized. # @POST: Returns shared ResourceService instance. # @RETURN: ResourceService - The shared resource service instance. def get_resource_service() -> ResourceService: - """Dependency injector for the ResourceService.""" + """Dependency injector for ResourceService.""" return resource_service # [/DEF:get_resource_service:Function] +# [DEF:get_mapping_service:Function] +# @PURPOSE: Dependency injector for MappingService. +# @PRE: Global config_manager must be initialized. +# @POST: Returns new MappingService instance. +# @RETURN: MappingService - A new mapping service instance. +def get_mapping_service() -> MappingService: + """Dependency injector for MappingService.""" + return MappingService(config_manager) +# [/DEF:get_mapping_service:Function] + # [DEF:oauth2_scheme:Variable] # @PURPOSE: OAuth2 password bearer scheme for token extraction. oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login") # [/DEF:oauth2_scheme:Variable] # [DEF:get_current_user:Function] -# @PURPOSE: Dependency for retrieving the currently authenticated user from a JWT. +# @PURPOSE: Dependency for retrieving currently authenticated user from a JWT. # @PRE: JWT token provided in Authorization header. -# @POST: Returns the User object if token is valid. +# @POST: Returns User object if token is valid. # @THROW: HTTPException 401 if token is invalid or user not found. # @PARAM: token (str) - Extracted JWT token. # @PARAM: db (Session) - Auth database session. @@ -157,4 +168,4 @@ def has_permission(resource: str, action: str): return permission_checker # [/DEF:has_permission:Function] -# [/DEF:Dependencies:Module] \ No newline at end of file +# [/DEF:Dependencies:Module] diff --git a/backend/src/scripts/test_dataset_dashboard_relations.py b/backend/src/scripts/test_dataset_dashboard_relations.py new file mode 100644 index 00000000..7bbfa8e2 --- /dev/null +++ b/backend/src/scripts/test_dataset_dashboard_relations.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +""" +Script to test dataset-to-dashboard relationships from Superset API. + +Usage: + cd backend && .venv/bin/python3 src/scripts/test_dataset_dashboard_relations.py +""" + +import json +import sys +from pathlib import Path + +# Add src to path (parent of scripts directory) +sys.path.append(str(Path(__file__).parent.parent.parent)) + +from src.core.superset_client import SupersetClient +from src.core.config_manager import ConfigManager +from src.core.logger import logger + + +def test_dashboard_dataset_relations(): + """Test fetching dataset-to-dashboard relationships.""" + + # Load environment from existing config + config_manager = ConfigManager() + environments = config_manager.get_environments() + + if not environments: + logger.error("No environments configured!") + return + + # Use first available environment + env = environments[0] + logger.info(f"Using environment: {env.name} ({env.url})") + + client = SupersetClient(env) + + try: + # Authenticate + logger.info("Authenticating to Superset...") + client.authenticate() + logger.info("Authentication successful!") + + # Test dashboard ID 13 + dashboard_id = 13 + logger.info(f"\n=== Fetching Dashboard {dashboard_id} ===") + dashboard = client.network.request(method="GET", endpoint=f"/dashboard/{dashboard_id}") + + print("\nDashboard structure:") + print(f" ID: {dashboard.get('id')}") + print(f" Title: {dashboard.get('dashboard_title')}") + print(f" Published: {dashboard.get('published')}") + + # Check for slices/charts + if 'slices' in dashboard: + logger.info(f"\n Found {len(dashboard['slices'])} slices/charts in dashboard") + for i, slice_data in enumerate(dashboard['slices'][:5]): # Show first 5 + print(f" Slice {i+1}:") + print(f" ID: {slice_data.get('slice_id')}") + print(f" Name: {slice_data.get('slice_name')}") + # Check for datasource_id + if 'datasource_id' in slice_data: + print(f" Datasource ID: {slice_data['datasource_id']}") + if 'datasource_name' in slice_data: + print(f" Datasource Name: {slice_data['datasource_name']}") + if 'datasource_type' in slice_data: + print(f" Datasource Type: {slice_data['datasource_type']}") + else: + logger.warning(" No 'slices' field found in dashboard response") + logger.info(f" Available fields: {list(dashboard.keys())}") + + # Test dataset ID 26 + dataset_id = 26 + logger.info(f"\n=== Fetching Dataset {dataset_id} ===") + dataset = client.get_dataset(dataset_id) + + print("\nDataset structure:") + print(f" ID: {dataset.get('id')}") + print(f" Table Name: {dataset.get('table_name')}") + print(f" Schema: {dataset.get('schema')}") + print(f" Database: {dataset.get('database', {}).get('database_name', 'Unknown')}") + + # Check for dashboards that use this dataset + logger.info(f"\n=== Finding Dashboards using Dataset {dataset_id} ===") + + # Method: Use Superset's related_objects API + try: + logger.info(f" Using /api/v1/dataset/{dataset_id}/related_objects endpoint...") + related_objects = client.network.request( + method="GET", + endpoint=f"/dataset/{dataset_id}/related_objects" + ) + + logger.info(f" Related objects response type: {type(related_objects)}") + logger.info(f" Related objects keys: {list(related_objects.keys()) if isinstance(related_objects, dict) else 'N/A'}") + + # Check for dashboards in related objects + if 'dashboards' in related_objects: + dashboards = related_objects['dashboards'] + logger.info(f" Found {len(dashboards)} dashboards using this dataset:") + + for dash in dashboards: + logger.info(f" - Dashboard ID {dash.get('id')}: {dash.get('dashboard_title', dash.get('title', 'Unknown'))}") + elif 'result' in related_objects: + # Some Superset versions use 'result' wrapper + result = related_objects['result'] + if 'dashboards' in result: + dashboards = result['dashboards'] + logger.info(f" Found {len(dashboards)} dashboards using this dataset:") + + for dash in dashboards: + logger.info(f" - Dashboard ID {dash.get('id')}: {dash.get('dashboard_title', dash.get('title', 'Unknown'))}") + else: + logger.warning(f" No 'dashboards' key in result. Keys: {list(result.keys())}") + else: + logger.warning(f" No 'dashboards' key in response. Available keys: {list(related_objects.keys())}") + logger.info(f" Full related_objects response:") + print(json.dumps(related_objects, indent=2, default=str)[:1000]) + + except Exception as e: + logger.error(f" Error fetching related objects: {e}") + import traceback + traceback.print_exc() + + # Method 2: Try to use the position_json from dashboard + logger.info(f"\n=== Analyzing Dashboard Position JSON ===") + if 'position_json' in dashboard: + position_data = json.loads(dashboard['position_json']) + logger.info(f" Position data type: {type(position_data)}") + + # Look for datasource references + datasource_ids = set() + if isinstance(position_data, dict): + for key, value in position_data.items(): + if 'datasource' in key.lower() or key == 'DASHBOARD_VERSION_KEY': + logger.debug(f" Key: {key}, Value type: {type(value)}") + elif isinstance(position_data, list): + logger.info(f" Position data has {len(position_data)} items") + for item in position_data[:3]: # Show first 3 + logger.debug(f" Item: {type(item)}, keys: {list(item.keys()) if isinstance(item, dict) else 'N/A'}") + if isinstance(item, dict): + if 'datasource_id' in item: + datasource_ids.add(item['datasource_id']) + + if datasource_ids: + logger.info(f" Found datasource IDs: {datasource_ids}") + + # Save full response for analysis + output_file = Path(__file__).parent / "dataset_dashboard_analysis.json" + with open(output_file, 'w') as f: + json.dump({ + 'dashboard': dashboard, + 'dataset': dataset + }, f, indent=2, default=str) + logger.info(f"\nFull response saved to: {output_file}") + + except Exception as e: + logger.error(f"Error: {e}", exc_info=True) + raise + + +if __name__ == "__main__": + test_dashboard_dataset_relations() diff --git a/frontend/src/lib/api.js b/frontend/src/lib/api.js index 0bd4e6ab..aa7f23be 100755 --- a/frontend/src/lib/api.js +++ b/frontend/src/lib/api.js @@ -165,12 +165,32 @@ export const api = { getStorageSettings: () => fetchApi('/settings/storage'), updateStorageSettings: (storage) => requestApi('/settings/storage', 'PUT', storage), getEnvironmentsList: () => fetchApi('/environments'), + getEnvironmentDatabases: (id) => fetchApi(`/environments/${id}/databases`), // Dashboards - getDashboards: (envId) => fetchApi(`/dashboards?env_id=${envId}`), + getDashboards: (envId, options = {}) => { + const params = new URLSearchParams({ env_id: envId }); + if (options.search) params.append('search', options.search); + if (options.page) params.append('page', options.page); + if (options.page_size) params.append('page_size', options.page_size); + return fetchApi(`/dashboards?${params.toString()}`); + }, + getDatabaseMappings: (sourceEnvId, targetEnvId) => fetchApi(`/dashboards/db-mappings?source_env_id=${sourceEnvId}&target_env_id=${targetEnvId}`), - // Datasets - getDatasets: (envId) => fetchApi(`/datasets?env_id=${envId}`), + // Datasets + getDatasets: (envId, options = {}) => { + const params = new URLSearchParams({ env_id: envId }); + if (options.search) params.append('search', options.search); + if (options.page) params.append('page', options.page); + if (options.page_size) params.append('page_size', options.page_size); + return fetchApi(`/datasets?${params.toString()}`); + }, + getDatasetIds: (envId, options = {}) => { + const params = new URLSearchParams({ env_id: envId }); + if (options.search) params.append('search', options.search); + return fetchApi(`/datasets/ids?${params.toString()}`); + }, + getDatasetDetail: (envId, datasetId) => fetchApi(`/datasets/${datasetId}?env_id=${envId}`), // Settings getConsolidatedSettings: () => fetchApi('/settings/consolidated'), diff --git a/frontend/src/routes/+layout.svelte b/frontend/src/routes/+layout.svelte index 9139dd5a..d5de61f1 100644 --- a/frontend/src/routes/+layout.svelte +++ b/frontend/src/routes/+layout.svelte @@ -29,14 +29,16 @@ -
+
- +
+ +
-
+
diff --git a/frontend/src/routes/datasets/+page.svelte b/frontend/src/routes/datasets/+page.svelte index 96ce291b..0c7df63c 100644 --- a/frontend/src/routes/datasets/+page.svelte +++ b/frontend/src/routes/datasets/+page.svelte @@ -10,7 +10,11 @@ * @UX_STATE: Loading -> Shows skeleton loader * @UX_STATE: Loaded -> Shows dataset grid with mapping progress * @UX_STATE: Error -> Shows error banner with retry button + * @UX_STATE: Selecting -> Checkboxes checked, floating action panel appears + * @UX_STATE: BulkAction-Modal -> Map Columns or Generate Docs modal open * @UX_FEEDBACK: Clicking task status opens Task Drawer + * @UX_FEEDBACK: Mapped % column shows progress bar + percentage text + * @UX_FEEDBACK: Floating panel slides up from bottom when items selected * @UX_RECOVERY: Refresh button reloads dataset list */ @@ -19,12 +23,45 @@ import { t } from '$lib/i18n'; import { openDrawerForTask } from '$lib/stores/taskDrawer.js'; import { api } from '$lib/api.js'; + import { debounce } from '$lib/utils/debounce.js'; // State let selectedEnv = null; let datasets = []; let isLoading = true; let error = null; + + // Pagination state + let currentPage = 1; + let pageSize = 10; + let totalPages = 1; + let total = 0; + + // Selection state + let selectedIds = new Set(); + let isAllSelected = false; + let isAllVisibleSelected = false; + + // Search state + let searchQuery = ''; + + // Bulk action modal state + let showMapColumnsModal = false; + let showGenerateDocsModal = false; + let mapSourceType = 'postgresql'; + let mapConnectionId = null; + let mapFileData = null; + let llmProvider = ''; + let llmOptions = {}; + + // Environment options - will be loaded from API + let environments = []; + + // Debounced search function + const debouncedSearch = debounce((query) => { + searchQuery = query; + loadDatasets(); + }, 300); // Load environments and datasets on mount onMount(async () => { @@ -59,7 +96,21 @@ isLoading = true; error = null; try { - const response = await api.getDatasets(selectedEnv); + const response = await api.getDatasets(selectedEnv, { + search: searchQuery || undefined, + page: currentPage, + page_size: pageSize + }); + + // Preserve selected IDs across pagination + const newSelectedIds = new Set(); + response.datasets.forEach(d => { + if (selectedIds.has(d.id)) { + newSelectedIds.add(d.id); + } + }); + selectedIds = newSelectedIds; + datasets = response.datasets.map(d => ({ id: d.id, table_name: d.table_name, @@ -75,6 +126,13 @@ } : null, actions: ['map_columns'] // All datasets have map columns option })); + + // Update pagination state + total = response.total; + totalPages = response.total_pages; + + // Update selection state + updateSelectionState(); } catch (err) { error = err.message || 'Failed to load datasets'; console.error('[DatasetHub][Coherence:Failed]', err); @@ -86,15 +144,155 @@ // Handle environment change function handleEnvChange(event) { selectedEnv = event.target.value; + currentPage = 1; + selectedIds.clear(); loadDatasets(); } + // Handle search input + function handleSearch(event) { + debouncedSearch(event.target.value); + } + + // Handle page change + function handlePageChange(page) { + currentPage = page; + loadDatasets(); + } + + // Handle page size change + function handlePageSizeChange(event) { + pageSize = parseInt(event.target.value); + currentPage = 1; + loadDatasets(); + } + + // Update selection state based on current selection + function updateSelectionState() { + const visibleCount = datasets.length; + const totalCount = total; + + isAllSelected = selectedIds.size === totalCount && totalCount > 0; + isAllVisibleSelected = selectedIds.size === visibleCount && visibleCount > 0; + } + + // Handle checkbox change for individual dataset + function handleCheckboxChange(dataset, event) { + if (event.target.checked) { + selectedIds.add(dataset.id); + } else { + selectedIds.delete(dataset.id); + } + selectedIds = selectedIds; // Trigger reactivity + updateSelectionState(); + } + + // Handle select all + async function handleSelectAll() { + if (isAllSelected) { + selectedIds.clear(); + } else { + // Get all dataset IDs from API (including non-visible ones) + try { + const response = await api.getDatasetIds(selectedEnv, { + search: searchQuery || undefined + }); + response.dataset_ids.forEach(id => selectedIds.add(id)); + } catch (err) { + console.error('[DatasetHub][Coherence:Failed] Failed to fetch all dataset IDs:', err); + // Fallback to selecting visible datasets if API fails + datasets.forEach(d => selectedIds.add(d.id)); + } + } + selectedIds = selectedIds; // Trigger reactivity + updateSelectionState(); + } + + // Handle select visible + function handleSelectVisible() { + if (isAllVisibleSelected) { + datasets.forEach(d => selectedIds.delete(d.id)); + } else { + datasets.forEach(d => selectedIds.add(d.id)); + } + selectedIds = selectedIds; // Trigger reactivity + updateSelectionState(); + } + // Handle action click function handleAction(dataset, action) { console.log(`[DatasetHub][Action] ${action} on dataset ${dataset.table_name}`); + if (action === 'map_columns') { - // Navigate to mapping interface - goto(`/mapper?dataset_id=${dataset.id}`); + // Show map columns modal + showMapColumnsModal = true; + mapSourceType = 'postgresql'; + mapConnectionId = null; + mapFileData = null; + } else if (action === 'generate_docs') { + // Show generate docs modal + showGenerateDocsModal = true; + llmProvider = ''; + llmOptions = {}; + } + } + + // Handle bulk map columns + async function handleBulkMapColumns() { + if (selectedIds.size === 0) return; + + try { + const response = await api.postApi('/datasets/map-columns', { + env_id: selectedEnv, + dataset_ids: Array.from(selectedIds), + source_type: mapSourceType, + connection_id: mapConnectionId || undefined, + file_data: mapFileData || undefined + }); + console.log('[DatasetHub][Action] Bulk map columns task created:', response.task_id); + + // Close modal and open task drawer + showMapColumnsModal = false; + selectedIds.clear(); + updateSelectionState(); + + if (response.task_id) { + openDrawerForTask(response.task_id); + } + } catch (err) { + console.error('[DatasetHub][Coherence:Failed]', err); + alert('Failed to create mapping task'); + } + } + + // Handle bulk generate docs + async function handleBulkGenerateDocs() { + if (selectedIds.size === 0) return; + if (!llmProvider) { + alert('Please select an LLM provider'); + return; + } + + try { + const response = await api.postApi('/datasets/generate-docs', { + env_id: selectedEnv, + dataset_ids: Array.from(selectedIds), + llm_provider: llmProvider, + options: llmOptions + }); + console.log('[DatasetHub][Action] Bulk generate docs task created:', response.task_id); + + // Close modal and open task drawer + showGenerateDocsModal = false; + selectedIds.clear(); + updateSelectionState(); + + if (response.task_id) { + openDrawerForTask(response.task_id); + } + } catch (err) { + console.error('[DatasetHub][Coherence:Failed]', err); + alert('Failed to create documentation generation task'); } } @@ -111,7 +309,7 @@ if (!status) return ''; switch (status.toLowerCase()) { case 'running': - return ''; + return ''; case 'success': return ''; case 'error': @@ -162,6 +360,10 @@ @apply px-4 py-2 bg-blue-600 text-white rounded-lg hover:bg-blue-700 transition-colors; } + .search-input { + @apply px-4 py-2 border border-gray-300 rounded-lg bg-white focus:outline-none focus:ring-2 focus:ring-blue-500; + } + .error-banner { @apply bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded mb-4 flex items-center justify-between; } @@ -170,6 +372,14 @@ @apply px-4 py-2 bg-red-600 text-white rounded hover:bg-red-700 transition-colors; } + .toolbar { + @apply flex items-center justify-between mb-4 gap-4; + } + + .selection-buttons { + @apply flex items-center gap-2; + } + .dataset-grid { @apply bg-white border border-gray-200 rounded-lg overflow-hidden; } @@ -186,6 +396,10 @@ @apply border-b-0; } + .col-checkbox { + @apply col-span-1; + } + .col-table-name { @apply col-span-3 font-medium text-gray-900; } @@ -203,7 +417,7 @@ } .col-actions { - @apply col-span-2; + @apply col-span-1; } .mapping-progress { @@ -233,6 +447,58 @@ .skeleton { @apply animate-pulse bg-gray-200 rounded; } + + .floating-panel { + @apply fixed bottom-0 left-0 right-0 bg-white border-t border-gray-200 shadow-lg p-4 transition-transform transform translate-y-full; + } + + .floating-panel.visible { + @apply transform translate-y-0; + } + + .modal-overlay { + @apply fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50; + } + + .modal { + @apply bg-white rounded-lg shadow-xl max-w-2xl w-full mx-4 max-h-[80vh] overflow-y-auto; + } + + .modal-header { + @apply px-6 py-4 border-b border-gray-200 flex items-center justify-between relative; + } + + .close-modal-btn { + @apply absolute top-4 right-4 p-2 text-gray-400 hover:text-gray-600 hover:bg-gray-100 rounded-full transition-all; + } + + .modal-body { + @apply px-6 py-4; + } + + .modal-footer { + @apply px-6 py-4 border-t border-gray-200 flex justify-end gap-3; + } + + .pagination { + @apply flex items-center justify-between px-4 py-3 bg-gray-50 border-t border-gray-200; + } + + .pagination-info { + @apply text-sm text-gray-600; + } + + .pagination-controls { + @apply flex items-center gap-2; + } + + .page-btn { + @apply px-3 py-1 border border-gray-300 rounded hover:bg-gray-100 disabled:opacity-50 disabled:cursor-not-allowed; + } + + .page-btn.active { + @apply bg-blue-600 text-white border-blue-600; + }
@@ -265,6 +531,7 @@ {#if isLoading}
+
@@ -273,6 +540,7 @@
{#each Array(5) as _}
+
@@ -290,10 +558,45 @@

{$t.datasets?.empty || 'No datasets found'}

{:else} + +
+
+ + + {#if selectedIds.size > 0} + + {selectedIds.size} selected + + {/if} +
+
+ +
+
+
+
{$t.datasets?.table_name || 'Table Name'}
{$t.datasets?.schema || 'Schema'}
{$t.datasets?.mapped_fields || 'Mapped Fields'}
@@ -304,9 +607,23 @@ {#each datasets as dataset}
+ +
+ handleCheckboxChange(dataset, e)} + /> +
+
- {dataset.table_name} + + {dataset.table_name} +
@@ -355,21 +672,241 @@
-
- {#if dataset.actions.includes('map_columns')} - - {/if} -
+ {#if dataset.actions.includes('map_columns')} + + {/if}
{/each}
+ + + {#if totalPages > 1} + + {/if} + + + {#if selectedIds.size > 0} +
+
+
+ + ✓ {selectedIds.size} selected + +
+
+ + + +
+
+
+ {/if} + {/if} + + + {#if showMapColumnsModal} +
showMapColumnsModal = false}> +
+
+

Bulk Column Mapping

+ +
+
+
+
+ + +
+ {#if mapSourceType === 'postgresql'} +
+ + +
+ {:else} +
+ + +
+ {/if} +
+ +
+ {#each Array.from(selectedIds) as id} + {#each datasets as d} + {#if d.id === id} +
{d.table_name}
+ {/if} + {/each} + {/each} +
+
+
+
+
+ + +
+
+
+ {/if} + + + {#if showGenerateDocsModal} +
showGenerateDocsModal = false}> +
+
+

Bulk Documentation Generation

+ +
+
+
+
+ + +
+
+ +
+ {#each Array.from(selectedIds) as id} + {#each datasets as d} + {#if d.id === id} +
{d.table_name}
+ {/if} + {/each} + {/each} +
+
+
+
+
+ + +
+
+
{/if}
diff --git a/frontend/src/routes/datasets/[id]/+page.svelte b/frontend/src/routes/datasets/[id]/+page.svelte new file mode 100644 index 00000000..539d680c --- /dev/null +++ b/frontend/src/routes/datasets/[id]/+page.svelte @@ -0,0 +1,418 @@ + + + + + +
+ +
+
+ + {#if dataset} +

{dataset.table_name}

+

{dataset.schema} • {dataset.database}

+ {:else if !isLoading} +

{$t.datasets?.detail_title || 'Dataset Details'}

+ {/if} +
+ +
+ + + {#if error} +
+ {error} + +
+ {/if} + + + {#if isLoading} +
+
+
+ {#each Array(5) as _} +
+
+
+
+ {/each} +
+
+
+
+ {#each Array(4) as _} +
+
+
+
+ {/each} +
+
+
+ {:else if dataset} +
+ +
+

{$t.datasets?.info || 'Dataset Information'}

+
+ {$t.datasets?.table_name || 'Table Name'} + {dataset.table_name} +
+
+ {$t.datasets?.schema || 'Schema'} + {dataset.schema || '-'} +
+
+ {$t.datasets?.database || 'Database'} + {dataset.database} +
+
+ {$t.datasets?.columns_count || 'Columns'} + {dataset.column_count} +
+
+ {$t.datasets?.linked_dashboards || 'Linked Dashboards'} + {dataset.linked_dashboard_count} +
+ {#if dataset.is_sqllab_view} +
+ {$t.datasets?.type || 'Type'} + SQL Lab View +
+ {/if} + {#if dataset.created_on} +
+ {$t.datasets?.created || 'Created'} + {new Date(dataset.created_on).toLocaleDateString()} +
+ {/if} + {#if dataset.changed_on} +
+ {$t.datasets?.updated || 'Updated'} + {new Date(dataset.changed_on).toLocaleDateString()} +
+ {/if} +
+ + + {#if dataset.linked_dashboards && dataset.linked_dashboards.length > 0} +
+

{$t.datasets?.linked_dashboards || 'Linked Dashboards'} ({dataset.linked_dashboard_count})

+
+ {#each dataset.linked_dashboards as dashboard} +
navigateToDashboard(dashboard.id)} + role="button" + tabindex="0" + > +
+ + + + + +
+
+
{dashboard.title}
+
ID: {dashboard.id}{#if dashboard.slug} • {dashboard.slug}{/if}
+
+ + + +
+ {/each} +
+
+ {/if} + + +
+

{$t.datasets?.columns || 'Columns'} ({dataset.column_count})

+ {#if dataset.columns && dataset.columns.length > 0} +
+ {#each dataset.columns as column} +
+
+ {column.name} + {#if column.type} + {column.type} + {/if} +
+
+ {#if column.is_dttm} + 📅 Date/Time + {/if} + {#if !column.is_active} + (Inactive) + {/if} + + {column.description ? '✓ Mapped' : 'Unmapped'} + +
+ {#if column.description} +

{column.description}

+ {/if} +
+ {/each} +
+ {:else} +
+ {$t.datasets?.no_columns || 'No columns found'} +
+ {/if} +
+ + + {#if dataset.sql} +
+

{$t.datasets?.sql_query || 'SQL Query'}

+ 
{dataset.sql}
+
+ {/if} +
+ {:else} +
+ + + +

{$t.datasets?.not_found || 'Dataset not found'}

+
+ {/if} +
+ + \ No newline at end of file diff --git a/frontend/src/routes/settings/+page.svelte b/frontend/src/routes/settings/+page.svelte index 15ff4e50..bf0ad075 100644 --- a/frontend/src/routes/settings/+page.svelte +++ b/frontend/src/routes/settings/+page.svelte @@ -68,6 +68,24 @@ addToast($t.settings?.save_failed || 'Failed to save settings', 'error'); } } + + // Placeholder functions for environment actions + function handleTestEnv(id) { + console.log(`[SettingsPage][Action] Test environment ${id}`); + addToast('Environment test started', 'info'); + } + + function editEnv(env) { + console.log(`[SettingsPage][Action] Edit environment ${env.id}`); + // TODO: Open edit modal + } + + function handleDeleteEnv(id) { + if (confirm('Are you sure you want to delete this environment?')) { + console.log(`[SettingsPage][Action] Delete environment ${id}`); + // TODO: Call API to delete + } + }