semantic cleanup
This commit is contained in:
@@ -3,30 +3,7 @@
|
||||
# @SEMANTICS: superset, async, client, httpx, dashboards, datasets
|
||||
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously.
|
||||
# @LAYER: Core
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously.
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously.
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously.
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously.
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously.
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously.
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously.
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously.
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously.
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously.
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously.
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously.
|
||||
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
import asyncio
|
||||
|
||||
@@ -69,12 +69,41 @@ class ConfigManager:
|
||||
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
# [DEF:_apply_features_from_env:Function]
|
||||
# @PURPOSE: Read FEATURES__* env vars and apply them to a GlobalSettings features config.
|
||||
# @SIDE_EFFECT: Reads os.environ; mutates settings.features in-place.
|
||||
# @RATIONALE: Env vars seed the initial defaults. After first bootstrap, DB is source of truth.
|
||||
@staticmethod
|
||||
def _apply_features_from_env(settings: GlobalSettings) -> None:
|
||||
with belief_scope("ConfigManager._apply_features_from_env"):
|
||||
dataset_review_env = os.getenv("FEATURES__DATASET_REVIEW")
|
||||
if dataset_review_env is not None:
|
||||
parsed = dataset_review_env.strip().lower() in ("true", "1", "yes")
|
||||
settings.features.dataset_review = parsed
|
||||
logger.reason(
|
||||
"Applied FEATURES__DATASET_REVIEW from env",
|
||||
extra={"value": parsed, "raw": dataset_review_env},
|
||||
)
|
||||
|
||||
health_monitor_env = os.getenv("FEATURES__HEALTH_MONITOR")
|
||||
if health_monitor_env is not None:
|
||||
parsed = health_monitor_env.strip().lower() in ("true", "1", "yes")
|
||||
settings.features.health_monitor = parsed
|
||||
logger.reason(
|
||||
"Applied FEATURES__HEALTH_MONITOR from env",
|
||||
extra={"value": parsed, "raw": health_monitor_env},
|
||||
)
|
||||
|
||||
# [/DEF:_apply_features_from_env:Function]
|
||||
|
||||
# [DEF:_default_config:Function]
|
||||
# @PURPOSE: Build default application configuration fallback.
|
||||
def _default_config(self) -> AppConfig:
|
||||
with belief_scope("ConfigManager._default_config"):
|
||||
logger.reason("Building default AppConfig fallback")
|
||||
return AppConfig(environments=[], settings=GlobalSettings())
|
||||
config = AppConfig(environments=[], settings=GlobalSettings())
|
||||
self._apply_features_from_env(config.settings)
|
||||
return config
|
||||
|
||||
# [/DEF:_default_config:Function]
|
||||
|
||||
@@ -200,6 +229,7 @@ class ConfigManager:
|
||||
"settings": self.raw_payload.get("settings", {}),
|
||||
}
|
||||
)
|
||||
self._apply_features_from_env(config.settings)
|
||||
logger.reason(
|
||||
"Legacy payload validated; persisting migrated configuration to database",
|
||||
extra={
|
||||
|
||||
@@ -71,6 +71,19 @@ class CleanReleaseConfig(BaseModel):
|
||||
# [/DEF:CleanReleaseConfig:DataClass]
|
||||
|
||||
|
||||
# [DEF:FeaturesConfig:DataClass]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Top-level feature flags that toggle entire project features on/off.
|
||||
# @RATIONALE: Features are read from environment variables on bootstrap and persisted in DB.
|
||||
# DB is source of truth after initial bootstrap; env vars only seed defaults.
|
||||
class FeaturesConfig(BaseModel):
|
||||
dataset_review: bool = True
|
||||
health_monitor: bool = True
|
||||
|
||||
|
||||
# [/DEF:FeaturesConfig:DataClass]
|
||||
|
||||
|
||||
# [DEF:GlobalSettings:DataClass]
|
||||
# @PURPOSE: Represents global application settings.
|
||||
class GlobalSettings(BaseModel):
|
||||
@@ -78,6 +91,7 @@ class GlobalSettings(BaseModel):
|
||||
clean_release: CleanReleaseConfig = Field(default_factory=CleanReleaseConfig)
|
||||
default_environment_id: Optional[str] = None
|
||||
logging: LoggingConfig = Field(default_factory=LoggingConfig)
|
||||
features: FeaturesConfig = Field(default_factory=FeaturesConfig)
|
||||
connections: List[dict] = []
|
||||
llm: dict = Field(
|
||||
default_factory=lambda: {
|
||||
|
||||
@@ -49,7 +49,7 @@ class SupersetClient:
|
||||
with belief_scope("SupersetClientInit"):
|
||||
app_logger.reason(
|
||||
"Initializing Superset client for environment",
|
||||
extra={"environment": getattr(env, "id", None), "name": env.name},
|
||||
extra={"environment": getattr(env, "id", None), "env_name": env.name},
|
||||
)
|
||||
self.env = env
|
||||
# Construct auth payload expected by Superset API
|
||||
|
||||
65
backend/src/core/superset_client/__init__.py
Normal file
65
backend/src/core/superset_client/__init__.py
Normal file
@@ -0,0 +1,65 @@
|
||||
# [DEF:SupersetClientModule:Module]
|
||||
#
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Предоставляет высокоуровневый клиент для взаимодействия с Superset REST API, инкапсулируя логику запросов, обработку ошибок и пагинацию.
|
||||
# @RELATION: DEPENDS_ON -> [ConfigModels]
|
||||
# @RELATION: DEPENDS_ON -> [APIClient]
|
||||
# @RELATION: DEPENDS_ON -> [SupersetAPIError]
|
||||
# @RELATION: DEPENDS_ON -> [get_filename_from_headers]
|
||||
#
|
||||
# @PUBLIC_API: SupersetClient
|
||||
#
|
||||
# @RATIONALE: Decomposed from monolithic superset_client.py (2145 lines) into
|
||||
# domain-scoped mixins to satisfy INV_7 (module < 400 lines). The composed class
|
||||
# preserves the original public API surface — all consumers continue to import
|
||||
# `from src.core.superset_client import SupersetClient` without changes.
|
||||
# @REJECTED: Keeping a single 2145-line file — violates fractal limit INV_7.
|
||||
|
||||
from ._base import SupersetClientBase
|
||||
from ._user_projection import SupersetUserProjectionMixin
|
||||
from ._dashboards_list import SupersetDashboardsListMixin
|
||||
from ._dashboards_filters import SupersetDashboardsFiltersMixin
|
||||
from ._dashboards_crud import SupersetDashboardsCrudMixin
|
||||
from ._charts import SupersetChartsMixin
|
||||
from ._datasets import SupersetDatasetsMixin
|
||||
from ._datasets_preview import SupersetDatasetsPreviewMixin
|
||||
from ._databases import SupersetDatabasesMixin
|
||||
|
||||
|
||||
# [DEF:SupersetClient:Class]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Класс-обёртка над Superset REST API, предоставляющий методы для работы с дашбордами и датасетами.
|
||||
# @RELATION: DEPENDS_ON -> [ConfigModels]
|
||||
# @RELATION: DEPENDS_ON -> [APIClient]
|
||||
# @RELATION: DEPENDS_ON -> [SupersetAPIError]
|
||||
# @RELATION: INHERITS -> [SupersetClientBase]
|
||||
# @RELATION: INHERITS -> [SupersetUserProjectionMixin]
|
||||
# @RELATION: INHERITS -> [SupersetDashboardsListMixin]
|
||||
# @RELATION: INHERITS -> [SupersetDashboardsFiltersMixin]
|
||||
# @RELATION: INHERITS -> [SupersetDashboardsCrudMixin]
|
||||
# @RELATION: INHERITS -> [SupersetChartsMixin]
|
||||
# @RELATION: INHERITS -> [SupersetDatasetsMixin]
|
||||
# @RELATION: INHERITS -> [SupersetDatasetsPreviewMixin]
|
||||
# @RELATION: INHERITS -> [SupersetDatabasesMixin]
|
||||
class SupersetClient(
|
||||
SupersetDatabasesMixin,
|
||||
SupersetDatasetsPreviewMixin,
|
||||
SupersetDatasetsMixin,
|
||||
SupersetChartsMixin,
|
||||
SupersetDashboardsCrudMixin,
|
||||
SupersetDashboardsFiltersMixin,
|
||||
SupersetDashboardsListMixin,
|
||||
SupersetUserProjectionMixin,
|
||||
SupersetClientBase,
|
||||
):
|
||||
"""Composed Superset REST API client.
|
||||
|
||||
MRO order ensures domain mixins resolve before base class.
|
||||
All consumers continue to use: from src.core.superset_client import SupersetClient
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
# [/DEF:SupersetClient:Class]
|
||||
|
||||
# [/DEF:SupersetClientModule:Module]
|
||||
313
backend/src/core/superset_client/_base.py
Normal file
313
backend/src/core/superset_client/_base.py
Normal file
@@ -0,0 +1,313 @@
|
||||
# [DEF:SupersetClientBase:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Base class for SupersetClient providing initialization, authentication, pagination, and import/export helpers.
|
||||
# @RELATION: DEPENDS_ON -> [ConfigModels]
|
||||
# @RELATION: DEPENDS_ON -> [APIClient]
|
||||
# @RELATION: DEPENDS_ON -> [SupersetAPIError]
|
||||
# @RELATION: DEPENDS_ON -> [get_filename_from_headers]
|
||||
|
||||
import json
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
||||
from requests import Response
|
||||
|
||||
from ..logger import logger as app_logger, belief_scope
|
||||
from ..utils.network import APIClient, SupersetAPIError
|
||||
from ..utils.fileio import get_filename_from_headers
|
||||
from ..config_models import Environment
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
|
||||
|
||||
# [DEF:SupersetClientBase:Class]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Base class providing Superset client initialization, auth, pagination, and import/export plumbing.
|
||||
# @RELATION: DEPENDS_ON -> [ConfigModels]
|
||||
# @RELATION: DEPENDS_ON -> [APIClient]
|
||||
# @RELATION: DEPENDS_ON -> [SupersetAPIError]
|
||||
class SupersetClientBase:
|
||||
# [DEF:SupersetClientInit:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент.
|
||||
# @RELATION: DEPENDS_ON -> [Environment]
|
||||
# @RELATION: DEPENDS_ON -> [APIClient]
|
||||
def __init__(self, env: Environment):
|
||||
with belief_scope("SupersetClientInit"):
|
||||
app_logger.reason(
|
||||
"Initializing Superset client for environment",
|
||||
extra={"environment": getattr(env, "id", None), "env_name": env.name},
|
||||
)
|
||||
self.env = env
|
||||
# Construct auth payload expected by Superset API
|
||||
auth_payload = {
|
||||
"username": env.username,
|
||||
"password": env.password,
|
||||
"provider": "db",
|
||||
"refresh": "true",
|
||||
}
|
||||
self.network = APIClient(
|
||||
config={"base_url": env.url, "auth": auth_payload},
|
||||
verify_ssl=env.verify_ssl,
|
||||
timeout=env.timeout,
|
||||
)
|
||||
self.delete_before_reimport: bool = False
|
||||
app_logger.reflect(
|
||||
"Superset client initialized",
|
||||
extra={"environment": getattr(self.env, "id", None)},
|
||||
)
|
||||
|
||||
# [/DEF:SupersetClientInit:Function]
|
||||
|
||||
# [DEF:SupersetClientAuthenticate:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Authenticates the client using the configured credentials.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def authenticate(self) -> Dict[str, str]:
|
||||
with belief_scope("SupersetClientAuthenticate"):
|
||||
app_logger.reason(
|
||||
"Authenticating Superset client",
|
||||
extra={"environment": getattr(self.env, "id", None)},
|
||||
)
|
||||
tokens = self.network.authenticate()
|
||||
app_logger.reflect(
|
||||
"Superset client authentication completed",
|
||||
extra={
|
||||
"environment": getattr(self.env, "id", None),
|
||||
"token_keys": sorted(tokens.keys()),
|
||||
},
|
||||
)
|
||||
return tokens
|
||||
# [/DEF:SupersetClientAuthenticate:Function]
|
||||
|
||||
@property
|
||||
# [DEF:SupersetClientHeaders:Function]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Возвращает базовые HTTP-заголовки, используемые сетевым клиентом.
|
||||
def headers(self) -> dict:
|
||||
with belief_scope("headers"):
|
||||
return self.network.headers
|
||||
|
||||
# [/DEF:SupersetClientHeaders:Function]
|
||||
|
||||
# --- Pagination helpers ---
|
||||
|
||||
# [DEF:SupersetClientValidateQueryParams:Function]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Ensures query parameters have default page and page_size.
|
||||
def _validate_query_params(self, query: Optional[Dict]) -> Dict:
|
||||
with belief_scope("_validate_query_params"):
|
||||
# Superset list endpoints commonly cap page_size at 100.
|
||||
# Using 100 avoids partial fetches when larger values are silently truncated.
|
||||
base_query = {"page": 0, "page_size": 100}
|
||||
return {**base_query, **(query or {})}
|
||||
|
||||
# [/DEF:SupersetClientValidateQueryParams:Function]
|
||||
|
||||
# [DEF:SupersetClientFetchTotalObjectCount:Function]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Fetches the total number of items for a given endpoint.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def _fetch_total_object_count(self, endpoint: str) -> int:
|
||||
with belief_scope("_fetch_total_object_count"):
|
||||
return self.network.fetch_paginated_count(
|
||||
endpoint=endpoint,
|
||||
query_params={"page": 0, "page_size": 1},
|
||||
count_field="count",
|
||||
)
|
||||
|
||||
# [/DEF:SupersetClientFetchTotalObjectCount:Function]
|
||||
|
||||
# [DEF:SupersetClientFetchAllPages:Function]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Iterates through all pages to collect all data items.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def _fetch_all_pages(self, endpoint: str, pagination_options: Dict) -> List[Dict]:
|
||||
with belief_scope("_fetch_all_pages"):
|
||||
return self.network.fetch_paginated_data(
|
||||
endpoint=endpoint, pagination_options=pagination_options
|
||||
)
|
||||
|
||||
# [/DEF:SupersetClientFetchAllPages:Function]
|
||||
|
||||
# --- Import/Export helpers ---
|
||||
|
||||
# [DEF:SupersetClientDoImport:Function]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Performs the actual multipart upload for import.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def _do_import(self, file_name: Union[str, Path]) -> Dict:
|
||||
with belief_scope("_do_import"):
|
||||
app_logger.debug(f"[_do_import][State] Uploading file: {file_name}")
|
||||
file_path = Path(file_name)
|
||||
if not file_path.exists():
|
||||
app_logger.error(
|
||||
f"[_do_import][Failure] File does not exist: {file_name}"
|
||||
)
|
||||
raise FileNotFoundError(f"File does not exist: {file_name}")
|
||||
|
||||
return self.network.upload_file(
|
||||
endpoint="/dashboard/import/",
|
||||
file_info={
|
||||
"file_obj": file_path,
|
||||
"file_name": file_path.name,
|
||||
"form_field": "formData",
|
||||
},
|
||||
extra_data={"overwrite": "true"},
|
||||
timeout=self.env.timeout * 2,
|
||||
)
|
||||
|
||||
# [/DEF:SupersetClientDoImport:Function]
|
||||
|
||||
# [DEF:SupersetClientValidateExportResponse:Function]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Validates that the export response is a non-empty ZIP archive.
|
||||
def _validate_export_response(self, response: Response, dashboard_id: int) -> None:
|
||||
with belief_scope("_validate_export_response"):
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if "application/zip" not in content_type:
|
||||
raise SupersetAPIError(
|
||||
f"Получен не ZIP-архив (Content-Type: {content_type})"
|
||||
)
|
||||
if not response.content:
|
||||
raise SupersetAPIError("Получены пустые данные при экспорте")
|
||||
|
||||
# [/DEF:SupersetClientValidateExportResponse:Function]
|
||||
|
||||
# [DEF:SupersetClientResolveExportFilename:Function]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Determines the filename for an exported dashboard.
|
||||
def _resolve_export_filename(self, response: Response, dashboard_id: int) -> str:
|
||||
with belief_scope("_resolve_export_filename"):
|
||||
filename = get_filename_from_headers(dict(response.headers))
|
||||
if not filename:
|
||||
from datetime import datetime
|
||||
|
||||
timestamp = datetime.now().strftime("%Y%m%dT%H%M%S")
|
||||
filename = f"dashboard_export_{dashboard_id}_{timestamp}.zip"
|
||||
app_logger.warning(
|
||||
"[_resolve_export_filename][Warning] Generated filename: %s",
|
||||
filename,
|
||||
)
|
||||
return filename
|
||||
|
||||
# [/DEF:SupersetClientResolveExportFilename:Function]
|
||||
|
||||
# [DEF:SupersetClientValidateImportFile:Function]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Validates that the file to be imported is a valid ZIP with metadata.yaml.
|
||||
def _validate_import_file(self, zip_path: Union[str, Path]) -> None:
|
||||
with belief_scope("_validate_import_file"):
|
||||
path = Path(zip_path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Файл {zip_path} не существует")
|
||||
if not zipfile.is_zipfile(path):
|
||||
raise SupersetAPIError(f"Файл {zip_path} не является ZIP-архивом")
|
||||
with zipfile.ZipFile(path, "r") as zf:
|
||||
if not any(n.endswith("metadata.yaml") for n in zf.namelist()):
|
||||
raise SupersetAPIError(
|
||||
f"Архив {zip_path} не содержит 'metadata.yaml'"
|
||||
)
|
||||
|
||||
# [/DEF:SupersetClientValidateImportFile:Function]
|
||||
|
||||
# [DEF:SupersetClientResolveTargetIdForDelete:Function]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Resolves a dashboard ID from either an ID or a slug.
|
||||
# @RELATION: CALLS -> [SupersetClientGetDashboards]
|
||||
def _resolve_target_id_for_delete(
|
||||
self, dash_id: Optional[int], dash_slug: Optional[str]
|
||||
) -> Optional[int]:
|
||||
with belief_scope("_resolve_target_id_for_delete"):
|
||||
if dash_id is not None:
|
||||
return dash_id
|
||||
if dash_slug is not None:
|
||||
app_logger.debug(
|
||||
"[_resolve_target_id_for_delete][State] Resolving ID by slug '%s'.",
|
||||
dash_slug,
|
||||
)
|
||||
try:
|
||||
_, candidates = self.get_dashboards(
|
||||
query={
|
||||
"filters": [{"col": "slug", "op": "eq", "value": dash_slug}]
|
||||
}
|
||||
)
|
||||
if candidates:
|
||||
target_id = candidates[0]["id"]
|
||||
app_logger.debug(
|
||||
"[_resolve_target_id_for_delete][Success] Resolved slug to ID %s.",
|
||||
target_id,
|
||||
)
|
||||
return target_id
|
||||
except Exception as e:
|
||||
app_logger.warning(
|
||||
"[_resolve_target_id_for_delete][Warning] Could not resolve slug '%s' to ID: %s",
|
||||
dash_slug,
|
||||
e,
|
||||
)
|
||||
return None
|
||||
|
||||
# [/DEF:SupersetClientResolveTargetIdForDelete:Function]
|
||||
|
||||
# [DEF:SupersetClientGetAllResources:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Fetches all resources of a given type with id, uuid, and name columns.
|
||||
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
|
||||
def get_all_resources(
|
||||
self, resource_type: str, since_dttm: Optional["datetime"] = None
|
||||
) -> List[Dict]:
|
||||
with belief_scope(
|
||||
"SupersetClient.get_all_resources",
|
||||
f"type={resource_type}, since={since_dttm}",
|
||||
):
|
||||
column_map = {
|
||||
"chart": {
|
||||
"endpoint": "/chart/",
|
||||
"columns": ["id", "uuid", "slice_name"],
|
||||
},
|
||||
"dataset": {
|
||||
"endpoint": "/dataset/",
|
||||
"columns": ["id", "uuid", "table_name"],
|
||||
},
|
||||
"dashboard": {
|
||||
"endpoint": "/dashboard/",
|
||||
"columns": ["id", "uuid", "slug", "dashboard_title"],
|
||||
},
|
||||
}
|
||||
config = column_map.get(resource_type)
|
||||
if not config:
|
||||
app_logger.warning(
|
||||
"[get_all_resources][Warning] Unknown resource type: %s",
|
||||
resource_type,
|
||||
)
|
||||
return []
|
||||
|
||||
query = {"columns": config["columns"]}
|
||||
|
||||
if since_dttm:
|
||||
import math
|
||||
|
||||
# Use int milliseconds to be safe
|
||||
timestamp_ms = math.floor(since_dttm.timestamp() * 1000)
|
||||
|
||||
query["filters"] = [
|
||||
{"col": "changed_on_dttm", "opr": "gt", "value": timestamp_ms}
|
||||
]
|
||||
|
||||
validated = self._validate_query_params(query)
|
||||
data = self._fetch_all_pages(
|
||||
endpoint=config["endpoint"],
|
||||
pagination_options={"base_query": validated, "results_field": "result"},
|
||||
)
|
||||
app_logger.info(
|
||||
"[get_all_resources][Exit] Fetched %d %s resources.",
|
||||
len(data),
|
||||
resource_type,
|
||||
)
|
||||
return data
|
||||
|
||||
# [/DEF:SupersetClientGetAllResources:Function]
|
||||
|
||||
|
||||
# [/DEF:SupersetClientBase:Class]
|
||||
# [/DEF:SupersetClientBase:Module]
|
||||
88
backend/src/core/superset_client/_charts.py
Normal file
88
backend/src/core/superset_client/_charts.py
Normal file
@@ -0,0 +1,88 @@
|
||||
# [DEF:SupersetChartsMixin:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Chart domain mixin for SupersetClient — list, get, extract IDs from layout.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional, Tuple, cast
|
||||
|
||||
from ..logger import logger as app_logger, belief_scope
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
|
||||
|
||||
# [DEF:SupersetChartsMixin:Class]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Mixin providing all chart-related Superset API operations.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
|
||||
class SupersetChartsMixin:
|
||||
# [DEF:SupersetClientGetChart:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Fetches a single chart by ID.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def get_chart(self, chart_id: int) -> Dict:
|
||||
with belief_scope("SupersetClient.get_chart", f"id={chart_id}"):
|
||||
response = self.network.request(method="GET", endpoint=f"/chart/{chart_id}")
|
||||
return cast(Dict, response)
|
||||
|
||||
# [/DEF:SupersetClientGetChart:Function]
|
||||
|
||||
# [DEF:SupersetClientGetCharts:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Fetches all charts with pagination support.
|
||||
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
|
||||
def get_charts(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
|
||||
with belief_scope("get_charts"):
|
||||
validated_query = self._validate_query_params(query or {})
|
||||
if "columns" not in validated_query:
|
||||
validated_query["columns"] = ["id", "uuid", "slice_name", "viz_type"]
|
||||
|
||||
paginated_data = self._fetch_all_pages(
|
||||
endpoint="/chart/",
|
||||
pagination_options={
|
||||
"base_query": validated_query,
|
||||
"results_field": "result",
|
||||
},
|
||||
)
|
||||
return len(paginated_data), paginated_data
|
||||
|
||||
# [/DEF:SupersetClientGetCharts:Function]
|
||||
|
||||
# [DEF:SupersetClientExtractChartIdsFromLayout:Function]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Traverses dashboard layout metadata and extracts chart IDs from common keys.
|
||||
def _extract_chart_ids_from_layout(
|
||||
self, payload: Any
|
||||
) -> set:
|
||||
with belief_scope("_extract_chart_ids_from_layout"):
|
||||
found = set()
|
||||
|
||||
def walk(node):
|
||||
if isinstance(node, dict):
|
||||
for key, value in node.items():
|
||||
if key in ("chartId", "chart_id", "slice_id", "sliceId"):
|
||||
try:
|
||||
found.add(int(value))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if key == "id" and isinstance(value, str):
|
||||
match = re.match(r"^CHART-(\d+)$", value)
|
||||
if match:
|
||||
try:
|
||||
found.add(int(match.group(1)))
|
||||
except ValueError:
|
||||
pass
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
walk(payload)
|
||||
return found
|
||||
|
||||
# [/DEF:SupersetClientExtractChartIdsFromLayout:Function]
|
||||
|
||||
|
||||
# [/DEF:SupersetChartsMixin:Class]
|
||||
# [/DEF:SupersetChartsMixin:Module]
|
||||
2
backend/src/core/superset_client/_dashboards.py
Normal file
2
backend/src/core/superset_client/_dashboards.py
Normal file
@@ -0,0 +1,2 @@
|
||||
# This file has been decomposed into _dashboards_list.py, _dashboards_filters.py, _dashboards_crud.py
|
||||
# See __init__.py for the composed SupersetClient class.
|
||||
374
backend/src/core/superset_client/_dashboards_crud.py
Normal file
374
backend/src/core/superset_client/_dashboards_crud.py
Normal file
@@ -0,0 +1,374 @@
|
||||
# [DEF:SupersetDashboardsCrudMixin:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Dashboard CRUD mixin for SupersetClient — detail, export, import, delete.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
|
||||
# @RELATION: DEPENDS_ON -> [SupersetDashboardsFiltersMixin]
|
||||
# @RELATION: DEPENDS_ON -> [SupersetChartsMixin]
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
||||
from requests import Response
|
||||
|
||||
from ..logger import logger as app_logger, belief_scope
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
|
||||
|
||||
# [DEF:SupersetDashboardsCrudMixin:Class]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Mixin providing dashboard detail resolution, export, import, and delete operations.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
|
||||
# @RELATION: DEPENDS_ON -> [SupersetDashboardsFiltersMixin]
|
||||
# @RELATION: DEPENDS_ON -> [SupersetChartsMixin]
|
||||
class SupersetDashboardsCrudMixin:
|
||||
# [DEF:SupersetClientGetDashboardDetail:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Fetches detailed dashboard information including related charts and datasets.
|
||||
# @RELATION: CALLS -> [SupersetClientGetDashboard]
|
||||
# @RELATION: CALLS -> [SupersetClientGetChart]
|
||||
def get_dashboard_detail(self, dashboard_ref: Union[int, str]) -> Dict:
|
||||
with belief_scope(
|
||||
"SupersetClient.get_dashboard_detail", f"ref={dashboard_ref}"
|
||||
):
|
||||
dashboard_response = self.get_dashboard(dashboard_ref)
|
||||
dashboard_data = dashboard_response.get("result", dashboard_response)
|
||||
|
||||
charts: List[Dict] = []
|
||||
datasets: List[Dict] = []
|
||||
|
||||
# [DEF:extract_dataset_id_from_form_data:Function]
|
||||
def extract_dataset_id_from_form_data(
|
||||
form_data: Optional[Dict],
|
||||
) -> Optional[int]:
|
||||
if not isinstance(form_data, dict):
|
||||
return None
|
||||
datasource = form_data.get("datasource")
|
||||
if isinstance(datasource, str):
|
||||
matched = re.match(r"^(\d+)__", datasource)
|
||||
if matched:
|
||||
try:
|
||||
return int(matched.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
if isinstance(datasource, dict):
|
||||
ds_id = datasource.get("id")
|
||||
try:
|
||||
return int(ds_id) if ds_id is not None else None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
ds_id = form_data.get("datasource_id")
|
||||
try:
|
||||
return int(ds_id) if ds_id is not None else None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
# [/DEF:extract_dataset_id_from_form_data:Function]
|
||||
|
||||
try:
|
||||
charts_response = self.network.request(
|
||||
method="GET", endpoint=f"/dashboard/{dashboard_ref}/charts"
|
||||
)
|
||||
charts_payload = (
|
||||
charts_response.get("result", [])
|
||||
if isinstance(charts_response, dict)
|
||||
else []
|
||||
)
|
||||
for chart_obj in charts_payload:
|
||||
if not isinstance(chart_obj, dict):
|
||||
continue
|
||||
chart_id = chart_obj.get("id")
|
||||
if chart_id is None:
|
||||
continue
|
||||
form_data = chart_obj.get("form_data")
|
||||
if isinstance(form_data, str):
|
||||
try:
|
||||
form_data = json.loads(form_data)
|
||||
except Exception:
|
||||
form_data = {}
|
||||
dataset_id = extract_dataset_id_from_form_data(
|
||||
form_data
|
||||
) or chart_obj.get("datasource_id")
|
||||
charts.append({
|
||||
"id": int(chart_id),
|
||||
"title": chart_obj.get("slice_name")
|
||||
or chart_obj.get("name") or f"Chart {chart_id}",
|
||||
"viz_type": (
|
||||
form_data.get("viz_type")
|
||||
if isinstance(form_data, dict) else None
|
||||
),
|
||||
"dataset_id": int(dataset_id) if dataset_id is not None else None,
|
||||
"last_modified": chart_obj.get("changed_on"),
|
||||
"overview": chart_obj.get("description")
|
||||
or (form_data.get("viz_type") if isinstance(form_data, dict) else None)
|
||||
or "Chart",
|
||||
})
|
||||
except Exception as e:
|
||||
app_logger.warning(
|
||||
"[get_dashboard_detail][Warning] Failed to fetch dashboard charts: %s", e,
|
||||
)
|
||||
|
||||
try:
|
||||
datasets_response = self.network.request(
|
||||
method="GET", endpoint=f"/dashboard/{dashboard_ref}/datasets"
|
||||
)
|
||||
datasets_payload = (
|
||||
datasets_response.get("result", [])
|
||||
if isinstance(datasets_response, dict)
|
||||
else []
|
||||
)
|
||||
for dataset_obj in datasets_payload:
|
||||
if not isinstance(dataset_obj, dict):
|
||||
continue
|
||||
dataset_id = dataset_obj.get("id")
|
||||
if dataset_id is None:
|
||||
continue
|
||||
db_payload = dataset_obj.get("database")
|
||||
db_name = (
|
||||
db_payload.get("database_name")
|
||||
if isinstance(db_payload, dict) else None
|
||||
)
|
||||
table_name = (
|
||||
dataset_obj.get("table_name")
|
||||
or dataset_obj.get("datasource_name")
|
||||
or dataset_obj.get("name") or f"Dataset {dataset_id}"
|
||||
)
|
||||
schema = dataset_obj.get("schema")
|
||||
fq_name = f"{schema}.{table_name}" if schema else table_name
|
||||
datasets.append({
|
||||
"id": int(dataset_id),
|
||||
"table_name": table_name,
|
||||
"schema": schema,
|
||||
"database": db_name or dataset_obj.get("database_name") or "Unknown",
|
||||
"last_modified": dataset_obj.get("changed_on"),
|
||||
"overview": fq_name,
|
||||
})
|
||||
except Exception as e:
|
||||
app_logger.warning(
|
||||
"[get_dashboard_detail][Warning] Failed to fetch dashboard datasets: %s", e,
|
||||
)
|
||||
|
||||
# Fallback: derive chart IDs from layout metadata
|
||||
if not charts:
|
||||
raw_position_json = dashboard_data.get("position_json")
|
||||
chart_ids_from_position = set()
|
||||
if isinstance(raw_position_json, str) and raw_position_json:
|
||||
try:
|
||||
parsed_position = json.loads(raw_position_json)
|
||||
chart_ids_from_position.update(
|
||||
self._extract_chart_ids_from_layout(parsed_position)
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
elif isinstance(raw_position_json, dict):
|
||||
chart_ids_from_position.update(
|
||||
self._extract_chart_ids_from_layout(raw_position_json)
|
||||
)
|
||||
|
||||
raw_json_metadata = dashboard_data.get("json_metadata")
|
||||
if isinstance(raw_json_metadata, str) and raw_json_metadata:
|
||||
try:
|
||||
parsed_metadata = json.loads(raw_json_metadata)
|
||||
chart_ids_from_position.update(
|
||||
self._extract_chart_ids_from_layout(parsed_metadata)
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
elif isinstance(raw_json_metadata, dict):
|
||||
chart_ids_from_position.update(
|
||||
self._extract_chart_ids_from_layout(raw_json_metadata)
|
||||
)
|
||||
|
||||
app_logger.info(
|
||||
"[get_dashboard_detail][State] Extracted %s fallback chart IDs from layout (dashboard_id=%s)",
|
||||
len(chart_ids_from_position), dashboard_ref,
|
||||
)
|
||||
|
||||
for chart_id in sorted(chart_ids_from_position):
|
||||
try:
|
||||
chart_response = self.get_chart(int(chart_id))
|
||||
chart_data = chart_response.get("result", chart_response)
|
||||
charts.append({
|
||||
"id": int(chart_id),
|
||||
"title": chart_data.get("slice_name")
|
||||
or chart_data.get("name") or f"Chart {chart_id}",
|
||||
"viz_type": chart_data.get("viz_type"),
|
||||
"dataset_id": chart_data.get("datasource_id"),
|
||||
"last_modified": chart_data.get("changed_on"),
|
||||
"overview": chart_data.get("description")
|
||||
or chart_data.get("viz_type") or "Chart",
|
||||
})
|
||||
except Exception as e:
|
||||
app_logger.warning(
|
||||
"[get_dashboard_detail][Warning] Failed to resolve fallback chart %s: %s",
|
||||
chart_id, e,
|
||||
)
|
||||
|
||||
# Backfill datasets from chart datasource IDs.
|
||||
dataset_ids_from_charts = {
|
||||
c.get("dataset_id") for c in charts if c.get("dataset_id") is not None
|
||||
}
|
||||
known_dataset_ids = {
|
||||
d.get("id") for d in datasets if d.get("id") is not None
|
||||
}
|
||||
missing_dataset_ids: List[int] = []
|
||||
for raw_dataset_id in dataset_ids_from_charts:
|
||||
if raw_dataset_id is None or raw_dataset_id in known_dataset_ids:
|
||||
continue
|
||||
try:
|
||||
missing_dataset_ids.append(int(raw_dataset_id))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
|
||||
for dataset_id in missing_dataset_ids:
|
||||
try:
|
||||
dataset_response = self.get_dataset(int(dataset_id))
|
||||
dataset_data = dataset_response.get("result", dataset_response)
|
||||
db_payload = dataset_data.get("database")
|
||||
db_name = (
|
||||
db_payload.get("database_name")
|
||||
if isinstance(db_payload, dict) else None
|
||||
)
|
||||
table_name = (
|
||||
dataset_data.get("table_name") or f"Dataset {dataset_id}"
|
||||
)
|
||||
schema = dataset_data.get("schema")
|
||||
fq_name = f"{schema}.{table_name}" if schema else table_name
|
||||
datasets.append({
|
||||
"id": int(dataset_id),
|
||||
"table_name": table_name,
|
||||
"schema": schema,
|
||||
"database": db_name or "Unknown",
|
||||
"last_modified": dataset_data.get("changed_on_utc")
|
||||
or dataset_data.get("changed_on"),
|
||||
"overview": fq_name,
|
||||
})
|
||||
except Exception as e:
|
||||
app_logger.warning(
|
||||
"[get_dashboard_detail][Warning] Failed to resolve dataset %s: %s",
|
||||
dataset_id, e,
|
||||
)
|
||||
|
||||
unique_charts = {chart["id"]: chart for chart in charts}
|
||||
unique_datasets = {dataset["id"]: dataset for dataset in datasets}
|
||||
|
||||
resolved_dashboard_id = dashboard_data.get("id", dashboard_ref)
|
||||
return {
|
||||
"id": resolved_dashboard_id,
|
||||
"title": dashboard_data.get("dashboard_title")
|
||||
or dashboard_data.get("title") or f"Dashboard {resolved_dashboard_id}",
|
||||
"slug": dashboard_data.get("slug"),
|
||||
"url": dashboard_data.get("url"),
|
||||
"description": dashboard_data.get("description") or "",
|
||||
"last_modified": dashboard_data.get("changed_on_utc")
|
||||
or dashboard_data.get("changed_on"),
|
||||
"published": dashboard_data.get("published"),
|
||||
"charts": list(unique_charts.values()),
|
||||
"datasets": list(unique_datasets.values()),
|
||||
"chart_count": len(unique_charts),
|
||||
"dataset_count": len(unique_datasets),
|
||||
}
|
||||
|
||||
# [/DEF:SupersetClientGetDashboardDetail:Function]
|
||||
|
||||
# [DEF:SupersetClientExportDashboard:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Экспортирует дашборд в виде ZIP-архива.
|
||||
# @SIDE_EFFECT: Performs network I/O to download archive.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def export_dashboard(self, dashboard_id: int) -> Tuple[bytes, str]:
|
||||
with belief_scope("export_dashboard"):
|
||||
app_logger.info(
|
||||
"[export_dashboard][Enter] Exporting dashboard %s.", dashboard_id
|
||||
)
|
||||
response = self.network.request(
|
||||
method="GET",
|
||||
endpoint="/dashboard/export/",
|
||||
params={"q": json.dumps([dashboard_id])},
|
||||
stream=True,
|
||||
raw_response=True,
|
||||
)
|
||||
response = cast(Response, response)
|
||||
self._validate_export_response(response, dashboard_id)
|
||||
filename = self._resolve_export_filename(response, dashboard_id)
|
||||
app_logger.info(
|
||||
"[export_dashboard][Exit] Exported dashboard %s to %s.",
|
||||
dashboard_id, filename,
|
||||
)
|
||||
return response.content, filename
|
||||
|
||||
# [/DEF:SupersetClientExportDashboard:Function]
|
||||
|
||||
# [DEF:SupersetClientImportDashboard:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Импортирует дашборд из ZIP-файла.
|
||||
# @SIDE_EFFECT: Performs network I/O to upload archive.
|
||||
# @RELATION: CALLS -> [SupersetClientDoImport]
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def import_dashboard(
|
||||
self,
|
||||
file_name: Union[str, Path],
|
||||
dash_id: Optional[int] = None,
|
||||
dash_slug: Optional[str] = None,
|
||||
) -> Dict:
|
||||
with belief_scope("import_dashboard"):
|
||||
if file_name is None:
|
||||
raise ValueError("file_name cannot be None")
|
||||
file_path = str(file_name)
|
||||
self._validate_import_file(file_path)
|
||||
try:
|
||||
return self._do_import(file_path)
|
||||
except Exception as exc:
|
||||
app_logger.error(
|
||||
"[import_dashboard][Failure] First import attempt failed: %s",
|
||||
exc, exc_info=True,
|
||||
)
|
||||
if not self.delete_before_reimport:
|
||||
raise
|
||||
|
||||
target_id = self._resolve_target_id_for_delete(dash_id, dash_slug)
|
||||
if target_id is None:
|
||||
app_logger.error(
|
||||
"[import_dashboard][Failure] No ID available for delete-retry."
|
||||
)
|
||||
raise
|
||||
|
||||
self.delete_dashboard(target_id)
|
||||
app_logger.info(
|
||||
"[import_dashboard][State] Deleted dashboard ID %s, retrying import.",
|
||||
target_id,
|
||||
)
|
||||
return self._do_import(file_path)
|
||||
|
||||
# [/DEF:SupersetClientImportDashboard:Function]
|
||||
|
||||
# [DEF:SupersetClientDeleteDashboard:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Удаляет дашборд по его ID или slug.
|
||||
# @SIDE_EFFECT: Deletes resource from upstream Superset environment.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def delete_dashboard(self, dashboard_id: Union[int, str]) -> None:
|
||||
with belief_scope("delete_dashboard"):
|
||||
app_logger.info(
|
||||
"[delete_dashboard][Enter] Deleting dashboard %s.", dashboard_id
|
||||
)
|
||||
response = self.network.request(
|
||||
method="DELETE", endpoint=f"/dashboard/{dashboard_id}"
|
||||
)
|
||||
response = cast(Dict, response)
|
||||
if response.get("result", True) is not False:
|
||||
app_logger.info(
|
||||
"[delete_dashboard][Success] Dashboard %s deleted.", dashboard_id
|
||||
)
|
||||
else:
|
||||
app_logger.warning(
|
||||
"[delete_dashboard][Warning] Unexpected response while deleting %s: %s",
|
||||
dashboard_id, response,
|
||||
)
|
||||
|
||||
# [/DEF:SupersetClientDeleteDashboard:Function]
|
||||
|
||||
|
||||
# [/DEF:SupersetDashboardsCrudMixin:Class]
|
||||
# [/DEF:SupersetDashboardsCrudMixin:Module]
|
||||
266
backend/src/core/superset_client/_dashboards_filters.py
Normal file
266
backend/src/core/superset_client/_dashboards_filters.py
Normal file
@@ -0,0 +1,266 @@
|
||||
# [DEF:SupersetDashboardsFiltersMixin:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Dashboard native filter extraction mixin for SupersetClient.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, Optional, Union, cast
|
||||
|
||||
from ..logger import logger as app_logger, belief_scope
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
|
||||
|
||||
# [DEF:SupersetDashboardsFiltersMixin:Class]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Mixin providing dashboard native filter extraction from permalink and URL state.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
|
||||
class SupersetDashboardsFiltersMixin:
|
||||
# [DEF:SupersetClientGetDashboard:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Fetches a single dashboard by ID or slug.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def get_dashboard(self, dashboard_ref: Union[int, str]) -> Dict:
|
||||
with belief_scope("SupersetClient.get_dashboard", f"ref={dashboard_ref}"):
|
||||
response = self.network.request(
|
||||
method="GET", endpoint=f"/dashboard/{dashboard_ref}"
|
||||
)
|
||||
return cast(Dict, response)
|
||||
|
||||
# [/DEF:SupersetClientGetDashboard:Function]
|
||||
|
||||
# [DEF:SupersetClientGetDashboardPermalinkState:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Fetches stored dashboard permalink state by permalink key.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def get_dashboard_permalink_state(self, permalink_key: str) -> Dict:
|
||||
with belief_scope(
|
||||
"SupersetClient.get_dashboard_permalink_state", f"key={permalink_key}"
|
||||
):
|
||||
response = self.network.request(
|
||||
method="GET", endpoint=f"/dashboard/permalink/{permalink_key}"
|
||||
)
|
||||
return cast(Dict, response)
|
||||
|
||||
# [/DEF:SupersetClientGetDashboardPermalinkState:Function]
|
||||
|
||||
# [DEF:SupersetClientGetNativeFilterState:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Fetches stored native filter state by filter state key.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def get_native_filter_state(
|
||||
self, dashboard_id: Union[int, str], filter_state_key: str
|
||||
) -> Dict:
|
||||
with belief_scope(
|
||||
"SupersetClient.get_native_filter_state",
|
||||
f"dashboard={dashboard_id}, key={filter_state_key}",
|
||||
):
|
||||
response = self.network.request(
|
||||
method="GET",
|
||||
endpoint=f"/dashboard/{dashboard_id}/filter_state/{filter_state_key}",
|
||||
)
|
||||
return cast(Dict, response)
|
||||
|
||||
# [/DEF:SupersetClientGetNativeFilterState:Function]
|
||||
|
||||
# [DEF:SupersetClientExtractNativeFiltersFromPermalink:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Extract native filters dataMask from a permalink key.
|
||||
# @RELATION: CALLS -> [SupersetClientGetDashboardPermalinkState]
|
||||
def extract_native_filters_from_permalink(self, permalink_key: str) -> Dict:
|
||||
with belief_scope(
|
||||
"SupersetClient.extract_native_filters_from_permalink",
|
||||
f"key={permalink_key}",
|
||||
):
|
||||
permalink_response = self.get_dashboard_permalink_state(permalink_key)
|
||||
|
||||
result = permalink_response.get("result", permalink_response)
|
||||
state = result.get("state", result)
|
||||
data_mask = state.get("dataMask", {})
|
||||
|
||||
extracted_filters = {}
|
||||
for filter_id, filter_data in data_mask.items():
|
||||
if not isinstance(filter_data, dict):
|
||||
continue
|
||||
extracted_filters[filter_id] = {
|
||||
"extraFormData": filter_data.get("extraFormData", {}),
|
||||
"filterState": filter_data.get("filterState", {}),
|
||||
"ownState": filter_data.get("ownState", {}),
|
||||
}
|
||||
|
||||
return {
|
||||
"dataMask": extracted_filters,
|
||||
"activeTabs": state.get("activeTabs", []),
|
||||
"anchor": state.get("anchor"),
|
||||
"chartStates": state.get("chartStates", {}),
|
||||
"permalink_key": permalink_key,
|
||||
}
|
||||
|
||||
# [/DEF:SupersetClientExtractNativeFiltersFromPermalink:Function]
|
||||
|
||||
# [DEF:SupersetClientExtractNativeFiltersFromKey:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Extract native filters from a native_filters_key URL parameter.
|
||||
# @RELATION: CALLS -> [SupersetClientGetNativeFilterState]
|
||||
def extract_native_filters_from_key(
|
||||
self, dashboard_id: Union[int, str], filter_state_key: str
|
||||
) -> Dict:
|
||||
with belief_scope(
|
||||
"SupersetClient.extract_native_filters_from_key",
|
||||
f"dashboard={dashboard_id}, key={filter_state_key}",
|
||||
):
|
||||
filter_response = self.get_native_filter_state(
|
||||
dashboard_id, filter_state_key
|
||||
)
|
||||
|
||||
result = filter_response.get("result", filter_response)
|
||||
value = result.get("value")
|
||||
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
parsed_value = json.loads(value)
|
||||
except json.JSONDecodeError as e:
|
||||
app_logger.warning(
|
||||
"[extract_native_filters_from_key][Warning] Failed to parse filter state JSON: %s",
|
||||
e,
|
||||
)
|
||||
parsed_value = {}
|
||||
elif isinstance(value, dict):
|
||||
parsed_value = value
|
||||
else:
|
||||
parsed_value = {}
|
||||
|
||||
extracted_filters = {}
|
||||
|
||||
if "id" in parsed_value and "extraFormData" in parsed_value:
|
||||
filter_id = parsed_value.get("id", filter_state_key)
|
||||
extracted_filters[filter_id] = {
|
||||
"extraFormData": parsed_value.get("extraFormData", {}),
|
||||
"filterState": parsed_value.get("filterState", {}),
|
||||
"ownState": parsed_value.get("ownState", {}),
|
||||
}
|
||||
else:
|
||||
for filter_id, filter_data in parsed_value.items():
|
||||
if not isinstance(filter_data, dict):
|
||||
continue
|
||||
extracted_filters[filter_id] = {
|
||||
"extraFormData": filter_data.get("extraFormData", {}),
|
||||
"filterState": filter_data.get("filterState", {}),
|
||||
"ownState": filter_data.get("ownState", {}),
|
||||
}
|
||||
|
||||
return {
|
||||
"dataMask": extracted_filters,
|
||||
"dashboard_id": dashboard_id,
|
||||
"filter_state_key": filter_state_key,
|
||||
}
|
||||
|
||||
# [/DEF:SupersetClientExtractNativeFiltersFromKey:Function]
|
||||
|
||||
# [DEF:SupersetClientParseDashboardUrlForFilters:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state if present.
|
||||
# @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromPermalink]
|
||||
# @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromKey]
|
||||
def parse_dashboard_url_for_filters(self, url: str) -> Dict:
|
||||
with belief_scope(
|
||||
"SupersetClient.parse_dashboard_url_for_filters", f"url={url}"
|
||||
):
|
||||
import urllib.parse
|
||||
|
||||
parsed_url = urllib.parse.urlparse(url)
|
||||
query_params = urllib.parse.parse_qs(parsed_url.query)
|
||||
path_parts = parsed_url.path.rstrip("/").split("/")
|
||||
|
||||
result = {
|
||||
"url": url,
|
||||
"dashboard_id": None,
|
||||
"filter_type": None,
|
||||
"filters": {},
|
||||
}
|
||||
|
||||
# Check for permalink URL: /dashboard/p/{key}/ or /superset/dashboard/p/{key}/
|
||||
if "p" in path_parts:
|
||||
try:
|
||||
p_index = path_parts.index("p")
|
||||
if p_index + 1 < len(path_parts):
|
||||
permalink_key = path_parts[p_index + 1]
|
||||
filter_data = self.extract_native_filters_from_permalink(
|
||||
permalink_key
|
||||
)
|
||||
result["filter_type"] = "permalink"
|
||||
result["filters"] = filter_data
|
||||
return result
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Check for native_filters_key in query params
|
||||
native_filters_key = query_params.get("native_filters_key", [None])[0]
|
||||
if native_filters_key:
|
||||
dashboard_ref = None
|
||||
if "dashboard" in path_parts:
|
||||
try:
|
||||
dash_index = path_parts.index("dashboard")
|
||||
if dash_index + 1 < len(path_parts):
|
||||
potential_id = path_parts[dash_index + 1]
|
||||
if potential_id not in ("p", "list", "new"):
|
||||
dashboard_ref = potential_id
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if dashboard_ref:
|
||||
resolved_id = None
|
||||
try:
|
||||
resolved_id = int(dashboard_ref)
|
||||
except (ValueError, TypeError):
|
||||
try:
|
||||
dash_resp = self.get_dashboard(dashboard_ref)
|
||||
dash_data = (
|
||||
dash_resp.get("result", dash_resp)
|
||||
if isinstance(dash_resp, dict)
|
||||
else {}
|
||||
)
|
||||
raw_id = dash_data.get("id")
|
||||
if raw_id is not None:
|
||||
resolved_id = int(raw_id)
|
||||
except Exception as e:
|
||||
app_logger.warning(
|
||||
"[parse_dashboard_url_for_filters][Warning] Failed to resolve dashboard slug '%s' to ID: %s",
|
||||
dashboard_ref,
|
||||
e,
|
||||
)
|
||||
|
||||
if resolved_id is not None:
|
||||
filter_data = self.extract_native_filters_from_key(
|
||||
resolved_id, native_filters_key
|
||||
)
|
||||
result["filter_type"] = "native_filters_key"
|
||||
result["dashboard_id"] = resolved_id
|
||||
result["filters"] = filter_data
|
||||
return result
|
||||
else:
|
||||
app_logger.warning(
|
||||
"[parse_dashboard_url_for_filters][Warning] Could not resolve dashboard_id from URL for native_filters_key"
|
||||
)
|
||||
|
||||
# Check for native_filters in query params (direct filter values)
|
||||
native_filters = query_params.get("native_filters", [None])[0]
|
||||
if native_filters:
|
||||
try:
|
||||
parsed_filters = json.loads(native_filters)
|
||||
result["filter_type"] = "native_filters"
|
||||
result["filters"] = {"dataMask": parsed_filters}
|
||||
return result
|
||||
except json.JSONDecodeError as e:
|
||||
app_logger.warning(
|
||||
"[parse_dashboard_url_for_filters][Warning] Failed to parse native_filters JSON: %s",
|
||||
e,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
# [/DEF:SupersetClientParseDashboardUrlForFilters:Function]
|
||||
|
||||
|
||||
# [/DEF:SupersetDashboardsFiltersMixin:Class]
|
||||
# [/DEF:SupersetDashboardsFiltersMixin:Module]
|
||||
205
backend/src/core/superset_client/_dashboards_list.py
Normal file
205
backend/src/core/superset_client/_dashboards_list.py
Normal file
@@ -0,0 +1,205 @@
|
||||
# [DEF:SupersetDashboardsListMixin:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Dashboard listing mixin for SupersetClient — paginated list, summary projection.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
|
||||
# @RELATION: DEPENDS_ON -> [SupersetUserProjectionMixin]
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional, Tuple, cast
|
||||
|
||||
from ..logger import logger as app_logger, belief_scope
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
|
||||
|
||||
# [DEF:SupersetDashboardsListMixin:Class]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Mixin providing dashboard listing and summary projection operations.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
|
||||
# @RELATION: DEPENDS_ON -> [SupersetUserProjectionMixin]
|
||||
class SupersetDashboardsListMixin:
|
||||
# [DEF:SupersetClientGetDashboards:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Получает полный список дашбордов, автоматически обрабатывая пагинацию.
|
||||
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
|
||||
def get_dashboards(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
|
||||
with belief_scope("get_dashboards"):
|
||||
app_logger.info("[get_dashboards][Enter] Fetching dashboards.")
|
||||
validated_query = self._validate_query_params(query or {})
|
||||
if "columns" not in validated_query:
|
||||
validated_query["columns"] = [
|
||||
"slug", "id", "url", "changed_on_utc", "dashboard_title",
|
||||
"published", "created_by", "changed_by", "changed_by_name", "owners",
|
||||
]
|
||||
|
||||
paginated_data = self._fetch_all_pages(
|
||||
endpoint="/dashboard/",
|
||||
pagination_options={
|
||||
"base_query": validated_query,
|
||||
"results_field": "result",
|
||||
},
|
||||
)
|
||||
total_count = len(paginated_data)
|
||||
app_logger.info("[get_dashboards][Exit] Found %d dashboards.", total_count)
|
||||
return total_count, paginated_data
|
||||
|
||||
# [/DEF:SupersetClientGetDashboards:Function]
|
||||
|
||||
# [DEF:SupersetClientGetDashboardsPage:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Fetches a single dashboards page from Superset without iterating all pages.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def get_dashboards_page(
|
||||
self, query: Optional[Dict] = None
|
||||
) -> Tuple[int, List[Dict]]:
|
||||
with belief_scope("get_dashboards_page"):
|
||||
validated_query = self._validate_query_params(query or {})
|
||||
if "columns" not in validated_query:
|
||||
validated_query["columns"] = [
|
||||
"slug", "id", "url", "changed_on_utc", "dashboard_title",
|
||||
"published", "created_by", "changed_by", "changed_by_name", "owners",
|
||||
]
|
||||
|
||||
response_json = cast(
|
||||
Dict[str, Any],
|
||||
self.network.request(
|
||||
method="GET",
|
||||
endpoint="/dashboard/",
|
||||
params={"q": json.dumps(validated_query)},
|
||||
),
|
||||
)
|
||||
result = response_json.get("result", [])
|
||||
total_count = response_json.get("count", len(result))
|
||||
return total_count, result
|
||||
|
||||
# [/DEF:SupersetClientGetDashboardsPage:Function]
|
||||
|
||||
# [DEF:SupersetClientGetDashboardsSummary:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Fetches dashboard metadata optimized for the grid.
|
||||
# @RELATION: CALLS -> [SupersetClientGetDashboards]
|
||||
def get_dashboards_summary(self, require_slug: bool = False) -> List[Dict]:
|
||||
with belief_scope("SupersetClient.get_dashboards_summary"):
|
||||
query: Dict[str, Any] = {}
|
||||
if require_slug:
|
||||
query["filters"] = [{"col": "slug", "opr": "neq", "value": ""}]
|
||||
_, dashboards = self.get_dashboards(query=query)
|
||||
|
||||
result = []
|
||||
max_debug_samples = 12
|
||||
for index, dash in enumerate(dashboards):
|
||||
raw_owners = dash.get("owners")
|
||||
raw_created_by = dash.get("created_by")
|
||||
raw_changed_by = dash.get("changed_by")
|
||||
raw_changed_by_name = dash.get("changed_by_name")
|
||||
|
||||
owners = self._extract_owner_labels(raw_owners)
|
||||
if not owners:
|
||||
owners = self._extract_owner_labels([raw_created_by, raw_changed_by])
|
||||
|
||||
projected_created_by = self._extract_user_display(None, raw_created_by)
|
||||
projected_modified_by = self._extract_user_display(
|
||||
raw_changed_by_name, raw_changed_by,
|
||||
)
|
||||
|
||||
raw_owner_usernames: List[str] = []
|
||||
if isinstance(raw_owners, list):
|
||||
for owner_payload in raw_owners:
|
||||
if isinstance(owner_payload, dict):
|
||||
owner_username = self._sanitize_user_text(
|
||||
owner_payload.get("username")
|
||||
)
|
||||
if owner_username:
|
||||
raw_owner_usernames.append(owner_username)
|
||||
|
||||
result.append({
|
||||
"id": dash.get("id"),
|
||||
"slug": dash.get("slug"),
|
||||
"title": dash.get("dashboard_title"),
|
||||
"url": dash.get("url"),
|
||||
"last_modified": dash.get("changed_on_utc"),
|
||||
"status": "published" if dash.get("published") else "draft",
|
||||
"created_by": projected_created_by,
|
||||
"modified_by": projected_modified_by,
|
||||
"owners": owners,
|
||||
})
|
||||
|
||||
if index < max_debug_samples:
|
||||
app_logger.reflect(
|
||||
"[REFLECT] Dashboard actor projection sample "
|
||||
f"(env={getattr(self.env, 'id', None)}, dashboard_id={dash.get('id')}, "
|
||||
f"raw_owners={raw_owners!r}, raw_owner_usernames={raw_owner_usernames!r}, "
|
||||
f"raw_created_by={raw_created_by!r}, raw_changed_by={raw_changed_by!r}, "
|
||||
f"raw_changed_by_name={raw_changed_by_name!r}, projected_owners={owners!r}, "
|
||||
f"projected_created_by={projected_created_by!r}, projected_modified_by={projected_modified_by!r})"
|
||||
)
|
||||
|
||||
app_logger.reflect(
|
||||
"[REFLECT] Dashboard actor projection summary "
|
||||
f"(env={getattr(self.env, 'id', None)}, dashboards={len(result)}, "
|
||||
f"sampled={min(len(result), max_debug_samples)})"
|
||||
)
|
||||
return result
|
||||
|
||||
# [/DEF:SupersetClientGetDashboardsSummary:Function]
|
||||
|
||||
# [DEF:SupersetClientGetDashboardsSummaryPage:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Fetches one page of dashboard metadata optimized for the grid.
|
||||
# @RELATION: CALLS -> [SupersetClientGetDashboardsPage]
|
||||
def get_dashboards_summary_page(
|
||||
self,
|
||||
page: int,
|
||||
page_size: int,
|
||||
search: Optional[str] = None,
|
||||
require_slug: bool = False,
|
||||
) -> Tuple[int, List[Dict]]:
|
||||
with belief_scope("SupersetClient.get_dashboards_summary_page"):
|
||||
query: Dict[str, Any] = {
|
||||
"page": max(page - 1, 0),
|
||||
"page_size": page_size,
|
||||
}
|
||||
filters: List[Dict[str, Any]] = []
|
||||
if require_slug:
|
||||
filters.append({"col": "slug", "opr": "neq", "value": ""})
|
||||
normalized_search = (search or "").strip()
|
||||
if normalized_search:
|
||||
filters.append({
|
||||
"col": "dashboard_title", "opr": "ct", "value": normalized_search,
|
||||
})
|
||||
if filters:
|
||||
query["filters"] = filters
|
||||
|
||||
total_count, dashboards = self.get_dashboards_page(query=query)
|
||||
|
||||
result = []
|
||||
for dash in dashboards:
|
||||
owners = self._extract_owner_labels(dash.get("owners"))
|
||||
if not owners:
|
||||
owners = self._extract_owner_labels(
|
||||
[dash.get("created_by"), dash.get("changed_by")],
|
||||
)
|
||||
|
||||
result.append({
|
||||
"id": dash.get("id"),
|
||||
"slug": dash.get("slug"),
|
||||
"title": dash.get("dashboard_title"),
|
||||
"url": dash.get("url"),
|
||||
"last_modified": dash.get("changed_on_utc"),
|
||||
"status": "published" if dash.get("published") else "draft",
|
||||
"created_by": self._extract_user_display(
|
||||
None, dash.get("created_by"),
|
||||
),
|
||||
"modified_by": self._extract_user_display(
|
||||
dash.get("changed_by_name"), dash.get("changed_by"),
|
||||
),
|
||||
"owners": owners,
|
||||
})
|
||||
|
||||
return total_count, result
|
||||
|
||||
# [/DEF:SupersetClientGetDashboardsSummaryPage:Function]
|
||||
|
||||
|
||||
# [/DEF:SupersetDashboardsListMixin:Class]
|
||||
# [/DEF:SupersetDashboardsListMixin:Module]
|
||||
89
backend/src/core/superset_client/_databases.py
Normal file
89
backend/src/core/superset_client/_databases.py
Normal file
@@ -0,0 +1,89 @@
|
||||
# [DEF:SupersetDatabasesMixin:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Database domain mixin for SupersetClient — list, get, summary, by_uuid.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple, cast
|
||||
|
||||
from ..logger import logger as app_logger, belief_scope
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
|
||||
|
||||
# [DEF:SupersetDatabasesMixin:Class]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Mixin providing all database-related Superset API operations.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
|
||||
class SupersetDatabasesMixin:
|
||||
# [DEF:SupersetClientGetDatabases:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Получает полный список баз данных.
|
||||
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
|
||||
def get_databases(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
|
||||
with belief_scope("get_databases"):
|
||||
app_logger.info("[get_databases][Enter] Fetching databases.")
|
||||
validated_query = self._validate_query_params(query or {})
|
||||
if "columns" not in validated_query:
|
||||
validated_query["columns"] = []
|
||||
|
||||
paginated_data = self._fetch_all_pages(
|
||||
endpoint="/database/",
|
||||
pagination_options={
|
||||
"base_query": validated_query,
|
||||
"results_field": "result",
|
||||
},
|
||||
)
|
||||
total_count = len(paginated_data)
|
||||
app_logger.info("[get_databases][Exit] Found %d databases.", total_count)
|
||||
return total_count, paginated_data
|
||||
|
||||
# [/DEF:SupersetClientGetDatabases:Function]
|
||||
|
||||
# [DEF:SupersetClientGetDatabase:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Получает информацию о конкретной базе данных по её ID.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def get_database(self, database_id: int) -> Dict:
|
||||
with belief_scope("get_database"):
|
||||
app_logger.info("[get_database][Enter] Fetching database %s.", database_id)
|
||||
response = self.network.request(
|
||||
method="GET", endpoint=f"/database/{database_id}"
|
||||
)
|
||||
response = cast(Dict, response)
|
||||
app_logger.info("[get_database][Exit] Got database %s.", database_id)
|
||||
return response
|
||||
|
||||
# [/DEF:SupersetClientGetDatabase:Function]
|
||||
|
||||
# [DEF:SupersetClientGetDatabasesSummary:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Fetch a summary of databases including uuid, name, and engine.
|
||||
# @RELATION: CALLS -> [SupersetClientGetDatabases]
|
||||
def get_databases_summary(self) -> List[Dict]:
|
||||
with belief_scope("SupersetClient.get_databases_summary"):
|
||||
query = {"columns": ["uuid", "database_name", "backend"]}
|
||||
_, databases = self.get_databases(query=query)
|
||||
|
||||
# Map 'backend' to 'engine' for consistency with contracts
|
||||
for db in databases:
|
||||
db["engine"] = db.pop("backend", None)
|
||||
|
||||
return databases
|
||||
|
||||
# [/DEF:SupersetClientGetDatabasesSummary:Function]
|
||||
|
||||
# [DEF:SupersetClientGetDatabaseByUuid:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Find a database by its UUID.
|
||||
# @RELATION: CALLS -> [SupersetClientGetDatabases]
|
||||
def get_database_by_uuid(self, db_uuid: str) -> Optional[Dict]:
|
||||
with belief_scope("SupersetClient.get_database_by_uuid", f"uuid={db_uuid}"):
|
||||
query = {"filters": [{"col": "uuid", "op": "eq", "value": db_uuid}]}
|
||||
_, databases = self.get_databases(query=query)
|
||||
return databases[0] if databases else None
|
||||
|
||||
# [/DEF:SupersetClientGetDatabaseByUuid:Function]
|
||||
|
||||
|
||||
# [/DEF:SupersetDatabasesMixin:Class]
|
||||
# [/DEF:SupersetDatabasesMixin:Module]
|
||||
217
backend/src/core/superset_client/_datasets.py
Normal file
217
backend/src/core/superset_client/_datasets.py
Normal file
@@ -0,0 +1,217 @@
|
||||
# [DEF:SupersetDatasetsMixin:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Dataset domain mixin for SupersetClient — list, get, detail, update.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional, Tuple, cast
|
||||
|
||||
from ..logger import logger as app_logger, belief_scope
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
|
||||
|
||||
# [DEF:SupersetDatasetsMixin:Class]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Mixin providing basic dataset CRUD operations (list, get, detail, update).
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
|
||||
class SupersetDatasetsMixin:
|
||||
# [DEF:SupersetClientGetDatasets:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Получает полный список датасетов, автоматически обрабатывая пагинацию.
|
||||
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
|
||||
def get_datasets(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
|
||||
with belief_scope("get_datasets"):
|
||||
app_logger.info("[get_datasets][Enter] Fetching datasets.")
|
||||
validated_query = self._validate_query_params(query)
|
||||
|
||||
paginated_data = self._fetch_all_pages(
|
||||
endpoint="/dataset/",
|
||||
pagination_options={
|
||||
"base_query": validated_query,
|
||||
"results_field": "result",
|
||||
},
|
||||
)
|
||||
total_count = len(paginated_data)
|
||||
app_logger.info("[get_datasets][Exit] Found %d datasets.", total_count)
|
||||
return total_count, paginated_data
|
||||
|
||||
# [/DEF:SupersetClientGetDatasets:Function]
|
||||
|
||||
# [DEF:SupersetClientGetDatasetsSummary:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Fetches dataset metadata optimized for the Dataset Hub grid.
|
||||
# @RELATION: CALLS -> [SupersetClientGetDatasets]
|
||||
def get_datasets_summary(self) -> List[Dict]:
|
||||
with belief_scope("SupersetClient.get_datasets_summary"):
|
||||
query = {"columns": ["id", "table_name", "schema", "database"]}
|
||||
_, datasets = self.get_datasets(query=query)
|
||||
|
||||
result = []
|
||||
for ds in datasets:
|
||||
result.append(
|
||||
{
|
||||
"id": ds.get("id"),
|
||||
"table_name": ds.get("table_name"),
|
||||
"schema": ds.get("schema"),
|
||||
"database": ds.get("database", {}).get(
|
||||
"database_name", "Unknown"
|
||||
),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
# [/DEF:SupersetClientGetDatasetsSummary:Function]
|
||||
|
||||
# [DEF:SupersetClientGetDatasetDetail:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Fetches detailed dataset information including columns and linked dashboards.
|
||||
# @RELATION: CALLS -> [SupersetClientGetDataset]
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def get_dataset_detail(self, dataset_id: int) -> Dict:
|
||||
with belief_scope("SupersetClient.get_dataset_detail", f"id={dataset_id}"):
|
||||
|
||||
def as_bool(value, default=False):
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in ("1", "true", "yes", "y", "on")
|
||||
return bool(value)
|
||||
|
||||
response = self.get_dataset(dataset_id)
|
||||
|
||||
if isinstance(response, dict) and "result" in response:
|
||||
dataset = response["result"]
|
||||
else:
|
||||
dataset = response
|
||||
|
||||
columns = dataset.get("columns", [])
|
||||
column_info = []
|
||||
for col in columns:
|
||||
col_id = col.get("id")
|
||||
if col_id is None:
|
||||
continue
|
||||
column_info.append(
|
||||
{
|
||||
"id": int(col_id),
|
||||
"name": col.get("column_name"),
|
||||
"type": col.get("type"),
|
||||
"is_dttm": as_bool(col.get("is_dttm"), default=False),
|
||||
"is_active": as_bool(col.get("is_active"), default=True),
|
||||
"description": col.get("description", ""),
|
||||
}
|
||||
)
|
||||
|
||||
linked_dashboards = []
|
||||
try:
|
||||
related_objects = self.network.request(
|
||||
method="GET", endpoint=f"/dataset/{dataset_id}/related_objects"
|
||||
)
|
||||
|
||||
if isinstance(related_objects, dict):
|
||||
if "dashboards" in related_objects:
|
||||
dashboards_data = related_objects["dashboards"]
|
||||
elif "result" in related_objects and isinstance(
|
||||
related_objects["result"], dict
|
||||
):
|
||||
dashboards_data = related_objects["result"].get("dashboards", [])
|
||||
else:
|
||||
dashboards_data = []
|
||||
|
||||
for dash in dashboards_data:
|
||||
if isinstance(dash, dict):
|
||||
dash_id = dash.get("id")
|
||||
if dash_id is None:
|
||||
continue
|
||||
linked_dashboards.append(
|
||||
{
|
||||
"id": int(dash_id),
|
||||
"title": dash.get("dashboard_title")
|
||||
or dash.get("title", f"Dashboard {dash_id}"),
|
||||
"slug": dash.get("slug"),
|
||||
}
|
||||
)
|
||||
else:
|
||||
try:
|
||||
dash_id = int(dash)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
linked_dashboards.append(
|
||||
{"id": dash_id, "title": f"Dashboard {dash_id}", "slug": None}
|
||||
)
|
||||
except Exception as e:
|
||||
app_logger.warning(
|
||||
f"[get_dataset_detail][Warning] Failed to fetch related dashboards: {e}"
|
||||
)
|
||||
linked_dashboards = []
|
||||
|
||||
sql = dataset.get("sql", "")
|
||||
|
||||
result = {
|
||||
"id": dataset.get("id"),
|
||||
"table_name": dataset.get("table_name"),
|
||||
"schema": dataset.get("schema"),
|
||||
"database": (
|
||||
dataset.get("database", {}).get("database_name", "Unknown")
|
||||
if isinstance(dataset.get("database"), dict)
|
||||
else dataset.get("database_name") or "Unknown"
|
||||
),
|
||||
"description": dataset.get("description", ""),
|
||||
"columns": column_info,
|
||||
"column_count": len(column_info),
|
||||
"sql": sql,
|
||||
"linked_dashboards": linked_dashboards,
|
||||
"linked_dashboard_count": len(linked_dashboards),
|
||||
"is_sqllab_view": as_bool(dataset.get("is_sqllab_view"), default=False),
|
||||
"created_on": dataset.get("created_on"),
|
||||
"changed_on": dataset.get("changed_on"),
|
||||
}
|
||||
|
||||
app_logger.info(
|
||||
f"[get_dataset_detail][Exit] Got dataset {dataset_id} with {len(column_info)} columns and {len(linked_dashboards)} linked dashboards"
|
||||
)
|
||||
return result
|
||||
|
||||
# [/DEF:SupersetClientGetDatasetDetail:Function]
|
||||
|
||||
# [DEF:SupersetClientGetDataset:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Получает информацию о конкретном датасете по его ID.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def get_dataset(self, dataset_id: int) -> Dict:
|
||||
with belief_scope("SupersetClient.get_dataset", f"id={dataset_id}"):
|
||||
app_logger.info("[get_dataset][Enter] Fetching dataset %s.", dataset_id)
|
||||
response = self.network.request(
|
||||
method="GET", endpoint=f"/dataset/{dataset_id}"
|
||||
)
|
||||
response = cast(Dict, response)
|
||||
app_logger.info("[get_dataset][Exit] Got dataset %s.", dataset_id)
|
||||
return response
|
||||
|
||||
# [/DEF:SupersetClientGetDataset:Function]
|
||||
|
||||
# [DEF:SupersetClientUpdateDataset:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Обновляет данные датасета по его ID.
|
||||
# @SIDE_EFFECT: Modifies resource in upstream Superset environment.
|
||||
# @RELATION: CALLS -> [APIClient]
|
||||
def update_dataset(self, dataset_id: int, data: Dict) -> Dict:
|
||||
with belief_scope("SupersetClient.update_dataset", f"id={dataset_id}"):
|
||||
app_logger.info("[update_dataset][Enter] Updating dataset %s.", dataset_id)
|
||||
response = self.network.request(
|
||||
method="PUT",
|
||||
endpoint=f"/dataset/{dataset_id}",
|
||||
data=json.dumps(data),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
response = cast(Dict, response)
|
||||
app_logger.info("[update_dataset][Exit] Updated dataset %s.", dataset_id)
|
||||
return response
|
||||
|
||||
# [/DEF:SupersetClientUpdateDataset:Function]
|
||||
|
||||
|
||||
# [/DEF:SupersetDatasetsMixin:Class]
|
||||
# [/DEF:SupersetDatasetsMixin:Module]
|
||||
397
backend/src/core/superset_client/_datasets_preview.py
Normal file
397
backend/src/core/superset_client/_datasets_preview.py
Normal file
@@ -0,0 +1,397 @@
|
||||
# [DEF:SupersetDatasetsPreviewMixin:Module]
|
||||
# @PURPOSE: Dataset preview compilation mixin for SupersetClient — build query context, compile SQL.
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from ..logger import logger as app_logger, belief_scope
|
||||
from ..utils.network import SupersetAPIError
|
||||
# [DEF:SupersetDatasetsPreviewMixin:Class]
|
||||
# @PURPOSE: Mixin providing dataset preview compilation and query context building.
|
||||
class SupersetDatasetsPreviewMixin:
|
||||
# [DEF:SupersetClientCompileDatasetPreview:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Compile dataset preview SQL through the strongest supported Superset preview endpoint family and return normalized SQL output.
|
||||
def compile_dataset_preview(self, dataset_id: int, template_params: Optional[Dict[str, Any]] = None, effective_filters: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]:
|
||||
with belief_scope('SupersetClientCompileDatasetPreview'):
|
||||
app_logger.reason('Belief protocol reasoning checkpoint for SupersetClientCompileDatasetPreview')
|
||||
dataset_response = self.get_dataset(dataset_id)
|
||||
dataset_record = dataset_response.get('result', dataset_response) if isinstance(dataset_response, dict) else {}
|
||||
query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or [])
|
||||
legacy_form_data = self.build_dataset_preview_legacy_form_data(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or [])
|
||||
legacy_form_data_payload = json.dumps(legacy_form_data, sort_keys=True, default=str)
|
||||
request_payload = json.dumps(query_context)
|
||||
strategy_attempts: List[Dict[str, Any]] = []
|
||||
strategy_candidates: List[Dict[str, Any]] = [{'endpoint_kind': 'legacy_explore_form_data', 'endpoint': '/explore_json/form_data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'legacy_data_form_data', 'endpoint': '/data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'v1_chart_data', 'endpoint': '/chart/data', 'request_transport': 'json_body', 'data': request_payload, 'headers': {'Content-Type': 'application/json'}}]
|
||||
for candidate in strategy_candidates:
|
||||
endpoint_kind = candidate['endpoint_kind']
|
||||
endpoint_path = candidate['endpoint']
|
||||
request_transport = candidate['request_transport']
|
||||
request_params = deepcopy(candidate.get('params') or {})
|
||||
request_body = candidate.get('data')
|
||||
request_headers = deepcopy(candidate.get('headers') or {})
|
||||
request_param_keys = sorted(request_params.keys())
|
||||
request_payload_keys: List[str] = []
|
||||
if isinstance(request_body, str):
|
||||
try:
|
||||
decoded_request_body = json.loads(request_body)
|
||||
if isinstance(decoded_request_body, dict):
|
||||
request_payload_keys = sorted(decoded_request_body.keys())
|
||||
except json.JSONDecodeError:
|
||||
request_payload_keys = []
|
||||
elif isinstance(request_body, dict):
|
||||
request_payload_keys = sorted(request_body.keys())
|
||||
strategy_diagnostics = {'endpoint': endpoint_path, 'endpoint_kind': endpoint_kind, 'request_transport': request_transport, 'contains_root_datasource': endpoint_kind == 'v1_chart_data' and 'datasource' in query_context, 'contains_form_datasource': endpoint_kind.startswith('legacy_') and 'datasource' in legacy_form_data, 'contains_query_object_datasource': bool(query_context.get('queries')) and isinstance(query_context['queries'][0], dict) and ('datasource' in query_context['queries'][0]), 'request_param_keys': request_param_keys, 'request_payload_keys': request_payload_keys}
|
||||
app_logger.reason('Attempting Superset dataset preview compilation strategy', extra={'dataset_id': dataset_id, **strategy_diagnostics, 'request_params': request_params, 'request_payload': request_body, 'legacy_form_data': legacy_form_data if endpoint_kind.startswith('legacy_') else None, 'query_context': query_context if endpoint_kind == 'v1_chart_data' else None, 'template_param_count': len(template_params or {}), 'filter_count': len(effective_filters or [])})
|
||||
try:
|
||||
response = self.network.request(method='POST', endpoint=endpoint_path, params=request_params or None, data=request_body, headers=request_headers or None)
|
||||
normalized = self._extract_compiled_sql_from_preview_response(response)
|
||||
normalized['query_context'] = query_context
|
||||
normalized['legacy_form_data'] = legacy_form_data
|
||||
normalized['endpoint'] = endpoint_path
|
||||
normalized['endpoint_kind'] = endpoint_kind
|
||||
normalized['dataset_id'] = dataset_id
|
||||
normalized['strategy_attempts'] = strategy_attempts + [{**strategy_diagnostics, 'success': True}]
|
||||
app_logger.reflect('Dataset preview compilation returned normalized SQL payload', extra={'dataset_id': dataset_id, **strategy_diagnostics, 'success': True, 'compiled_sql_length': len(str(normalized.get('compiled_sql') or '')), 'response_diagnostics': normalized.get('response_diagnostics')})
|
||||
app_logger.reflect('Belief protocol postcondition checkpoint for SupersetClientCompileDatasetPreview')
|
||||
return normalized
|
||||
except Exception as exc:
|
||||
failure_diagnostics = {**strategy_diagnostics, 'success': False, 'error': str(exc)}
|
||||
strategy_attempts.append(failure_diagnostics)
|
||||
app_logger.explore('Superset dataset preview compilation strategy failed', extra={'dataset_id': dataset_id, **failure_diagnostics, 'request_params': request_params, 'request_payload': request_body})
|
||||
raise SupersetAPIError(f'Superset preview compilation failed for all known strategies (attempts={strategy_attempts!r})')
|
||||
# [/DEF:SupersetClientCompileDatasetPreview:Function]
|
||||
|
||||
# [DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Build browser-style legacy form_data payload for Superset preview endpoints inferred from observed deployment traffic.
|
||||
def build_dataset_preview_legacy_form_data(self, dataset_id: int, dataset_record: Dict[str, Any], template_params: Dict[str, Any], effective_filters: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
with belief_scope('SupersetClientBuildDatasetPreviewLegacyFormData'):
|
||||
app_logger.reason('Belief protocol reasoning checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData')
|
||||
query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params, effective_filters=effective_filters)
|
||||
query_object = deepcopy(query_context.get('queries', [{}])[0] if query_context.get('queries') else {})
|
||||
legacy_form_data = deepcopy(query_context.get('form_data', {}))
|
||||
legacy_form_data.pop('datasource', None)
|
||||
legacy_form_data['metrics'] = deepcopy(query_object.get('metrics', ['count']))
|
||||
legacy_form_data['columns'] = deepcopy(query_object.get('columns', []))
|
||||
legacy_form_data['orderby'] = deepcopy(query_object.get('orderby', []))
|
||||
legacy_form_data['annotation_layers'] = deepcopy(query_object.get('annotation_layers', []))
|
||||
legacy_form_data['row_limit'] = query_object.get('row_limit', 1000)
|
||||
legacy_form_data['series_limit'] = query_object.get('series_limit', 0)
|
||||
legacy_form_data['url_params'] = deepcopy(query_object.get('url_params', template_params))
|
||||
legacy_form_data['applied_time_extras'] = deepcopy(query_object.get('applied_time_extras', {}))
|
||||
legacy_form_data['result_format'] = query_context.get('result_format', 'json')
|
||||
legacy_form_data['result_type'] = query_context.get('result_type', 'query')
|
||||
legacy_form_data['force'] = bool(query_context.get('force', True))
|
||||
extras = query_object.get('extras')
|
||||
if isinstance(extras, dict):
|
||||
legacy_form_data['extras'] = deepcopy(extras)
|
||||
time_range = query_object.get('time_range')
|
||||
if time_range:
|
||||
legacy_form_data['time_range'] = time_range
|
||||
app_logger.reflect('Built Superset legacy preview form_data payload from browser-observed request shape', extra={'dataset_id': dataset_id, 'legacy_endpoint_inference': 'POST /explore_json/form_data?form_data=... primary, POST /data?form_data=... fallback, based on observed browser traffic', 'contains_form_datasource': 'datasource' in legacy_form_data, 'legacy_form_data_keys': sorted(legacy_form_data.keys()), 'legacy_extra_filters': legacy_form_data.get('extra_filters', []), 'legacy_extra_form_data': legacy_form_data.get('extra_form_data', {})})
|
||||
app_logger.reflect('Belief protocol postcondition checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData')
|
||||
return legacy_form_data
|
||||
# [/DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function]
|
||||
|
||||
# [DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Build a reduced-scope chart-data query context for deterministic dataset preview compilation.
|
||||
def build_dataset_preview_query_context(
|
||||
self,
|
||||
dataset_id: int,
|
||||
dataset_record: Dict[str, Any],
|
||||
template_params: Dict[str, Any],
|
||||
effective_filters: List[Dict[str, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
with belief_scope("SupersetClientBuildDatasetPreviewQueryContext"):
|
||||
app_logger.reason(
|
||||
"Building Superset dataset preview query context",
|
||||
extra={"dataset_id": dataset_id, "filter_count": len(effective_filters or [])},
|
||||
)
|
||||
normalized_template_params = deepcopy(template_params or {})
|
||||
normalized_filter_payload = (
|
||||
self._normalize_effective_filters_for_query_context(
|
||||
effective_filters or []
|
||||
)
|
||||
)
|
||||
normalized_filters = normalized_filter_payload["filters"]
|
||||
normalized_extra_form_data = normalized_filter_payload["extra_form_data"]
|
||||
|
||||
datasource_payload: Dict[str, Any] = {
|
||||
"id": dataset_id,
|
||||
"type": "table",
|
||||
}
|
||||
datasource = dataset_record.get("datasource")
|
||||
if isinstance(datasource, dict):
|
||||
datasource_id = datasource.get("id")
|
||||
datasource_type = datasource.get("type")
|
||||
if datasource_id is not None:
|
||||
datasource_payload["id"] = datasource_id
|
||||
if datasource_type:
|
||||
datasource_payload["type"] = datasource_type
|
||||
|
||||
serialized_dataset_template_params = dataset_record.get("template_params")
|
||||
if (
|
||||
isinstance(serialized_dataset_template_params, str)
|
||||
and serialized_dataset_template_params.strip()
|
||||
):
|
||||
try:
|
||||
parsed_dataset_template_params = json.loads(
|
||||
serialized_dataset_template_params
|
||||
)
|
||||
if isinstance(parsed_dataset_template_params, dict):
|
||||
for key, value in parsed_dataset_template_params.items():
|
||||
normalized_template_params.setdefault(str(key), value)
|
||||
except json.JSONDecodeError:
|
||||
app_logger.explore(
|
||||
"Dataset template_params could not be parsed while building preview query context",
|
||||
extra={"dataset_id": dataset_id},
|
||||
)
|
||||
|
||||
extra_form_data: Dict[str, Any] = deepcopy(normalized_extra_form_data)
|
||||
if normalized_filters:
|
||||
extra_form_data["filters"] = deepcopy(normalized_filters)
|
||||
|
||||
query_object: Dict[str, Any] = {
|
||||
"filters": normalized_filters,
|
||||
"extras": {"where": ""},
|
||||
"columns": [],
|
||||
"metrics": ["count"],
|
||||
"orderby": [],
|
||||
"annotation_layers": [],
|
||||
"row_limit": 1000,
|
||||
"series_limit": 0,
|
||||
"url_params": normalized_template_params,
|
||||
"applied_time_extras": {},
|
||||
"result_type": "query",
|
||||
}
|
||||
|
||||
schema = dataset_record.get("schema")
|
||||
if schema:
|
||||
query_object["schema"] = schema
|
||||
|
||||
time_range = extra_form_data.get("time_range") or dataset_record.get(
|
||||
"default_time_range"
|
||||
)
|
||||
if time_range:
|
||||
query_object["time_range"] = time_range
|
||||
extra_form_data["time_range"] = time_range
|
||||
|
||||
result_format = dataset_record.get("result_format") or "json"
|
||||
result_type = "query"
|
||||
|
||||
form_data: Dict[str, Any] = {
|
||||
"datasource": f"{datasource_payload['id']}__{datasource_payload['type']}",
|
||||
"datasource_id": datasource_payload["id"],
|
||||
"datasource_type": datasource_payload["type"],
|
||||
"viz_type": "table",
|
||||
"slice_id": None,
|
||||
"query_mode": "raw",
|
||||
"url_params": normalized_template_params,
|
||||
"extra_filters": deepcopy(normalized_filters),
|
||||
"adhoc_filters": [],
|
||||
}
|
||||
if extra_form_data:
|
||||
form_data["extra_form_data"] = extra_form_data
|
||||
|
||||
payload = {
|
||||
"datasource": datasource_payload,
|
||||
"queries": [query_object],
|
||||
"form_data": form_data,
|
||||
"result_format": result_format,
|
||||
"result_type": result_type,
|
||||
"force": True,
|
||||
}
|
||||
app_logger.reflect(
|
||||
"Built Superset dataset preview query context",
|
||||
extra={
|
||||
"dataset_id": dataset_id,
|
||||
"datasource": datasource_payload,
|
||||
"normalized_effective_filters": normalized_filters,
|
||||
"normalized_filter_diagnostics": normalized_filter_payload[
|
||||
"diagnostics"
|
||||
],
|
||||
"result_type": result_type,
|
||||
"result_format": result_format,
|
||||
},
|
||||
)
|
||||
return payload
|
||||
|
||||
# [/DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]
|
||||
|
||||
# [DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]
|
||||
def _normalize_effective_filters_for_query_context(
|
||||
self,
|
||||
effective_filters: List[Dict[str, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
with belief_scope(
|
||||
"SupersetClient._normalize_effective_filters_for_query_context"
|
||||
):
|
||||
normalized_filters: List[Dict[str, Any]] = []
|
||||
merged_extra_form_data: Dict[str, Any] = {}
|
||||
diagnostics: List[Dict[str, Any]] = []
|
||||
|
||||
for item in effective_filters:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
|
||||
display_name = str(
|
||||
item.get("display_name")
|
||||
or item.get("filter_name")
|
||||
or item.get("variable_name")
|
||||
or "unresolved_filter"
|
||||
).strip()
|
||||
value = item.get("effective_value")
|
||||
normalized_payload = item.get("normalized_filter_payload")
|
||||
preserved_clauses: List[Dict[str, Any]] = []
|
||||
preserved_extra_form_data: Dict[str, Any] = {}
|
||||
used_preserved_clauses = False
|
||||
|
||||
if isinstance(normalized_payload, dict):
|
||||
raw_clauses = normalized_payload.get("filter_clauses")
|
||||
if isinstance(raw_clauses, list):
|
||||
preserved_clauses = [
|
||||
deepcopy(clause)
|
||||
for clause in raw_clauses
|
||||
if isinstance(clause, dict)
|
||||
]
|
||||
raw_extra_form_data = normalized_payload.get("extra_form_data")
|
||||
if isinstance(raw_extra_form_data, dict):
|
||||
preserved_extra_form_data = deepcopy(raw_extra_form_data)
|
||||
|
||||
if isinstance(preserved_extra_form_data, dict):
|
||||
for key, extra_value in preserved_extra_form_data.items():
|
||||
if key == "filters":
|
||||
continue
|
||||
merged_extra_form_data[key] = deepcopy(extra_value)
|
||||
|
||||
outgoing_clauses: List[Dict[str, Any]] = []
|
||||
if preserved_clauses:
|
||||
for clause in preserved_clauses:
|
||||
clause_copy = deepcopy(clause)
|
||||
if "val" not in clause_copy and value is not None:
|
||||
clause_copy["val"] = deepcopy(value)
|
||||
outgoing_clauses.append(clause_copy)
|
||||
used_preserved_clauses = True
|
||||
elif preserved_extra_form_data:
|
||||
outgoing_clauses = []
|
||||
else:
|
||||
column = str(
|
||||
item.get("variable_name") or item.get("filter_name") or ""
|
||||
).strip()
|
||||
if column and value is not None:
|
||||
operator = "IN" if isinstance(value, list) else "=="
|
||||
outgoing_clauses.append(
|
||||
{"col": column, "op": operator, "val": value}
|
||||
)
|
||||
|
||||
normalized_filters.extend(outgoing_clauses)
|
||||
diagnostics.append(
|
||||
{
|
||||
"filter_name": display_name,
|
||||
"value_origin": (
|
||||
normalized_payload.get("value_origin")
|
||||
if isinstance(normalized_payload, dict)
|
||||
else None
|
||||
),
|
||||
"used_preserved_clauses": used_preserved_clauses,
|
||||
"outgoing_clauses": deepcopy(outgoing_clauses),
|
||||
}
|
||||
)
|
||||
app_logger.reason(
|
||||
"Normalized effective preview filter for Superset query context",
|
||||
extra={
|
||||
"filter_name": display_name,
|
||||
"used_preserved_clauses": used_preserved_clauses,
|
||||
"outgoing_clauses": outgoing_clauses,
|
||||
"value_origin": (
|
||||
normalized_payload.get("value_origin")
|
||||
if isinstance(normalized_payload, dict)
|
||||
else "heuristic_reconstruction"
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"filters": normalized_filters,
|
||||
"extra_form_data": merged_extra_form_data,
|
||||
"diagnostics": diagnostics,
|
||||
}
|
||||
|
||||
# [/DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]
|
||||
|
||||
# [DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function]
|
||||
def _extract_compiled_sql_from_preview_response(
|
||||
self, response: Any
|
||||
) -> Dict[str, Any]:
|
||||
with belief_scope("SupersetClient._extract_compiled_sql_from_preview_response"):
|
||||
if not isinstance(response, dict):
|
||||
raise SupersetAPIError(
|
||||
"Superset preview response was not a JSON object"
|
||||
)
|
||||
|
||||
response_diagnostics: List[Dict[str, Any]] = []
|
||||
result_payload = response.get("result")
|
||||
if isinstance(result_payload, list):
|
||||
for index, item in enumerate(result_payload):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
compiled_sql = str(
|
||||
item.get("query")
|
||||
or item.get("sql")
|
||||
or item.get("compiled_sql")
|
||||
or ""
|
||||
).strip()
|
||||
response_diagnostics.append(
|
||||
{
|
||||
"index": index,
|
||||
"status": item.get("status"),
|
||||
"applied_filters": item.get("applied_filters"),
|
||||
"rejected_filters": item.get("rejected_filters"),
|
||||
"has_query": bool(compiled_sql),
|
||||
"source": "result_list",
|
||||
}
|
||||
)
|
||||
if compiled_sql:
|
||||
return {
|
||||
"compiled_sql": compiled_sql,
|
||||
"raw_response": response,
|
||||
"response_diagnostics": response_diagnostics,
|
||||
}
|
||||
|
||||
top_level_candidates: List[Tuple[str, Any]] = [
|
||||
("query", response.get("query")),
|
||||
("sql", response.get("sql")),
|
||||
("compiled_sql", response.get("compiled_sql")),
|
||||
]
|
||||
if isinstance(result_payload, dict):
|
||||
top_level_candidates.extend(
|
||||
[
|
||||
("result.query", result_payload.get("query")),
|
||||
("result.sql", result_payload.get("sql")),
|
||||
("result.compiled_sql", result_payload.get("compiled_sql")),
|
||||
]
|
||||
)
|
||||
|
||||
for source, candidate in top_level_candidates:
|
||||
compiled_sql = str(candidate or "").strip()
|
||||
response_diagnostics.append(
|
||||
{"source": source, "has_query": bool(compiled_sql)}
|
||||
)
|
||||
if compiled_sql:
|
||||
return {
|
||||
"compiled_sql": compiled_sql,
|
||||
"raw_response": response,
|
||||
"response_diagnostics": response_diagnostics,
|
||||
}
|
||||
|
||||
raise SupersetAPIError(
|
||||
"Superset preview response did not expose compiled SQL "
|
||||
f"(diagnostics={response_diagnostics!r})"
|
||||
)
|
||||
|
||||
# [/DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function]
|
||||
# [/DEF:SupersetDatasetsPreviewMixin:Class]
|
||||
# [/DEF:SupersetDatasetsPreviewMixin:Module]
|
||||
86
backend/src/core/superset_client/_user_projection.py
Normal file
86
backend/src/core/superset_client/_user_projection.py
Normal file
@@ -0,0 +1,86 @@
|
||||
# [DEF:SupersetUserProjection:Module]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: User/owner payload normalization helpers for Superset client responses.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
|
||||
# [DEF:SupersetUserProjectionMixin:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Mixin providing user/owner payload normalization for Superset API responses.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetClientBase]
|
||||
class SupersetUserProjectionMixin:
|
||||
# [DEF:SupersetClientExtractOwnerLabels:Function]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Normalize dashboard owners payload to stable display labels.
|
||||
def _extract_owner_labels(self, owners_payload: Any) -> List[str]:
|
||||
if owners_payload is None:
|
||||
return []
|
||||
|
||||
owners_list: List[Any]
|
||||
if isinstance(owners_payload, list):
|
||||
owners_list = owners_payload
|
||||
else:
|
||||
owners_list = [owners_payload]
|
||||
|
||||
normalized: List[str] = []
|
||||
for owner in owners_list:
|
||||
label: Optional[str] = None
|
||||
if isinstance(owner, dict):
|
||||
label = self._extract_user_display(None, owner)
|
||||
else:
|
||||
label = self._sanitize_user_text(owner)
|
||||
if label and label not in normalized:
|
||||
normalized.append(label)
|
||||
return normalized
|
||||
|
||||
# [/DEF:SupersetClientExtractOwnerLabels:Function]
|
||||
|
||||
# [DEF:SupersetClientExtractUserDisplay:Function]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Normalize user payload to a stable display name.
|
||||
def _extract_user_display(
|
||||
self, preferred_value: Optional[str], user_payload: Optional[Dict]
|
||||
) -> Optional[str]:
|
||||
preferred = self._sanitize_user_text(preferred_value)
|
||||
if preferred:
|
||||
return preferred
|
||||
|
||||
if isinstance(user_payload, dict):
|
||||
full_name = self._sanitize_user_text(user_payload.get("full_name"))
|
||||
if full_name:
|
||||
return full_name
|
||||
first_name = self._sanitize_user_text(user_payload.get("first_name")) or ""
|
||||
last_name = self._sanitize_user_text(user_payload.get("last_name")) or ""
|
||||
combined = " ".join(
|
||||
part for part in [first_name, last_name] if part
|
||||
).strip()
|
||||
if combined:
|
||||
return combined
|
||||
username = self._sanitize_user_text(user_payload.get("username"))
|
||||
if username:
|
||||
return username
|
||||
email = self._sanitize_user_text(user_payload.get("email"))
|
||||
if email:
|
||||
return email
|
||||
return None
|
||||
|
||||
# [/DEF:SupersetClientExtractUserDisplay:Function]
|
||||
|
||||
# [DEF:SupersetClientSanitizeUserText:Function]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Convert scalar value to non-empty user-facing text.
|
||||
def _sanitize_user_text(self, value: Optional[Union[str, int]]) -> Optional[str]:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = str(value).strip()
|
||||
if not normalized:
|
||||
return None
|
||||
return normalized
|
||||
|
||||
# [/DEF:SupersetClientSanitizeUserText:Function]
|
||||
|
||||
|
||||
# [/DEF:SupersetUserProjectionMixin:Class]
|
||||
# [/DEF:SupersetUserProjection:Module]
|
||||
Reference in New Issue
Block a user