semantics

This commit is contained in:
2026-05-26 09:30:41 +03:00
parent 1e7bcecaea
commit 9ffa8af1dc
623 changed files with 28045 additions and 26557 deletions

View File

@@ -1,13 +1,13 @@
# #region AsyncNetworkModule [C:5] [TYPE Module] [SEMANTICS auth, superset, token, async-client]
#
# @BRIEF Provides async Superset API client with shared auth-token cache to avoid per-request re-login.
# @LAYER: Infra
# @PRE: Config payloads contain a Superset base URL and authentication fields needed for login.
# @POST: Async network clients reuse cached auth tokens and expose stable async request/error translation flow.
# @SIDE_EFFECT: Performs upstream HTTP I/O and mutates process-local auth cache entries.
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions]
# @LAYER Infrastructure
# @PRE Config payloads contain a Superset base URL and authentication fields needed for login.
# @POST Async network clients reuse cached auth tokens and expose stable async request/error translation flow.
# @SIDE_EFFECT Performs upstream HTTP I/O and mutates process-local auth cache entries.
# @DATA_CONTRACT Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions]
# @RELATION DEPENDS_ON -> [SupersetAuthCache]
# @INVARIANT: Async client reuses cached auth tokens per environment credentials and invalidates on 401.
# @INVARIANT Async client reuses cached auth tokens per environment credentials and invalidates on 401.
import asyncio
from typing import Any
@@ -34,11 +34,11 @@ class AsyncAPIClient:
_auth_locks: dict[tuple[str, str, bool], asyncio.Lock] = {}
# #region AsyncAPIClient.__init__ [TYPE Function] [C:3]
# @PURPOSE: Initialize async API client for one environment.
# @PRE: config contains base_url and auth payload.
# @POST: Client is ready for async request/authentication flow.
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> self._auth_cache_key[str]
# @RELATION: [CALLS] ->[AsyncAPIClient._normalize_base_url]
# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]
# @PRE config contains base_url and auth payload.
# @POST Client is ready for async request/authentication flow.
# @DATA_CONTRACT Input[config: Dict[str, Any]] -> self._auth_cache_key[str]
# @RELATION CALLS -> [AsyncAPIClient._normalize_base_url]
# @RELATION DEPENDS_ON -> [SupersetAuthCache]
def __init__(self, config: dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
self.base_url: str = self._normalize_base_url(config.get("base_url", ""))
self.api_base_url: str = f"{self.base_url}/api/v1"
@@ -59,7 +59,7 @@ class AsyncAPIClient:
# #endregion AsyncAPIClient.__init__
# #region AsyncAPIClient._normalize_base_url [TYPE Function] [C:1]
# @PURPOSE: Normalize base URL for Superset API root construction.
# @POST: Returns canonical base URL without trailing slash and duplicate /api/v1 suffix.
# @POST Returns canonical base URL without trailing slash and duplicate /api/v1 suffix.
def _normalize_base_url(self, raw_url: str) -> str:
normalized = str(raw_url or "").strip().rstrip("/")
if normalized.lower().endswith("/api/v1"):
@@ -68,7 +68,7 @@ class AsyncAPIClient:
# #endregion AsyncAPIClient._normalize_base_url
# #region AsyncAPIClient._build_api_url [TYPE Function] [C:1]
# @PURPOSE: Build full API URL from relative Superset endpoint.
# @POST: Returns absolute URL for upstream request.
# @POST Returns absolute URL for upstream request.
def _build_api_url(self, endpoint: str) -> str:
normalized_endpoint = str(endpoint or "").strip()
if normalized_endpoint.startswith("http://") or normalized_endpoint.startswith("https://"):
@@ -81,7 +81,7 @@ class AsyncAPIClient:
# #endregion AsyncAPIClient._build_api_url
# #region AsyncAPIClient._get_auth_lock [TYPE Function] [C:1]
# @PURPOSE: Return per-cache-key async lock to serialize fresh login attempts.
# @POST: Returns stable asyncio.Lock instance.
# @POST Returns stable asyncio.Lock instance.
@classmethod
def _get_auth_lock(cls, cache_key: tuple[str, str, bool]) -> asyncio.Lock:
existing_lock = cls._auth_locks.get(cache_key)
@@ -93,12 +93,12 @@ class AsyncAPIClient:
# #endregion AsyncAPIClient._get_auth_lock
# #region AsyncAPIClient.authenticate [TYPE Function] [C:3]
# @PURPOSE: Authenticate against Superset and cache access/csrf tokens.
# @POST: Client tokens are populated and reusable across requests.
# @SIDE_EFFECT: Performs network requests to Superset authentication endpoints.
# @DATA_CONTRACT: None -> Output[Dict[str, str]]
# @RELATION: [CALLS] ->[SupersetAuthCache.get]
# @RELATION: [CALLS] ->[SupersetAuthCache.set]
# @RELATION: [CALLS] ->[AsyncAPIClient._get_auth_lock]
# @POST Client tokens are populated and reusable across requests.
# @SIDE_EFFECT Performs network requests to Superset authentication endpoints.
# @DATA_CONTRACT None -> Output[Dict[str, str]]
# @RELATION CALLS -> [SupersetAuthCache.get]
# @RELATION CALLS -> [SupersetAuthCache.set]
# @RELATION CALLS -> [AsyncAPIClient._get_auth_lock]
async def authenticate(self) -> dict[str, str]:
cached_tokens = SupersetAuthCache.get(self._auth_cache_key)
if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"):
@@ -150,8 +150,8 @@ class AsyncAPIClient:
# #endregion AsyncAPIClient.authenticate
# #region AsyncAPIClient.get_headers [TYPE Function] [C:3]
# @PURPOSE: Return authenticated Superset headers for async requests.
# @POST: Headers include Authorization and CSRF tokens.
# @RELATION: [CALLS] ->[AsyncAPIClient.authenticate]
# @POST Headers include Authorization and CSRF tokens.
# @RELATION CALLS -> [AsyncAPIClient.authenticate]
async def get_headers(self) -> dict[str, str]:
if not self._authenticated:
await self.authenticate()
@@ -164,48 +164,17 @@ class AsyncAPIClient:
# #endregion AsyncAPIClient.get_headers
# #region AsyncAPIClient.request [TYPE Function] [C:3]
# @PURPOSE: Perform one authenticated async Superset API request.
# @POST: Returns JSON payload or raw httpx.Response when raw_response=true.
# @SIDE_EFFECT: Performs network I/O.
# @RELATION: [CALLS] ->[AsyncAPIClient.get_headers]
# @RELATION: [CALLS] ->[AsyncAPIClient._handle_http_error]
# @RELATION: [CALLS] ->[AsyncAPIClient._handle_network_error]
async def request(
self,
method: str,
endpoint: str,
headers: dict[str, str] | None = None,
raw_response: bool = False,
**kwargs,
) -> httpx.Response | dict[str, Any]:
full_url = self._build_api_url(endpoint)
request_headers = await self.get_headers()
if headers:
request_headers.update(headers)
if "allow_redirects" in kwargs and "follow_redirects" not in kwargs:
kwargs["follow_redirects"] = bool(kwargs.pop("allow_redirects"))
try:
response = await self._client.request(method, full_url, headers=request_headers, **kwargs)
response.raise_for_status()
return response if raw_response else response.json()
except httpx.HTTPStatusError as exc:
if exc.response is not None and exc.response.status_code == 401:
self._authenticated = False
self._tokens = {}
SupersetAuthCache.invalidate(self._auth_cache_key)
self._handle_http_error(exc, endpoint)
except httpx.HTTPError as exc:
self._handle_network_error(exc, full_url)
# #endregion AsyncAPIClient.request
# #region AsyncAPIClient._handle_http_error [TYPE Function] [C:3]
# @PURPOSE: Translate upstream HTTP errors into stable domain exceptions.
# @POST: Raises domain-specific exception for caller flow control.
# @DATA_CONTRACT: Input[httpx.HTTPStatusError] -> Exception
# @RELATION: [CALLS] ->[AsyncAPIClient._is_dashboard_endpoint]
# @RELATION: [DEPENDS_ON] ->[DashboardNotFoundError]
# @RELATION: [DEPENDS_ON] ->[SupersetAPIError]
# @RELATION: [DEPENDS_ON] ->[PermissionDeniedError]
# @RELATION: [DEPENDS_ON] ->[AuthenticationError]
# @RELATION: [DEPENDS_ON] ->[NetworkError]
# @POST Returns JSON payload or raw httpx.Response when raw_response=true.
# @SIDE_EFFECT Performs network I/O.
# @RELATION CALLS -> [AsyncAPIClient.get_headers]
# @RELATION CALLS -> [EXT:method:AsyncAPIClient._handle_http_error]
# @RELATION CALLS -> [AsyncAPIClient._handle_network_error]
# @RELATION CALLS -> [AsyncAPIClient._is_dashboard_endpoint]
# @RELATION DEPENDS_ON -> [DashboardNotFoundError]
# @RELATION DEPENDS_ON -> [SupersetAPIError]
# @RELATION DEPENDS_ON -> [PermissionDeniedError]
# @RELATION DEPENDS_ON -> [AuthenticationError]
# @RELATION DEPENDS_ON -> [NetworkError]
def _handle_http_error(self, exc: httpx.HTTPStatusError, endpoint: str) -> None:
with belief_scope("AsyncAPIClient._handle_http_error"):
status_code = exc.response.status_code
@@ -228,7 +197,7 @@ class AsyncAPIClient:
# #endregion AsyncAPIClient._handle_http_error
# #region AsyncAPIClient._is_dashboard_endpoint [TYPE Function] [C:2]
# @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.
# @POST: Returns true only for dashboard-specific endpoints.
# @POST Returns true only for dashboard-specific endpoints.
def _is_dashboard_endpoint(self, endpoint: str) -> bool:
normalized_endpoint = str(endpoint or "").strip().lower()
if not normalized_endpoint:
@@ -244,9 +213,9 @@ class AsyncAPIClient:
# #endregion AsyncAPIClient._is_dashboard_endpoint
# #region AsyncAPIClient._handle_network_error [TYPE Function] [C:3]
# @PURPOSE: Translate generic httpx errors into NetworkError.
# @POST: Raises NetworkError with URL context.
# @DATA_CONTRACT: Input[httpx.HTTPError] -> NetworkError
# @RELATION: [DEPENDS_ON] ->[NetworkError]
# @POST Raises NetworkError with URL context.
# @DATA_CONTRACT Input[httpx.HTTPError] -> NetworkError
# @RELATION DEPENDS_ON -> [NetworkError]
def _handle_network_error(self, exc: httpx.HTTPError, url: str) -> None:
with belief_scope("AsyncAPIClient._handle_network_error"):
if isinstance(exc, httpx.TimeoutException):
@@ -259,9 +228,9 @@ class AsyncAPIClient:
# #endregion AsyncAPIClient._handle_network_error
# #region AsyncAPIClient.aclose [TYPE Function] [C:3]
# @PURPOSE: Close underlying httpx client.
# @POST: Client resources are released.
# @SIDE_EFFECT: Closes network connections.
# @RELATION: [DEPENDS_ON] ->[AsyncAPIClient.__init__]
# @POST Client resources are released.
# @SIDE_EFFECT Closes network connections.
# @RELATION DEPENDS_ON -> [AsyncAPIClient.__init__]
async def aclose(self) -> None:
await self._client.aclose()
# #endregion AsyncAPIClient.aclose

View File

@@ -2,10 +2,10 @@
#
# @BRIEF Этот модуль отвечает за обновление метаданных (verbose_name) в датасетах Superset,
# извлекая их из Superset SQL Lab (любая БД, подключённая к Superset) или XLSX-файлов.
# @LAYER: Domain
# @LAYER Domain
# @RELATION DEPENDS_ON -> backend.core.superset_client
# @RELATION DEPENDS_ON -> pandas
# @PUBLIC_API: DatasetMapper
# @RELATION DEPENDS_ON -> [EXT:Library:pandas]
# @PUBLIC_API DatasetMapper
from typing import Any
import pandas as pd # type: ignore
@@ -18,23 +18,23 @@ from ..logger import belief_scope, logger as app_logger
class DatasetMapper:
# #region __init__ [TYPE Function]
# @PURPOSE: Initializes the mapper.
# @POST: Объект DatasetMapper инициализирован.
# @POST Объект DatasetMapper инициализирован.
def __init__(self):
pass
# #endregion __init__
# #region get_sqllab_mappings [TYPE Function]
# @PURPOSE: Извлекает маппинги column_name -> verbose_name через SQL Lab Superset.
# @PRE: sqllab_executor должен быть инициализирован с database_id.
# @PRE: dataset_id должен существовать в Superset.
# @POST: Возвращается словарь column_name -> verbose_name из результатов SQL-запроса.
# @PRE sqllab_executor должен быть инициализирован с database_id.
# @PRE dataset_id должен существовать в Superset.
# @POST Возвращается словарь column_name -> verbose_name из результатов SQL-запроса.
# Если sql_query не указан, строится запрос к information_schema.columns.
# @PARAM: client (SupersetClient) - Авторизованный клиент Superset.
# @PARAM: dataset_id (int) - ID датасета (для получения table_name/schema).
# @PARAM: sqllab_executor (SupersetSqlLabExecutor) - Исполнитель SQL Lab.
# @PARAM: database_id (int) - ID базы данных в Superset.
# @PARAM: sql_query (Optional[str]) - Произвольный SQL-запрос (должен вернуть column_name + verbose_name).
# @RETURN: Dict[str, str] - Словарь column_name -> verbose_name.
# @PARAM client (SupersetClient) - Авторизованный клиент Superset.
# @PARAM dataset_id (int) - ID датасета (для получения table_name/schema).
# @PARAM sqllab_executor (SupersetSqlLabExecutor) - Исполнитель SQL Lab.
# @PARAM database_id (int) - ID базы данных в Superset.
# @PARAM sql_query (Optional[str]) - Произвольный SQL-запрос (должен вернуть column_name + verbose_name).
# @RETURN Dict[str, str] - Словарь column_name -> verbose_name.
def get_sqllab_mappings(
self,
client: Any,
@@ -89,11 +89,11 @@ class DatasetMapper:
# #region load_excel_mappings [TYPE Function]
# @PURPOSE: Загружает маппинги column_name -> verbose_name из XLSX файла.
# @PRE: file_path должен указывать на существующий XLSX файл.
# @POST: Возвращается словарь с маппингами из файла.
# @PRE file_path должен указывать на существующий XLSX файл.
# @POST Возвращается словарь с маппингами из файла.
# @THROW: Exception - При ошибках чтения файла или парсинга.
# @PARAM: file_path (str) - Путь к XLSX файлу.
# @RETURN: Dict[str, str] - Словарь с маппингами.
# @PARAM file_path (str) - Путь к XLSX файлу.
# @RETURN Dict[str, str] - Словарь с маппингами.
def load_excel_mappings(self, file_path: str) -> dict[str, str]:
with belief_scope("Load mappings from Excel"):
app_logger.info("[load_excel_mappings][Enter] Loading mappings from %s.", file_path)
@@ -109,20 +109,20 @@ class DatasetMapper:
# #region run_mapping [TYPE Function]
# @PURPOSE: Основная функция для выполнения маппинга и обновления verbose_name датасета в Superset.
# @PRE: superset_client должен быть авторизован.
# @PRE: dataset_id должен быть существующим ID в Superset.
# @POST: Если найдены изменения, датасет в Superset обновлен через API.
# @RELATION: CALLS -> self.get_sqllab_mappings
# @RELATION: CALLS -> self.load_excel_mappings
# @RELATION: CALLS -> superset_client.get_dataset
# @RELATION: CALLS -> superset_client.update_dataset
# @PARAM: superset_client (Any) - Клиент Superset.
# @PARAM: dataset_id (int) - ID датасета для обновления.
# @PARAM: source (str) - Источник данных ('sqllab' или 'excel').
# @PARAM: sqllab_executor (Optional[Any]) - Исполнитель SQL Lab (для sqllab source).
# @PARAM: database_id (Optional[int]) - ID базы данных Superset (для sqllab source).
# @PARAM: sql_query (Optional[str]) - Произвольный SQL-запрос (для sqllab source).
# @PARAM: excel_path (Optional[str]) - Путь к XLSX файлу.
# @PRE superset_client должен быть авторизован.
# @PRE dataset_id должен быть существующим ID в Superset.
# @POST Если найдены изменения, датасет в Superset обновлен через API.
# @RELATION CALLS -> [EXT:method:DatasetMapper.get_sqllab_mappings]
# @RELATION CALLS -> [EXT:method:DatasetMapper.load_excel_mappings]
# @RELATION CALLS -> [EXT:method:SupersetClient.get_dataset]
# @RELATION CALLS -> [EXT:method:SupersetClient.update_dataset]
# @PARAM superset_client (Any) - Клиент Superset.
# @PARAM dataset_id (int) - ID датасета для обновления.
# @PARAM source (str) - Источник данных ('sqllab' или 'excel').
# @PARAM sqllab_executor (Optional[Any]) - Исполнитель SQL Lab (для sqllab source).
# @PARAM database_id (Optional[int]) - ID базы данных Superset (для sqllab source).
# @PARAM sql_query (Optional[str]) - Произвольный SQL-запрос (для sqllab source).
# @PARAM excel_path (Optional[str]) - Путь к XLSX файлу.
def run_mapping(
self,
superset_client: Any,

View File

@@ -1,10 +1,8 @@
# #region FileIO [TYPE Module] [SEMANTICS fileio, archive, zip, yaml, utility]
#
# @TIER: STANDARD
# #region FileIO [C:3] [TYPE Module] [SEMANTICS fileio, archive, zip, yaml, utility]
# @BRIEF Предоставляет набор утилит для управления файловыми операциями, включая работу с временными файлами, архивами ZIP, файлами YAML и очистку директорий.
# @LAYER: Infra
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [LoggerModule]
# @PUBLIC_API: create_temp_file, remove_empty_directories, read_dashboard_from_disk, calculate_crc32, RetentionPolicy, archive_exports, save_and_unpack_dashboard, update_yamls, create_dashboard_export, sanitize_filename, get_filename_from_headers, consolidate_archive_folders
# @PUBLIC_API create_temp_file, remove_empty_directories, read_dashboard_from_disk, calculate_crc32, RetentionPolicy, archive_exports, save_and_unpack_dashboard, update_yamls, create_dashboard_export, sanitize_filename, get_filename_from_headers, consolidate_archive_folders
from collections.abc import Generator
from contextlib import contextmanager
from dataclasses import dataclass
@@ -28,9 +26,9 @@ class InvalidZipFormatError(Exception):
# #endregion InvalidZipFormatError
# #region create_temp_file [TYPE Function]
# @BRIEF Контекстный менеджер для создания временного файла или директории с гарантированным удалением.
# @PRE: suffix должен быть строкой, определяющей тип ресурса.
# @POST: Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста.
# @YIELDS: Path - Путь к временному ресурсу.
# @PRE suffix должен быть строкой, определяющей тип ресурса.
# @POST Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста.
# @YIELDS Path - Путь к временному ресурсу.
@contextmanager
def create_temp_file(content: bytes | None = None, suffix: str = ".zip", mode: str = 'wb', dry_run = False) -> Generator[Path]:
with belief_scope("Create temporary resource"):
@@ -64,8 +62,8 @@ def create_temp_file(content: bytes | None = None, suffix: str = ".zip", mode: s
# #endregion create_temp_file
# #region remove_empty_directories [TYPE Function]
# @BRIEF Рекурсивно удаляет все пустые поддиректории, начиная с указанного пути.
# @PRE: root_dir должен быть путем к существующей директории.
# @POST: Все пустые поддиректории удалены, возвращено их количество.
# @PRE root_dir должен быть путем к существующей директории.
# @POST Все пустые поддиректории удалены, возвращено их количество.
def remove_empty_directories(root_dir: str) -> int:
with belief_scope(f"Remove empty directories in {root_dir}"):
app_logger.info("[remove_empty_directories][Enter] Starting cleanup of empty directories in %s", root_dir)
@@ -86,8 +84,8 @@ def remove_empty_directories(root_dir: str) -> int:
# #endregion remove_empty_directories
# #region read_dashboard_from_disk [TYPE Function]
# @BRIEF Читает бинарное содержимое файла с диска.
# @PRE: file_path должен указывать на существующий файл.
# @POST: Возвращает байты содержимого и имя файла.
# @PRE file_path должен указывать на существующий файл.
# @POST Возвращает байты содержимого и имя файла.
def read_dashboard_from_disk(file_path: str) -> tuple[bytes, str]:
with belief_scope(f"Read dashboard from {file_path}"):
path = Path(file_path)
@@ -100,8 +98,8 @@ def read_dashboard_from_disk(file_path: str) -> tuple[bytes, str]:
# #endregion read_dashboard_from_disk
# #region calculate_crc32 [TYPE Function]
# @BRIEF Вычисляет контрольную сумму CRC32 для файла.
# @PRE: file_path должен быть объектом Path к существующему файлу.
# @POST: Возвращает 8-значную hex-строку CRC32.
# @PRE file_path должен быть объектом Path к существующему файлу.
# @POST Возвращает 8-значную hex-строку CRC32.
def calculate_crc32(file_path: Path) -> str:
with belief_scope(f"Calculate CRC32 for {file_path}"), open(file_path, 'rb') as f:
crc32_value = zlib.crc32(f.read())
@@ -117,10 +115,10 @@ class RetentionPolicy:
# #endregion RetentionPolicy
# #region archive_exports [TYPE Function]
# @BRIEF Управляет архивом экспортированных файлов, применяя политику хранения и дедупликацию.
# @PRE: output_dir должен быть путем к существующей директории.
# @POST: Старые или дублирующиеся архивы удалены согласно политике.
# @RELATION CALLS -> apply_retention_policy
# @RELATION CALLS -> calculate_crc32
# @PRE output_dir должен быть путем к существующей директории.
# @POST Старые или дублирующиеся архивы удалены согласно политике.
# @RELATION CALLS -> [apply_retention_policy]
# @RELATION CALLS -> [calculate_crc32]
def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool = False) -> None:
with belief_scope(f"Archive exports in {output_dir}"):
output_path = Path(output_dir)
@@ -192,8 +190,8 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
# #endregion archive_exports
# #region apply_retention_policy [TYPE Function]
# @BRIEF (Helper) Применяет политику хранения к списку файлов, возвращая те, что нужно сохранить.
# @PRE: files_with_dates is a list of (Path, date) tuples.
# @POST: Returns a set of files to keep.
# @PRE files_with_dates is a list of (Path, date) tuples.
# @POST Returns a set of files to keep.
def apply_retention_policy(files_with_dates: list[tuple[Path, date]], policy: RetentionPolicy) -> set:
with belief_scope("Apply retention policy"):
# Сортируем по дате (от новой к старой)
@@ -223,8 +221,8 @@ def apply_retention_policy(files_with_dates: list[tuple[Path, date]], policy: Re
# #endregion apply_retention_policy
# #region save_and_unpack_dashboard [TYPE Function]
# @BRIEF Сохраняет бинарное содержимое ZIP-архива на диск и опционально распаковывает его.
# @PRE: zip_content должен быть байтами валидного ZIP-архива.
# @POST: ZIP-файл сохранен, и если unpack=True, он распакован в output_dir.
# @PRE zip_content должен быть байтами валидного ZIP-архива.
# @POST ZIP-файл сохранен, и если unpack=True, он распакован в output_dir.
def save_and_unpack_dashboard(zip_content: bytes, output_dir: str | Path, unpack: bool = False, original_filename: str | None = None) -> tuple[Path, Path | None]:
with belief_scope("Save and unpack dashboard"):
app_logger.info("[save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: %s", unpack)
@@ -247,9 +245,9 @@ def save_and_unpack_dashboard(zip_content: bytes, output_dir: str | Path, unpack
# #endregion save_and_unpack_dashboard
# #region update_yamls [TYPE Function]
# @BRIEF Обновляет конфигурации в YAML-файлах, заменяя значения или применяя regex.
# @PRE: path должен быть существующей директорией.
# @POST: Все YAML файлы в директории обновлены согласно переданным параметрам.
# @RELATION CALLS -> _update_yaml_file
# @PRE path должен быть существующей директорией.
# @POST Все YAML файлы в директории обновлены согласно переданным параметрам.
# @RELATION CALLS -> [_update_yaml_file]
def update_yamls(db_configs: list[dict[str, Any]] | None = None, path: str = "dashboards", regexp_pattern: LiteralString | None = None, replace_string: LiteralString | None = None) -> None:
with belief_scope("Update YAML configurations"):
app_logger.info("[update_yamls][Enter] Starting YAML configuration update.")
@@ -263,8 +261,8 @@ def update_yamls(db_configs: list[dict[str, Any]] | None = None, path: str = "da
# #endregion update_yamls
# #region _update_yaml_file [TYPE Function]
# @BRIEF (Helper) Обновляет один YAML файл.
# @PRE: file_path должен быть объектом Path к существующему YAML файлу.
# @POST: Файл обновлен согласно переданным конфигурациям или регулярному выражению.
# @PRE file_path должен быть объектом Path к существующему YAML файлу.
# @POST Файл обновлен согласно переданным конфигурациям или регулярному выражению.
def _update_yaml_file(file_path: Path, db_configs: list[dict[str, Any]], regexp_pattern: str | None, replace_string: str | None) -> None:
with belief_scope(f"Update YAML file: {file_path}"):
# Читаем содержимое файла
@@ -306,8 +304,8 @@ def _update_yaml_file(file_path: Path, db_configs: list[dict[str, Any]], regexp_
# #region replacer [TYPE Function]
# @PURPOSE: Функция замены, сохраняющая кавычки если они были.
# @PRE: match должен быть объектом совпадения регулярного выражения.
# @POST: Возвращает строку с новым значением, сохраняя префикс и кавычки.
# @PRE match должен быть объектом совпадения регулярного выражения.
# @POST Возвращает строку с новым значением, сохраняя префикс и кавычки.
def replacer(match):
prefix = match.group(1)
quote_open = match.group(2)
@@ -324,8 +322,8 @@ def _update_yaml_file(file_path: Path, db_configs: list[dict[str, Any]], regexp_
# #endregion _update_yaml_file
# #region create_dashboard_export [TYPE Function]
# @BRIEF Создает ZIP-архив из указанных исходных путей.
# @PRE: source_paths должен содержать существующие пути.
# @POST: ZIP-архив создан по пути zip_path.
# @PRE source_paths должен содержать существующие пути.
# @POST ZIP-архив создан по пути zip_path.
def create_dashboard_export(zip_path: str | Path, source_paths: list[str | Path], exclude_extensions: list[str] | None = None) -> bool:
with belief_scope(f"Create dashboard export: {zip_path}"):
app_logger.info("[create_dashboard_export][Enter] Packing dashboard: %s -> %s", source_paths, zip_path)
@@ -347,16 +345,16 @@ def create_dashboard_export(zip_path: str | Path, source_paths: list[str | Path]
# #endregion create_dashboard_export
# #region sanitize_filename [TYPE Function]
# @BRIEF Очищает строку от символов, недопустимых в именах файлов.
# @PRE: filename должен быть строкой.
# @POST: Возвращает строку без спецсимволов.
# @PRE filename должен быть строкой.
# @POST Возвращает строку без спецсимволов.
def sanitize_filename(filename: str) -> str:
with belief_scope(f"Sanitize filename: {filename}"):
return re.sub(r'[\\/*?:"<>|]', "_", filename).strip()
# #endregion sanitize_filename
# #region get_filename_from_headers [TYPE Function]
# @BRIEF Извлекает имя файла из HTTP заголовка 'Content-Disposition'.
# @PRE: headers должен быть словарем заголовков.
# @POST: Возвращает имя файла или None, если заголовок отсутствует.
# @PRE headers должен быть словарем заголовков.
# @POST Возвращает имя файла или None, если заголовок отсутствует.
def get_filename_from_headers(headers: dict) -> str | None:
with belief_scope("Get filename from headers"):
content_disposition = headers.get("Content-Disposition", "")
@@ -366,8 +364,8 @@ def get_filename_from_headers(headers: dict) -> str | None:
# #endregion get_filename_from_headers
# #region consolidate_archive_folders [TYPE Function]
# @BRIEF Консолидирует директории архивов на основе общего слага в имени.
# @PRE: root_directory должен быть объектом Path к существующей директории.
# @POST: Директории с одинаковым префиксом объединены в одну.
# @PRE root_directory должен быть объектом Path к существующей директории.
# @POST Директории с одинаковым префиксом объединены в одну.
def consolidate_archive_folders(root_directory: Path) -> None:
with belief_scope(f"Consolidate archives in {root_directory}"):
assert isinstance(root_directory, Path), "root_directory must be a Path object."

View File

@@ -1,10 +1,10 @@
# #region FuzzyMatching [TYPE Module] [SEMANTICS fuzzy, matching, rapidfuzz, database, mapping]
#
# @BRIEF Provides utility functions for fuzzy matching database names.
# @LAYER: Core
# @RELATION DEPENDS_ON -> rapidfuzz
# @LAYER Core
# @RELATION DEPENDS_ON -> [EXT:Library:rapidfuzz]
#
# @INVARIANT: Confidence scores are returned as floats between 0.0 and 1.0.
# @INVARIANT Confidence scores are returned as floats between 0.0 and 1.0.
from rapidfuzz import fuzz, process
@@ -12,8 +12,8 @@ from rapidfuzz import fuzz, process
# #region suggest_mappings [TYPE Function]
# @BRIEF Suggests mappings between source and target databases using fuzzy matching.
# @PRE: source_databases and target_databases are lists of dictionaries with 'uuid' and 'database_name'.
# @POST: Returns a list of suggested mappings with confidence scores.
# @PRE source_databases and target_databases are lists of dictionaries with 'uuid' and 'database_name'.
# @POST Returns a list of suggested mappings with confidence scores.
def suggest_mappings(source_databases: list[dict], target_databases: list[dict], threshold: int = 60) -> list[dict]:
"""
Suggest mappings between source and target databases using fuzzy matching.

View File

@@ -1,9 +1,9 @@
# #region NetworkModule [C:3] [TYPE Module] [SEMANTICS network, http, retry, tenacity, session]
#
# @BRIEF Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API, включая аутентификацию, управление сессией, retry-логику и обработку ошибок.
# @LAYER: Infra
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [LoggerModule]
# @PUBLIC_API: APIClient
# @PUBLIC_API APIClient
import io
import json
from pathlib import Path
@@ -24,8 +24,8 @@ from ..logger import belief_scope, logger as app_logger
class SupersetAPIError(Exception):
# #region __init__ [TYPE Function] [C:1]
# @PURPOSE: Initializes the exception with a message and context.
# @PRE: message is a string, context is a dict.
# @POST: Exception is initialized with context.
# @PRE message is a string, context is a dict.
# @POST Exception is initialized with context.
def __init__(self, message: str = "Superset API error", **context: Any):
with belief_scope("SupersetAPIError.__init__"):
self.context = context
@@ -37,8 +37,8 @@ class SupersetAPIError(Exception):
class AuthenticationError(SupersetAPIError):
# #region __init__ [TYPE Function] [C:1]
# @PURPOSE: Initializes the authentication error.
# @PRE: message is a string, context is a dict.
# @POST: AuthenticationError is initialized.
# @PRE message is a string, context is a dict.
# @POST AuthenticationError is initialized.
def __init__(self, message: str = "Authentication failed", **context: Any):
with belief_scope("AuthenticationError.__init__"):
super().__init__(message, type="authentication", **context)
@@ -49,8 +49,8 @@ class AuthenticationError(SupersetAPIError):
class PermissionDeniedError(AuthenticationError):
# #region __init__ [TYPE Function]
# @PURPOSE: Initializes the permission denied error.
# @PRE: message is a string, context is a dict.
# @POST: PermissionDeniedError is initialized.
# @PRE message is a string, context is a dict.
# @POST PermissionDeniedError is initialized.
def __init__(self, message: str = "Permission denied", **context: Any):
with belief_scope("PermissionDeniedError.__init__"):
super().__init__(message, **context)
@@ -61,8 +61,8 @@ class PermissionDeniedError(AuthenticationError):
class DashboardNotFoundError(SupersetAPIError):
# #region __init__ [TYPE Function]
# @PURPOSE: Initializes the not found error with resource ID.
# @PRE: resource_id is provided.
# @POST: DashboardNotFoundError is initialized.
# @PRE resource_id is provided.
# @POST DashboardNotFoundError is initialized.
def __init__(self, resource_id: int | str, message: str = "Dashboard not found", **context: Any):
with belief_scope("DashboardNotFoundError.__init__"):
super().__init__(f"Dashboard '{resource_id}' {message}", subtype="not_found", resource_id=resource_id, **context)
@@ -73,8 +73,8 @@ class DashboardNotFoundError(SupersetAPIError):
class NetworkError(Exception):
# #region NetworkError.__init__ [TYPE Function]
# @PURPOSE: Initializes the network error.
# @PRE: message is a string.
# @POST: NetworkError is initialized.
# @PRE message is a string.
# @POST NetworkError is initialized.
def __init__(self, message: str = "Network connection failed", **context: Any):
with belief_scope("NetworkError.__init__"):
self.context = context
@@ -83,8 +83,8 @@ class NetworkError(Exception):
# #endregion NetworkError
# #region SupersetAuthCache [TYPE Class]
# @BRIEF Process-local cache for Superset access/csrf tokens keyed by environment credentials.
# @PRE: base_url and username are stable strings.
# @POST: Cached entries expire automatically by TTL and can be reused across requests.
# @PRE base_url and username are stable strings.
# @POST Cached entries expire automatically by TTL and can be reused across requests.
class SupersetAuthCache:
TTL_SECONDS = 300
_lock = threading.Lock()
@@ -142,11 +142,11 @@ class APIClient:
DEFAULT_TIMEOUT = 30
# #region APIClient.__init__ [TYPE Function]
# @PURPOSE: Инициализирует API клиент с конфигурацией, сессией и логгером.
# @PARAM: config (Dict[str, Any]) - Конфигурация.
# @PARAM: verify_ssl (bool) - Проверять ли SSL.
# @PARAM: timeout (int) - Таймаут запросов.
# @PRE: config must contain 'base_url' and 'auth'.
# @POST: APIClient instance is initialized with a session.
# @PARAM config (Dict[str, Any]) - Конфигурация.
# @PARAM verify_ssl (bool) - Проверять ли SSL.
# @PARAM timeout (int) - Таймаут запросов.
# @PRE config must contain 'base_url' and 'auth'.
# @POST APIClient instance is initialized with a session.
def __init__(self, config: dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
with belief_scope("__init__"):
app_logger.reason("Initializing APIClient.", extra={"src": "APIClient.__init__"})
@@ -166,9 +166,9 @@ class APIClient:
# #endregion APIClient.__init__
# #region _init_session [TYPE Function]
# @PURPOSE: Создает и настраивает `requests.Session` с retry-логикой.
# @PRE: self.request_settings must be initialized.
# @POST: Returns a configured requests.Session instance.
# @RETURN: requests.Session - Настроенная сессия.
# @PRE self.request_settings must be initialized.
# @POST Returns a configured requests.Session instance.
# @RETURN requests.Session - Настроенная сессия.
def _init_session(self) -> requests.Session:
with belief_scope("_init_session"):
session = requests.Session()
@@ -213,9 +213,9 @@ class APIClient:
# #region _normalize_base_url [TYPE Function]
# @PURPOSE: Normalize Superset environment URL to base host/path without trailing slash and /api/v1 suffix.
# Auto-prepends https:// if no scheme is present.
# @PRE: raw_url can be empty.
# @POST: Returns canonical base URL with scheme, suitable for building API endpoints.
# @RETURN: str
# @PRE raw_url can be empty.
# @POST Returns canonical base URL with scheme, suitable for building API endpoints.
# @RETURN str
def _normalize_base_url(self, raw_url: str) -> str:
normalized = str(raw_url or "").strip().rstrip("/")
if normalized.lower().endswith("/api/v1"):
@@ -228,9 +228,9 @@ class APIClient:
# #endregion _normalize_base_url
# #region _build_api_url [TYPE Function]
# @PURPOSE: Build absolute Superset API URL for endpoint using canonical /api/v1 base.
# @PRE: endpoint is relative path or absolute URL.
# @POST: Returns full URL without accidental duplicate slashes.
# @RETURN: str
# @PRE endpoint is relative path or absolute URL.
# @POST Returns full URL without accidental duplicate slashes.
# @RETURN str
def _build_api_url(self, endpoint: str) -> str:
normalized_endpoint = str(endpoint or "").strip()
if normalized_endpoint.startswith("http://") or normalized_endpoint.startswith("https://"):
@@ -243,12 +243,12 @@ class APIClient:
# #endregion _build_api_url
# #region APIClient.authenticate [TYPE Function]
# @PURPOSE: Выполняет аутентификацию в Superset API и получает access и CSRF токены.
# @PRE: self.auth and self.base_url must be valid.
# @POST: `self._tokens` заполнен, `self._authenticated` установлен в `True`.
# @RETURN: Dict[str, str] - Словарь с токенами.
# @PRE self.auth and self.base_url must be valid.
# @POST `self._tokens` заполнен, `self._authenticated` установлен в `True`.
# @RETURN Dict[str, str] - Словарь с токенами.
# @THROW: AuthenticationError, NetworkError - при ошибках.
# @RELATION: [CALLS] ->[SupersetAuthCache.get]
# @RELATION: [CALLS] ->[SupersetAuthCache.set]
# @RELATION CALLS -> [SupersetAuthCache.get]
# @RELATION CALLS -> [SupersetAuthCache.set]
def authenticate(self) -> dict[str, str]:
with belief_scope("authenticate"):
app_logger.info("[authenticate][Enter] Authenticating to %s", self.base_url)
@@ -329,8 +329,8 @@ class APIClient:
@property
# #region headers [TYPE Function]
# @PURPOSE: Возвращает HTTP-заголовки для аутентифицированных запросов.
# @PRE: APIClient is initialized and authenticated or can be authenticated.
# @POST: Returns headers including auth tokens.
# @PRE APIClient is initialized and authenticated or can be authenticated.
# @POST Returns headers including auth tokens.
def headers(self) -> dict[str, str]:
if not self._authenticated:
self.authenticate()
@@ -343,13 +343,13 @@ class APIClient:
# #endregion headers
# #region request [TYPE Function]
# @PURPOSE: Выполняет универсальный HTTP-запрос к API.
# @PARAM: method (str) - HTTP метод.
# @PARAM: endpoint (str) - API эндпоинт.
# @PARAM: headers (Optional[Dict]) - Дополнительные заголовки.
# @PARAM: raw_response (bool) - Возвращать ли сырой ответ.
# @PRE: method and endpoint must be strings.
# @POST: Returns response content or raw Response object.
# @RETURN: `requests.Response` если `raw_response=True`, иначе `dict`.
# @PARAM method (str) - HTTP метод.
# @PARAM endpoint (str) - API эндпоинт.
# @PARAM headers (Optional[Dict]) - Дополнительные заголовки.
# @PARAM raw_response (bool) - Возвращать ли сырой ответ.
# @PRE method and endpoint must be strings.
# @POST Returns response content or raw Response object.
# @RETURN `requests.Response` если `raw_response=True`, иначе `dict`.
# @THROW: SupersetAPIError, NetworkError и их подклассы.
def request(self, method: str, endpoint: str, headers: dict | None = None, raw_response: bool = False, **kwargs) -> requests.Response | dict[str, Any]:
full_url = self._build_api_url(endpoint)
@@ -372,10 +372,10 @@ class APIClient:
# #endregion request
# #region _handle_http_error [TYPE Function]
# @PURPOSE: (Helper) Преобразует HTTP ошибки в кастомные исключения.
# @PARAM: e (requests.exceptions.HTTPError) - Ошибка.
# @PARAM: endpoint (str) - Эндпоинт.
# @PRE: e must be a valid HTTPError with a response.
# @POST: Raises a specific SupersetAPIError or subclass.
# @PARAM e (requests.exceptions.HTTPError) - Ошибка.
# @PARAM endpoint (str) - Эндпоинт.
# @PRE e must be a valid HTTPError with a response.
# @POST Raises a specific SupersetAPIError or subclass.
def _handle_http_error(self, e: requests.exceptions.HTTPError, endpoint: str):
with belief_scope("_handle_http_error"):
status_code = e.response.status_code
@@ -398,8 +398,8 @@ class APIClient:
# #endregion _handle_http_error
# #region _is_dashboard_endpoint [TYPE Function]
# @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.
# @PRE: endpoint may be relative or absolute.
# @POST: Returns true only for dashboard-specific endpoints.
# @PRE endpoint may be relative or absolute.
# @POST Returns true only for dashboard-specific endpoints.
def _is_dashboard_endpoint(self, endpoint: str) -> bool:
normalized_endpoint = str(endpoint or "").strip().lower()
if not normalized_endpoint:
@@ -415,10 +415,10 @@ class APIClient:
# #endregion _is_dashboard_endpoint
# #region _handle_network_error [TYPE Function]
# @PURPOSE: (Helper) Преобразует сетевые ошибки в `NetworkError`.
# @PARAM: e (requests.exceptions.RequestException) - Ошибка.
# @PARAM: url (str) - URL.
# @PRE: e must be a RequestException.
# @POST: Raises a NetworkError.
# @PARAM e (requests.exceptions.RequestException) - Ошибка.
# @PARAM url (str) - URL.
# @PRE e must be a RequestException.
# @POST Raises a NetworkError.
def _handle_network_error(self, e: requests.exceptions.RequestException, url: str):
with belief_scope("_handle_network_error"):
if isinstance(e, requests.exceptions.Timeout):
@@ -431,13 +431,13 @@ class APIClient:
# #endregion _handle_network_error
# #region upload_file [TYPE Function]
# @PURPOSE: Загружает файл на сервер через multipart/form-data.
# @PARAM: endpoint (str) - Эндпоинт.
# @PARAM: file_info (Dict[str, Any]) - Информация о файле.
# @PARAM: extra_data (Optional[Dict]) - Дополнительные данные.
# @PARAM: timeout (Optional[int]) - Таймаут.
# @PRE: file_info must contain 'file_obj' and 'file_name'.
# @POST: File is uploaded and response returned.
# @RETURN: Ответ API в виде словаря.
# @PARAM endpoint (str) - Эндпоинт.
# @PARAM file_info (Dict[str, Any]) - Информация о файле.
# @PARAM extra_data (Optional[Dict]) - Дополнительные данные.
# @PARAM timeout (Optional[int]) - Таймаут.
# @PRE file_info must contain 'file_obj' and 'file_name'.
# @POST File is uploaded and response returned.
# @RETURN Ответ API в виде словаря.
# @THROW: SupersetAPIError, NetworkError, TypeError.
def upload_file(self, endpoint: str, file_info: dict[str, Any], extra_data: dict | None = None, timeout: int | None = None) -> dict:
with belief_scope("upload_file"):
@@ -461,14 +461,14 @@ class APIClient:
# #endregion upload_file
# #region _perform_upload [TYPE Function]
# @PURPOSE: (Helper) Выполняет POST запрос с файлом.
# @PARAM: url (str) - URL.
# @PARAM: files (Dict) - Файлы.
# @PARAM: data (Optional[Dict]) - Данные.
# @PARAM: headers (Dict) - Заголовки.
# @PARAM: timeout (Optional[int]) - Таймаут.
# @PRE: url, files, and headers must be provided.
# @POST: POST request is performed and JSON response returned.
# @RETURN: Dict - Ответ.
# @PARAM url (str) - URL.
# @PARAM files (Dict) - Файлы.
# @PARAM data (Optional[Dict]) - Данные.
# @PARAM headers (Dict) - Заголовки.
# @PARAM timeout (Optional[int]) - Таймаут.
# @PRE url, files, and headers must be provided.
# @POST POST request is performed and JSON response returned.
# @RETURN Dict - Ответ.
def _perform_upload(self, url: str, files: dict, data: dict | None, headers: dict, timeout: int | None) -> dict:
with belief_scope("_perform_upload"):
try:
@@ -488,12 +488,12 @@ class APIClient:
# #endregion _perform_upload
# #region fetch_paginated_count [TYPE Function]
# @PURPOSE: Получает общее количество элементов для пагинации.
# @PARAM: endpoint (str) - Эндпоинт.
# @PARAM: query_params (Dict) - Параметры запроса.
# @PARAM: count_field (str) - Поле с количеством.
# @PRE: query_params must be a dictionary.
# @POST: Returns total count of items.
# @RETURN: int - Количество.
# @PARAM endpoint (str) - Эндпоинт.
# @PARAM query_params (Dict) - Параметры запроса.
# @PARAM count_field (str) - Поле с количеством.
# @PRE query_params must be a dictionary.
# @POST Returns total count of items.
# @RETURN int - Количество.
def fetch_paginated_count(self, endpoint: str, query_params: dict, count_field: str = "count") -> int:
with belief_scope("fetch_paginated_count"):
response_json = cast(dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query_params)}))
@@ -501,11 +501,11 @@ class APIClient:
# #endregion fetch_paginated_count
# #region fetch_paginated_data [TYPE Function]
# @PURPOSE: Автоматически собирает данные со всех страниц пагинированного эндпоинта.
# @PARAM: endpoint (str) - Эндпоинт.
# @PARAM: pagination_options (Dict[str, Any]) - Опции пагинации.
# @PRE: pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional.
# @POST: Returns all items across all pages.
# @RETURN: List[Any] - Список данных.
# @PARAM endpoint (str) - Эндпоинт.
# @PARAM pagination_options (Dict[str, Any]) - Опции пагинации.
# @PRE pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional.
# @POST Returns all items across all pages.
# @RETURN List[Any] - Список данных.
def fetch_paginated_data(self, endpoint: str, pagination_options: dict[str, Any]) -> list[Any]:
with belief_scope("fetch_paginated_data"):
base_query = pagination_options["base_query"]

View File

@@ -1,12 +1,12 @@
# #region SupersetCompilationAdapter [C:4] [TYPE Module] [SEMANTICS superset, preview, compile, execution, adapter]
# @BRIEF Interact with Superset preview compilation and SQL Lab execution endpoints using the current approved execution context.
# @LAYER: Infra
# @LAYER Infrastructure
# @RELATION CALLS -> [SupersetClient]
# @RELATION DEPENDS_ON -> [CompiledPreview]
# @PRE: effective template params and dataset execution reference are available.
# @POST: preview and launch calls return Superset-originated artifacts or explicit errors.
# @SIDE_EFFECT: performs upstream Superset preview and SQL Lab calls.
# @INVARIANT: The adapter never fabricates compiled SQL locally; preview truth is delegated to Superset only.
# @PRE effective template params and dataset execution reference are available.
# @POST preview and launch calls return Superset-originated artifacts or explicit errors.
# @SIDE_EFFECT performs upstream Superset preview and SQL Lab calls.
# @INVARIANT The adapter never fabricates compiled SQL locally; preview truth is delegated to Superset only.
from __future__ import annotations
# #region SupersetCompilationAdapter.imports [TYPE Block]
@@ -44,9 +44,9 @@ class SqlLabLaunchPayload:
# #region SupersetCompilationAdapter [C:4] [TYPE Class]
# @BRIEF Delegate preview compilation and SQL Lab launch to Superset without local SQL fabrication.
# @RELATION CALLS -> [SupersetClient]
# @PRE: environment is configured and Superset is reachable for the target session.
# @POST: adapter can return explicit ready/failed preview artifacts and canonical SQL Lab references.
# @SIDE_EFFECT: issues network requests to Superset API surfaces.
# @PRE environment is configured and Superset is reachable for the target session.
# @POST adapter can return explicit ready/failed preview artifacts and canonical SQL Lab references.
# @SIDE_EFFECT issues network requests to Superset API surfaces.
class SupersetCompilationAdapter:
# #region SupersetCompilationAdapter.__init__ [TYPE Function] [C:2]
# @PURPOSE: Bind adapter to one Superset environment and client instance.
@@ -66,11 +66,11 @@ class SupersetCompilationAdapter:
# #endregion SupersetCompilationAdapter._supports_client_method
# #region SupersetCompilationAdapter.compile_preview [TYPE Function] [C:4]
# @PURPOSE: Request Superset-side compiled SQL preview for the current effective inputs.
# @RELATION: [CALLS] ->[SupersetCompilationAdapter._request_superset_preview]
# @PRE: dataset_id and effective inputs are available for the current session.
# @POST: returns a ready or failed preview artifact backed only by Superset-originated SQL or diagnostics.
# @SIDE_EFFECT: performs upstream preview requests.
# @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[CompiledPreview]
# @RELATION CALLS -> [SupersetCompilationAdapter._request_superset_preview]
# @PRE dataset_id and effective inputs are available for the current session.
# @POST returns a ready or failed preview artifact backed only by Superset-originated SQL or diagnostics.
# @SIDE_EFFECT performs upstream preview requests.
# @DATA_CONTRACT Input[PreviewCompilationPayload] -> Output[CompiledPreview]
def compile_preview(self, payload: PreviewCompilationPayload) -> CompiledPreview:
with belief_scope("SupersetCompilationAdapter.compile_preview"):
if payload.dataset_id <= 0:
@@ -154,19 +154,19 @@ class SupersetCompilationAdapter:
# #endregion SupersetCompilationAdapter.compile_preview
# #region SupersetCompilationAdapter.mark_preview_stale [TYPE Function] [C:2]
# @PURPOSE: Invalidate previous preview after mapping or value changes.
# @PRE: preview is a persisted preview artifact or current in-memory snapshot.
# @POST: preview status becomes stale without fabricating a replacement artifact.
# @PRE preview is a persisted preview artifact or current in-memory snapshot.
# @POST preview status becomes stale without fabricating a replacement artifact.
def mark_preview_stale(self, preview: CompiledPreview) -> CompiledPreview:
preview.preview_status = PreviewStatus.STALE
return preview
# #endregion SupersetCompilationAdapter.mark_preview_stale
# #region SupersetCompilationAdapter.create_sql_lab_session [TYPE Function] [C:4]
# @PURPOSE: Create the canonical audited execution session after all launch gates pass.
# @RELATION: [CALLS] ->[SupersetCompilationAdapter._request_sql_lab_session]
# @PRE: compiled_sql is Superset-originated and launch gates are already satisfied.
# @POST: returns one canonical SQL Lab session reference from Superset.
# @SIDE_EFFECT: performs upstream SQL Lab execution/session creation.
# @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[str]
# @RELATION CALLS -> [SupersetCompilationAdapter._request_sql_lab_session]
# @PRE compiled_sql is Superset-originated and launch gates are already satisfied.
# @POST returns one canonical SQL Lab session reference from Superset.
# @SIDE_EFFECT performs upstream SQL Lab execution/session creation.
# @DATA_CONTRACT Input[SqlLabLaunchPayload] -> Output[str]
def create_sql_lab_session(self, payload: SqlLabLaunchPayload) -> str:
with belief_scope("SupersetCompilationAdapter.create_sql_lab_session"):
compiled_sql = str(payload.compiled_sql or "").strip()
@@ -218,11 +218,11 @@ class SupersetCompilationAdapter:
# #endregion SupersetCompilationAdapter.create_sql_lab_session
# #region SupersetCompilationAdapter._request_superset_preview [TYPE Function] [C:4]
# @PURPOSE: Request preview compilation through explicit client support backed by real Superset endpoints only.
# @RELATION: [CALLS] ->[SupersetClient.compile_dataset_preview]
# @PRE: payload contains a valid dataset identifier and deterministic execution inputs for one preview attempt.
# @POST: returns one normalized upstream compilation response including the chosen strategy metadata.
# @SIDE_EFFECT: issues one or more Superset preview requests through the client fallback chain.
# @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[Dict[str,Any]]
# @RELATION CALLS -> [EXT:method:SupersetClient.compile_dataset_preview]
# @PRE payload contains a valid dataset identifier and deterministic execution inputs for one preview attempt.
# @POST returns one normalized upstream compilation response including the chosen strategy metadata.
# @SIDE_EFFECT issues one or more Superset preview requests through the client fallback chain.
# @DATA_CONTRACT Input[PreviewCompilationPayload] -> Output[Dict[str,Any]]
def _request_superset_preview(
self, payload: PreviewCompilationPayload
) -> dict[str, Any]:
@@ -356,11 +356,11 @@ class SupersetCompilationAdapter:
# #endregion SupersetCompilationAdapter._request_superset_preview
# #region SupersetCompilationAdapter._request_sql_lab_session [TYPE Function] [C:4]
# @PURPOSE: Probe supported SQL Lab execution surfaces and return the first successful response.
# @RELATION: [CALLS] ->[SupersetClient.get_dataset]
# @PRE: payload carries non-empty Superset-originated SQL and a preview identifier for the current launch.
# @POST: returns the first successful SQL Lab execution response from Superset.
# @SIDE_EFFECT: issues Superset dataset lookup and SQL Lab execution requests.
# @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[Dict[str,Any]]
# @RELATION CALLS -> [EXT:method:SupersetClient.get_dataset]
# @PRE payload carries non-empty Superset-originated SQL and a preview identifier for the current launch.
# @POST returns the first successful SQL Lab execution response from Superset.
# @SIDE_EFFECT issues Superset dataset lookup and SQL Lab execution requests.
# @DATA_CONTRACT Input[SqlLabLaunchPayload] -> Output[Dict[str,Any]]
def _request_sql_lab_session(self, payload: SqlLabLaunchPayload) -> dict[str, Any]:
dataset_raw = self.client.get_dataset(payload.dataset_id)
dataset_record = (
@@ -411,7 +411,7 @@ class SupersetCompilationAdapter:
# #endregion SupersetCompilationAdapter._request_sql_lab_session
# #region SupersetCompilationAdapter._normalize_preview_response [TYPE Function] [C:3]
# @PURPOSE: Normalize candidate Superset preview responses into one compiled-sql structure.
# @RELATION: [DEPENDS_ON] ->[CompiledPreview]
# @RELATION DEPENDS_ON -> [CompiledPreview]
def _normalize_preview_response(self, response: Any) -> dict[str, Any] | None:
if not isinstance(response, dict):
return None

View File

@@ -1,19 +1,19 @@
# #region SupersetContextExtractorPackage [C:4] [TYPE Module] [SEMANTICS superset, package, dashboard, dataset]
# @LAYER: Infra
# @LAYER Infrastructure
# @BRIEF Recover dataset and dashboard context from Superset links while preserving explicit partial-recovery markers.
# @RELATION DEPENDS_ON -> [ImportedFilter]
# @RELATION DEPENDS_ON -> [TemplateVariable]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
# @POST: Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
# @SIDE_EFFECT: Performs upstream Superset API reads.
# @INVARIANT: Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context.
# @PRE Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
# @POST Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
# @SIDE_EFFECT Performs upstream Superset API reads.
# @INVARIANT Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context.
#
# @RATIONALE: Decomposed from monolithic superset_context_extractor.py (1397 lines) into
# @RATIONALE Decomposed from monolithic superset_context_extractor.py (1397 lines) into
# domain-scoped mixins to satisfy INV_7 (module < 400 lines). The composed class
# preserves the original public API surface — all consumers continue to import
# from `src.core.utils.superset_context_extractor` without changes.
# @REJECTED: Keeping a single 1397-line file — violates fractal limit INV_7.
# @REJECTED Keeping a single 1397-line file — violates fractal limit INV_7.
# #endregion SupersetContextExtractorPackage
from ._base import SupersetContextExtractorBase, SupersetParsedContext
@@ -32,9 +32,9 @@ from ._templates import SupersetContextTemplatesMixin
# @RELATION INHERITS -> [SupersetContextRecoveryMixin]
# @RELATION INHERITS -> [SupersetContextTemplatesMixin]
# @RELATION INHERITS -> [SupersetContextFiltersExtractMixin]
# @PRE: constructor receives a configured environment with a usable Superset base URL.
# @POST: extractor instance is ready to parse links against one Superset environment.
# @SIDE_EFFECT: downstream parse operations may call Superset APIs through SupersetClient.
# @PRE constructor receives a configured environment with a usable Superset base URL.
# @POST extractor instance is ready to parse links against one Superset environment.
# @SIDE_EFFECT downstream parse operations may call Superset APIs through SupersetClient.
class SupersetContextExtractor(
SupersetContextFiltersExtractMixin,
SupersetContextRecoveryMixin,

View File

@@ -1,13 +1,13 @@
# #region SupersetContextExtractorBase [C:4] [TYPE Module] [SEMANTICS superset, context, extract, base, recovery]
# @LAYER: Infra
# @LAYER Infrastructure
# @BRIEF Base class and helpers for recovering dataset/dashboard context from Superset links with explicit partial-recovery markers.
# @RELATION DEPENDS_ON -> [ImportedFilter]
# @RELATION DEPENDS_ON -> [TemplateVariable]
# @RELATION CALLS -> [SupersetClient]
# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
# @POST: Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
# @SIDE_EFFECT: Performs upstream Superset API reads.
# @INVARIANT: Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context.
# @PRE Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
# @POST Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
# @SIDE_EFFECT Performs upstream Superset API reads.
# @INVARIANT Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context.
# #endregion SupersetContextExtractorBase
# #region _base_imports [TYPE Block]
from __future__ import annotations
@@ -42,9 +42,9 @@ class SupersetParsedContext:
# #region SupersetContextExtractorBase [C:4] [TYPE Class]
# @BRIEF Bind extractor to one Superset environment and client instance; provide shared URL-parsing helpers.
# @RELATION DEPENDS_ON -> [Environment]
# @PRE: constructor receives a configured environment with a usable Superset base URL.
# @POST: extractor instance is ready to parse links against one Superset environment.
# @SIDE_EFFECT: downstream parse operations may call Superset APIs through SupersetClient.
# @PRE constructor receives a configured environment with a usable Superset base URL.
# @POST extractor instance is ready to parse links against one Superset environment.
# @SIDE_EFFECT downstream parse operations may call Superset APIs through SupersetClient.
class SupersetContextExtractorBase:
# #region SupersetContextExtractorBase.__init__ [TYPE Function] [C:2]
# @PURPOSE: Bind extractor to one Superset environment and client instance.
@@ -138,7 +138,7 @@ class SupersetContextExtractorBase:
# #endregion SupersetContextExtractorBase._extract_chart_id_from_state
# #region SupersetContextExtractorBase._search_nested_numeric_key [TYPE Function] [C:3]
# @PURPOSE: Recursively search nested dict/list payloads for the first numeric value under a candidate key set.
# @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase]
# @RELATION DEPENDS_ON -> [SupersetContextExtractorBase]
def _search_nested_numeric_key(
self, payload: Any, candidate_keys: set[str]
) -> int | None:

View File

@@ -1,5 +1,5 @@
# #region SupersetContextFiltersExtractMixin [C:3] [TYPE Module] [SEMANTICS superset, filter, native, context, extract]
# @LAYER: Infra
# @LAYER Infrastructure
# @BRIEF Extract normalized imported filters from decoded Superset query state (native_filters, dataMask, native_filter_state, form_data).
# @RELATION DEPENDS_ON -> [SupersetContextExtractorBase]
# #endregion SupersetContextFiltersExtractMixin

View File

@@ -1,5 +1,5 @@
# #region SupersetContextParsingMixin [C:4] [TYPE Module] [SEMANTICS superset, transform, dashboard, dataset, session, review]
# @LAYER: Infra
# @LAYER Infrastructure
# @BRIEF Parse supported Superset URLs and recover canonical dataset/dashboard references for review-session intake.
# @RELATION CALLS -> [SupersetClient]
# @RELATION CALLS -> [SupersetContextExtractorBase]
@@ -20,11 +20,11 @@ logger = cast(Any, logger)
class SupersetContextParsingMixin:
# #region SupersetContextParsingMixin.parse_superset_link [TYPE Function] [C:4]
# @PURPOSE: Extract candidate identifiers and query state from supported Superset URLs.
# @RELATION: CALLS -> [SupersetClient.get_dashboard_detail]
# @PRE: link is a non-empty Superset URL compatible with the configured environment.
# @POST: returns resolved dataset/dashboard context, preserving explicit partial-recovery state if some identifiers cannot be confirmed.
# @SIDE_EFFECT: may issue Superset API reads to resolve dataset references from dashboard or chart URLs.
# @DATA_CONTRACT: Input[link:str] -> Output[SupersetParsedContext]
# @RELATION CALLS -> [EXT:method:SupersetClient.get_dashboard_detail]
# @PRE link is a non-empty Superset URL compatible with the configured environment.
# @POST returns resolved dataset/dashboard context, preserving explicit partial-recovery state if some identifiers cannot be confirmed.
# @SIDE_EFFECT may issue Superset API reads to resolve dataset references from dashboard or chart URLs.
# @DATA_CONTRACT Input[link:str] -> Output[SupersetParsedContext]
def parse_superset_link(self, link: str) -> SupersetParsedContext:
with belief_scope("SupersetContextExtractor.parse_superset_link"):
normalized_link = str(link or "").strip()
@@ -316,7 +316,7 @@ class SupersetContextParsingMixin:
# #endregion SupersetContextParsingMixin.parse_superset_link
# #region SupersetContextParsingMixin._recover_dataset_binding_from_dashboard [TYPE Function] [C:3]
# @PURPOSE: Recover a dataset binding from resolved dashboard context while preserving explicit unresolved markers.
# @RELATION: CALLS -> [SupersetClient.get_dashboard_detail]
# @RELATION CALLS -> [EXT:method:SupersetClient.get_dashboard_detail]
def _recover_dataset_binding_from_dashboard(
self,
dashboard_id: int,

View File

@@ -1,5 +1,5 @@
# #region SupersetContextExtractorPII [C:3] [TYPE Module] [SEMANTICS superset, dataset, assistant, review]
# @LAYER: Infra
# @LAYER Infrastructure
# @BRIEF PII redaction helpers for assistant-facing dataset-review context — mask emails, UUIDs, long digit strings, and mixed identifiers.
# #endregion SupersetContextExtractorPII

View File

@@ -1,5 +1,5 @@
# #region SupersetContextRecoveryMixin [C:4] [TYPE Module] [SEMANTICS superset, context, recovery, dashboard, dataset]
# @LAYER: Infra
# @LAYER Infrastructure
# @BRIEF Recover imported filters from Superset parsed context and dashboard metadata.
# @RELATION CALLS -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetContextExtractorBase]
@@ -21,11 +21,11 @@ logger = cast(Any, logger)
class SupersetContextRecoveryMixin:
# #region SupersetContextRecoveryMixin.recover_imported_filters [TYPE Function] [C:4]
# @PURPOSE: Build imported filter entries from URL state and Superset-side saved context.
# @RELATION: CALLS -> [SupersetClient.get_dashboard]
# @PRE: parsed_context comes from a successful Superset link parse for one environment.
# @POST: returns explicit recovered and partial filter entries with preserved provenance and confirmation requirements.
# @SIDE_EFFECT: may issue Superset reads for dashboard metadata enrichment.
# @DATA_CONTRACT: Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]]
# @RELATION CALLS -> [EXT:method:SupersetClient.get_dashboard]
# @PRE parsed_context comes from a successful Superset link parse for one environment.
# @POST returns explicit recovered and partial filter entries with preserved provenance and confirmation requirements.
# @SIDE_EFFECT may issue Superset reads for dashboard metadata enrichment.
# @DATA_CONTRACT Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]]
def recover_imported_filters(
self, parsed_context: SupersetParsedContext
) -> list[dict[str, Any]]:

View File

@@ -1,5 +1,5 @@
# #region SupersetContextTemplatesMixin [C:3] [TYPE Module] [SEMANTICS superset, search, dataset, execution]
# @LAYER: Infra
# @LAYER Infrastructure
# @BRIEF Deterministically detect runtime variables and Jinja references from dataset query-bearing fields without execution.
# @RELATION DEPENDS_ON -> [TemplateVariable]
# @RELATION DEPENDS_ON -> [SupersetContextExtractorBase]
@@ -19,10 +19,10 @@ logger = cast(Any, logger)
class SupersetContextTemplatesMixin:
# #region SupersetContextTemplatesMixin.discover_template_variables [TYPE Function] [C:4]
# @PURPOSE: Detect runtime variables and Jinja references from dataset query-bearing fields.
# @PRE: dataset_payload is a Superset dataset-detail style payload with query-bearing fields when available.
# @POST: returns deduplicated explicit variable records without executing Jinja or fabricating runtime values.
# @SIDE_EFFECT: none.
# @DATA_CONTRACT: Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]]
# @PRE dataset_payload is a Superset dataset-detail style payload with query-bearing fields when available.
# @POST returns deduplicated explicit variable records without executing Jinja or fabricating runtime values.
# @SIDE_EFFECT none.
# @DATA_CONTRACT Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]]
def discover_template_variables(
self, dataset_payload: dict[str, Any]
) -> list[dict[str, Any]]:
@@ -112,7 +112,7 @@ class SupersetContextTemplatesMixin:
# #endregion SupersetContextTemplatesMixin.discover_template_variables
# #region SupersetContextTemplatesMixin._collect_query_bearing_expressions [TYPE Function] [C:3]
# @PURPOSE: Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery.
# @RELATION: DEPENDS_ON -> [SupersetContextTemplatesMixin.discover_template_variables]
# @RELATION DEPENDS_ON -> [SupersetContextTemplatesMixin.discover_template_variables]
def _collect_query_bearing_expressions(
self, dataset_payload: dict[str, Any]
) -> list[str]: