busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

semantics: complete DEF-to-region migration, fix regressions

Browse Source 
- Convert legacy [DEF🆔Type] anchors to #region/#endregion across 329 files
- Reinstate _normalize_timestamp_value in sql_generator.py
- Fix MarkerLogger→logger migration in events.py (molecular CoT markers)
- Fix dataset_review orchestrator dependencies (_build_execution_snapshot)
- Fix config_manager stale-record deletion (moved to save path only)
- Add 77 missing [/DEF:] closers in 5 unbalanced test files
- Update assistant_chat.integration.test.js for #region format
- Apply molecular-cot-logging markers (REASON/REFLECT/EXPLORE) via logger.* methods
This commit is contained in:
busya
2026-05-12 23:54:55 +03:00
parent fe8978f716
commit 306c5ae742
331 changed files with 9630 additions and 10312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
backend/src/core/utils/async_network.py
View File

@@ -1,22 +1,20 @@
# #region AsyncNetworkModule [C:5] [TYPE Module] [SEMANTICS network, httpx, async, superset, authentication, cache]
# @BRIEF Provides async Superset API client with shared auth-token cache to avoid per-request re-login.
# @LAYER Infra
# @PRE Config payloads contain a Superset base URL and authentication fields needed for login.
# @POST Async network clients reuse cached auth tokens and expose stable async request/error translation flow.
# @SIDE_EFFECT Performs upstream HTTP I/O and mutates process-local auth cache entries.
# @DATA_CONTRACT Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions]
# @INVARIANT Async client reuses cached auth tokens per environment credentials and invalidates on 401.
# @RELATION DEPENDS_ON -> [SupersetAuthCache]
#
# @BRIEF Provides async Superset API client with shared auth-token cache to avoid per-request re-login.
# @LAYER: Infra
# @PRE: Config payloads contain a Superset base URL and authentication fields needed for login.
# @POST: Async network clients reuse cached auth tokens and expose stable async request/error translation flow.
# @SIDE_EFFECT: Performs upstream HTTP I/O and mutates process-local auth cache entries.
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions]
# @RELATION DEPENDS_ON -> [SupersetAuthCache]
# @INVARIANT: Async client reuses cached auth tokens per environment credentials and invalidates on 401.
# [SECTION: IMPORTS]
from typing import Optional, Dict, Any, Union
import asyncio
import httpx
from ..logger import logger as app_logger, belief_scope
from ..cot_logger import MarkerLogger
from .network import (
AuthenticationError,
DashboardNotFoundError,
@@ -25,9 +23,7 @@ from .network import (
SupersetAPIError,
SupersetAuthCache,
)
# [/SECTION]
log = MarkerLogger("AsyncNetwork")
# #region AsyncAPIClient [C:3] [TYPE Class]
# @BRIEF Async Superset API client backed by httpx.AsyncClient with shared auth cache.
@@ -38,13 +34,14 @@ class AsyncAPIClient:
DEFAULT_TIMEOUT = 30
_auth_locks: Dict[tuple[str, str, bool], asyncio.Lock] = {}
# #region AsyncAPIClient.__init__ [C:3] [TYPE Function]
# @BRIEF Initialize async API client for one environment.
# @PRE config contains base_url and auth payload.
# @POST Client is ready for async request/authentication flow.
# @DATA_CONTRACT Input[config: Dict[str, Any]] -> self._auth_cache_key[str]
# @RELATION CALLS -> [AsyncAPIClient._normalize_base_url]
# @RELATION DEPENDS_ON -> [SupersetAuthCache]
# [DEF:AsyncAPIClient.__init__:Function]
# @COMPLEXITY: 3
# @PURPOSE: Initialize async API client for one environment.
# @PRE: config contains base_url and auth payload.
# @POST: Client is ready for async request/authentication flow.
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> self._auth_cache_key[str]
# @RELATION: [CALLS] ->[AsyncAPIClient._normalize_base_url]
# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]
def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
self.base_url: str = self._normalize_base_url(config.get("base_url", ""))
self.api_base_url: str = f"{self.base_url}/api/v1"
@@ -63,21 +60,23 @@ class AsyncAPIClient:
verify_ssl,
)
# #endregion AsyncAPIClient.__init__
# [/DEF:AsyncAPIClient.__init__:Function]
# #region AsyncAPIClient._normalize_base_url [C:1] [TYPE Function]
# @BRIEF Normalize base URL for Superset API root construction.
# @POST Returns canonical base URL without trailing slash and duplicate /api/v1 suffix.
# [DEF:AsyncAPIClient._normalize_base_url:Function]
# @COMPLEXITY: 1
# @PURPOSE: Normalize base URL for Superset API root construction.
# @POST: Returns canonical base URL without trailing slash and duplicate /api/v1 suffix.
def _normalize_base_url(self, raw_url: str) -> str:
normalized = str(raw_url or "").strip().rstrip("/")
if normalized.lower().endswith("/api/v1"):
normalized = normalized[:-len("/api/v1")]
return normalized.rstrip("/")
# #endregion AsyncAPIClient._normalize_base_url
# [/DEF:AsyncAPIClient._normalize_base_url:Function]
# #region AsyncAPIClient._build_api_url [C:1] [TYPE Function]
# @BRIEF Build full API URL from relative Superset endpoint.
# @POST Returns absolute URL for upstream request.
# [DEF:AsyncAPIClient._build_api_url:Function]
# @COMPLEXITY: 1
# @PURPOSE: Build full API URL from relative Superset endpoint.
# @POST: Returns absolute URL for upstream request.
def _build_api_url(self, endpoint: str) -> str:
normalized_endpoint = str(endpoint or "").strip()
if normalized_endpoint.startswith("http://") or normalized_endpoint.startswith("https://"):
@@ -87,11 +86,12 @@ class AsyncAPIClient:
if normalized_endpoint.startswith("/api/v1/") or normalized_endpoint == "/api/v1":
return f"{self.base_url}{normalized_endpoint}"
return f"{self.api_base_url}{normalized_endpoint}"
# #endregion AsyncAPIClient._build_api_url
# [/DEF:AsyncAPIClient._build_api_url:Function]
# #region AsyncAPIClient._get_auth_lock [C:1] [TYPE Function]
# @BRIEF Return per-cache-key async lock to serialize fresh login attempts.
# @POST Returns stable asyncio.Lock instance.
# [DEF:AsyncAPIClient._get_auth_lock:Function]
# @COMPLEXITY: 1
# @PURPOSE: Return per-cache-key async lock to serialize fresh login attempts.
# @POST: Returns stable asyncio.Lock instance.
@classmethod
def _get_auth_lock(cls, cache_key: tuple[str, str, bool]) -> asyncio.Lock:
existing_lock = cls._auth_locks.get(cache_key)
@@ -100,40 +100,24 @@ class AsyncAPIClient:
created_lock = asyncio.Lock()
cls._auth_locks[cache_key] = created_lock
return created_lock
# #endregion AsyncAPIClient._get_auth_lock
# [/DEF:AsyncAPIClient._get_auth_lock:Function]
# #region AsyncAPIClient.authenticate [C:3] [TYPE Function]
# @BRIEF Authenticate against Superset and cache access/csrf tokens.
# @POST Client tokens are populated and reusable across requests.
# @SIDE_EFFECT Performs network requests to Superset authentication endpoints.
# @DATA_CONTRACT None -> Output[Dict[str, str]]
# @RELATION CALLS -> [SupersetAuthCache.get]
# @RELATION CALLS -> [SupersetAuthCache.set]
# @RELATION CALLS -> [AsyncAPIClient._get_auth_lock]
# [DEF:AsyncAPIClient.authenticate:Function]
# @COMPLEXITY: 3
# @PURPOSE: Authenticate against Superset and cache access/csrf tokens.
# @POST: Client tokens are populated and reusable across requests.
# @SIDE_EFFECT: Performs network requests to Superset authentication endpoints.
# @DATA_CONTRACT: None -> Output[Dict[str, str]]
# @RELATION: [CALLS] ->[SupersetAuthCache.get]
# @RELATION: [CALLS] ->[SupersetAuthCache.set]
# @RELATION: [CALLS] ->[AsyncAPIClient._get_auth_lock]
async def authenticate(self) -> Dict[str, str]:
cached_tokens = SupersetAuthCache.get(self._auth_cache_key)
if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"):
self._tokens = cached_tokens
# Always fetch a fresh CSRF token to establish session cookie in this httpx client.
# The cached access_token may still be valid; we avoid a full POST login.
try:
csrf_url = f"{self.api_base_url}/security/csrf_token/"
csrf_response = await self._client.get(
csrf_url,
headers={"Authorization": f"Bearer {self._tokens['access_token']}"},
)
csrf_response.raise_for_status()
self._tokens["csrf_token"] = csrf_response.json()["result"]
SupersetAuthCache.set(self._auth_cache_key, self._tokens)
self._authenticated = True
log.reason("Reused cached tokens + refreshed CSRF token", payload={"base_url": self.base_url})
return self._tokens
except Exception as csrf_err:
log.explore("CSRF refresh failed, performing full re-auth", error=str(csrf_err))
SupersetAuthCache.invalidate(self._auth_cache_key)
self._tokens = {}
self._authenticated = False
self._authenticated = True
app_logger.info("[async_authenticate][CacheHit] Reusing cached Superset auth tokens for %s", self.base_url)
return self._tokens
auth_lock = self._get_auth_lock(self._auth_cache_key)
async with auth_lock:
@@ -141,11 +125,11 @@ class AsyncAPIClient:
if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"):
self._tokens = cached_tokens
self._authenticated = True
log.reason("Reusing cached Superset auth tokens (after lock wait)", payload={"base_url": self.base_url})
app_logger.info("[async_authenticate][CacheHitAfterWait] Reusing cached Superset auth tokens for %s", self.base_url)
return self._tokens
with belief_scope("AsyncAPIClient.authenticate"):
log.reason("Performing full authentication", payload={"base_url": self.base_url})
app_logger.info("[async_authenticate][Enter] Authenticating to %s", self.base_url)
try:
login_url = f"{self.api_base_url}/security/login"
response = await self._client.post(login_url, json=self.auth)
@@ -165,7 +149,7 @@ class AsyncAPIClient:
}
self._authenticated = True
SupersetAuthCache.set(self._auth_cache_key, self._tokens)
log.reflect("Authenticated successfully")
app_logger.info("[async_authenticate][Exit] Authenticated successfully.")
return self._tokens
except httpx.HTTPStatusError as exc:
SupersetAuthCache.invalidate(self._auth_cache_key)
@@ -179,12 +163,13 @@ class AsyncAPIClient:
except (httpx.HTTPError, KeyError) as exc:
SupersetAuthCache.invalidate(self._auth_cache_key)
raise NetworkError(f"Network or parsing error during authentication: {exc}") from exc
# #endregion AsyncAPIClient.authenticate
# [/DEF:AsyncAPIClient.authenticate:Function]
# #region AsyncAPIClient.get_headers [C:3] [TYPE Function]
# @BRIEF Return authenticated Superset headers for async requests.
# @POST Headers include Authorization and CSRF tokens.
# @RELATION CALLS -> [AsyncAPIClient.authenticate]
# [DEF:AsyncAPIClient.get_headers:Function]
# @COMPLEXITY: 3
# @PURPOSE: Return authenticated Superset headers for async requests.
# @POST: Headers include Authorization and CSRF tokens.
# @RELATION: [CALLS] ->[AsyncAPIClient.authenticate]
async def get_headers(self) -> Dict[str, str]:
if not self._authenticated:
await self.authenticate()
@@ -194,15 +179,16 @@ class AsyncAPIClient:
"Referer": self.base_url,
"Content-Type": "application/json",
}
# #endregion AsyncAPIClient.get_headers
# [/DEF:AsyncAPIClient.get_headers:Function]
# #region AsyncAPIClient.request [C:3] [TYPE Function]
# @BRIEF Perform one authenticated async Superset API request.
# @POST Returns JSON payload or raw httpx.Response when raw_response=true.
# @SIDE_EFFECT Performs network I/O.
# @RELATION CALLS -> [AsyncAPIClient.get_headers]
# @RELATION CALLS -> [AsyncAPIClient._handle_http_error]
# @RELATION CALLS -> [AsyncAPIClient._handle_network_error]
# [DEF:AsyncAPIClient.request:Function]
# @COMPLEXITY: 3
# @PURPOSE: Perform one authenticated async Superset API request.
# @POST: Returns JSON payload or raw httpx.Response when raw_response=true.
# @SIDE_EFFECT: Performs network I/O.
# @RELATION: [CALLS] ->[AsyncAPIClient.get_headers]
# @RELATION: [CALLS] ->[AsyncAPIClient._handle_http_error]
# @RELATION: [CALLS] ->[AsyncAPIClient._handle_network_error]
async def request(
self,
method: str,
@@ -230,18 +216,19 @@ class AsyncAPIClient:
self._handle_http_error(exc, endpoint)
except httpx.HTTPError as exc:
self._handle_network_error(exc, full_url)
# #endregion AsyncAPIClient.request
# [/DEF:AsyncAPIClient.request:Function]
# #region AsyncAPIClient._handle_http_error [C:3] [TYPE Function]
# @BRIEF Translate upstream HTTP errors into stable domain exceptions.
# @POST Raises domain-specific exception for caller flow control.
# @DATA_CONTRACT Input[httpx.HTTPStatusError] -> Exception
# @RELATION CALLS -> [AsyncAPIClient._is_dashboard_endpoint]
# @RELATION DEPENDS_ON -> [DashboardNotFoundError]
# @RELATION DEPENDS_ON -> [SupersetAPIError]
# @RELATION DEPENDS_ON -> [PermissionDeniedError]
# @RELATION DEPENDS_ON -> [AuthenticationError]
# @RELATION DEPENDS_ON -> [NetworkError]
# [DEF:AsyncAPIClient._handle_http_error:Function]
# @COMPLEXITY: 3
# @PURPOSE: Translate upstream HTTP errors into stable domain exceptions.
# @POST: Raises domain-specific exception for caller flow control.
# @DATA_CONTRACT: Input[httpx.HTTPStatusError] -> Exception
# @RELATION: [CALLS] ->[AsyncAPIClient._is_dashboard_endpoint]
# @RELATION: [DEPENDS_ON] ->[DashboardNotFoundError]
# @RELATION: [DEPENDS_ON] ->[SupersetAPIError]
# @RELATION: [DEPENDS_ON] ->[PermissionDeniedError]
# @RELATION: [DEPENDS_ON] ->[AuthenticationError]
# @RELATION: [DEPENDS_ON] ->[NetworkError]
def _handle_http_error(self, exc: httpx.HTTPStatusError, endpoint: str) -> None:
with belief_scope("AsyncAPIClient._handle_http_error"):
status_code = exc.response.status_code
@@ -261,11 +248,12 @@ class AsyncAPIClient:
if status_code == 401:
raise AuthenticationError() from exc
raise SupersetAPIError(f"API Error {status_code}: {exc.response.text}") from exc
# #endregion AsyncAPIClient._handle_http_error
# [/DEF:AsyncAPIClient._handle_http_error:Function]
# #region AsyncAPIClient._is_dashboard_endpoint [C:2] [TYPE Function]
# @BRIEF Determine whether an API endpoint represents a dashboard resource for 404 translation.
# @POST Returns true only for dashboard-specific endpoints.
# [DEF:AsyncAPIClient._is_dashboard_endpoint:Function]
# @COMPLEXITY: 2
# @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.
# @POST: Returns true only for dashboard-specific endpoints.
def _is_dashboard_endpoint(self, endpoint: str) -> bool:
normalized_endpoint = str(endpoint or "").strip().lower()
if not normalized_endpoint:
@@ -278,13 +266,14 @@ class AsyncAPIClient:
if normalized_endpoint.startswith("/api/v1/"):
normalized_endpoint = normalized_endpoint[len("/api/v1"):]
return normalized_endpoint.startswith("/dashboard/") or normalized_endpoint == "/dashboard"
# #endregion AsyncAPIClient._is_dashboard_endpoint
# [/DEF:AsyncAPIClient._is_dashboard_endpoint:Function]
# #region AsyncAPIClient._handle_network_error [C:3] [TYPE Function]
# @BRIEF Translate generic httpx errors into NetworkError.
# @POST Raises NetworkError with URL context.
# @DATA_CONTRACT Input[httpx.HTTPError] -> NetworkError
# @RELATION DEPENDS_ON -> [NetworkError]
# [DEF:AsyncAPIClient._handle_network_error:Function]
# @COMPLEXITY: 3
# @PURPOSE: Translate generic httpx errors into NetworkError.
# @POST: Raises NetworkError with URL context.
# @DATA_CONTRACT: Input[httpx.HTTPError] -> NetworkError
# @RELATION: [DEPENDS_ON] ->[NetworkError]
def _handle_network_error(self, exc: httpx.HTTPError, url: str) -> None:
with belief_scope("AsyncAPIClient._handle_network_error"):
if isinstance(exc, httpx.TimeoutException):
@@ -294,14 +283,17 @@ class AsyncAPIClient:
else:
message = f"Unknown network error: {exc}"
raise NetworkError(message, url=url) from exc
# #endregion AsyncAPIClient._handle_network_error
# [/DEF:AsyncAPIClient._handle_network_error:Function]
# #region AsyncAPIClient.aclose [C:3] [TYPE Function]
# @BRIEF Close underlying httpx client.
# @POST Client resources are released.
# @SIDE_EFFECT Closes network connections.
# @RELATION DEPENDS_ON -> [AsyncAPIClient.__init__]
# [DEF:AsyncAPIClient.aclose:Function]
# @COMPLEXITY: 3
# @PURPOSE: Close underlying httpx client.
# @POST: Client resources are released.
# @SIDE_EFFECT: Closes network connections.
# @RELATION: [DEPENDS_ON] ->[AsyncAPIClient.__init__]
async def aclose(self) -> None:
await self._client.aclose()
# #endregion AsyncAPIClient.aclose
# [/DEF:AsyncAPIClient.aclose:Function]
# #endregion AsyncAPIClient
# #endregion AsyncNetworkModule

99
backend/src/core/utils/dataset_mapper.py
View File

@@ -1,37 +1,32 @@
# #region DatasetMapperModule [TYPE Module] [SEMANTICS dataset, mapping, postgresql, xlsx, superset]
# @BRIEF Этот модуль отвечает за обновление метаданных (verbose_map) в датасетах Superset, извлекая их из PostgreSQL или XLSX-файлов.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [backend.core.superset_client]
# @RELATION DEPENDS_ON -> [pandas]
# @RELATION DEPENDS_ON -> [psycopg2]
#
# @BRIEF Этот модуль отвечает за обновление метаданных (verbose_map) в датасетах Superset, извлекая их из PostgreSQL или XLSX-файлов.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> backend.core.superset_client
# @RELATION DEPENDS_ON -> pandas
# @RELATION DEPENDS_ON -> psycopg2
# @PUBLIC_API: DatasetMapper
# [SECTION: IMPORTS]
import pandas as pd # type: ignore
import psycopg2 # type: ignore
from typing import Dict, Optional, Any
from ..logger import belief_scope
from ..cot_logger import MarkerLogger
# [/SECTION]
log = MarkerLogger("DatasetMapper")
from ..logger import logger as app_logger, belief_scope
# #region DatasetMapper [TYPE Class]
# @BRIEF Класс для меппинга и обновления verbose_map в датасетах Superset.
class DatasetMapper:
# #region __init__ [TYPE Function]
# @BRIEF Initializes the mapper.
# @POST Объект DatasetMapper инициализирован.
# [DEF:__init__:Function]
# @PURPOSE: Initializes the mapper.
# @POST: Объект DatasetMapper инициализирован.
def __init__(self):
pass
# #endregion __init__
# [/DEF:__init__:Function]
# #region get_postgres_comments [TYPE Function]
# @BRIEF Извлекает комментарии к колонкам из системного каталога PostgreSQL.
# @PRE db_config должен содержать валидные параметры подключения (host, port, user, password, dbname).
# @PRE table_name и table_schema должны быть строками.
# @POST Возвращается словарь, где ключи - имена колонок, значения - комментарии из БД.
# [DEF:get_postgres_comments:Function]
# @PURPOSE: Извлекает комментарии к колонкам из системного каталога PostgreSQL.
# @PRE: db_config должен содержать валидные параметры подключения (host, port, user, password, dbname).
# @PRE: table_name и table_schema должны быть строками.
# @POST: Возвращается словарь, где ключи - имена колонок, значения - комментарии из БД.
# @THROW: Exception - При ошибках подключения или выполнения запроса к БД.
# @PARAM: db_config (Dict) - Конфигурация для подключения к БД.
# @PARAM: table_name (str) - Имя таблицы.
@@ -39,7 +34,7 @@ class DatasetMapper:
# @RETURN: Dict[str, str] - Словарь с комментариями к колонкам.
def get_postgres_comments(self, db_config: Dict, table_name: str, table_schema: str) -> Dict[str, str]:
with belief_scope("Fetch comments from PostgreSQL"):
log.reason("Fetching comments from PostgreSQL", payload={"table_schema": table_schema, "table_name": table_name})
app_logger.info("[get_postgres_comments][Enter] Fetching comments from PostgreSQL for %s.%s.", table_schema, table_name)
query = f"""
SELECT
cols.column_name,
@@ -85,42 +80,42 @@ class DatasetMapper:
for row in cursor.fetchall():
if row[1]:
comments[row[0]] = row[1]
log.reflect("PostgreSQL comments fetched", payload={"count": len(comments)})
app_logger.info("[get_postgres_comments][Success] Fetched %d comments.", len(comments))
except Exception as e:
log.explore("Failed to fetch PostgreSQL comments", payload={"table_schema": table_schema, "table_name": table_name}, error=str(e))
app_logger.error("[get_postgres_comments][Failure] %s", e, exc_info=True)
raise
return comments
# #endregion get_postgres_comments
# [/DEF:get_postgres_comments:Function]
# #region load_excel_mappings [TYPE Function]
# @BRIEF Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла.
# @PRE file_path должен указывать на существующий XLSX файл.
# @POST Возвращается словарь с меппингами из файла.
# [DEF:load_excel_mappings:Function]
# @PURPOSE: Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла.
# @PRE: file_path должен указывать на существующий XLSX файл.
# @POST: Возвращается словарь с меппингами из файла.
# @THROW: Exception - При ошибках чтения файла или парсинга.
# @PARAM: file_path (str) - Путь к XLSX файлу.
# @RETURN: Dict[str, str] - Словарь с меппингами.
def load_excel_mappings(self, file_path: str) -> Dict[str, str]:
with belief_scope("Load mappings from Excel"):
log.reason("Loading mappings from Excel", payload={"file_path": file_path})
app_logger.info("[load_excel_mappings][Enter] Loading mappings from %s.", file_path)
try:
df = pd.read_excel(file_path)
mappings = df.set_index('column_name')['verbose_name'].to_dict()
log.reflect("Excel mappings loaded", payload={"count": len(mappings)})
app_logger.info("[load_excel_mappings][Success] Loaded %d mappings.", len(mappings))
return mappings
except Exception as e:
log.explore("Failed to load Excel mappings", payload={"file_path": file_path}, error=str(e))
app_logger.error("[load_excel_mappings][Failure] %s", e, exc_info=True)
raise
# #endregion load_excel_mappings
# [/DEF:load_excel_mappings:Function]
# #region run_mapping [TYPE Function]
# @BRIEF Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset.
# @PRE superset_client должен быть авторизован.
# @PRE dataset_id должен быть существующим ID в Superset.
# @POST Если найдены изменения, датасет в Superset обновлен через API.
# @RELATION CALLS -> [self.get_postgres_comments]
# @RELATION CALLS -> [self.load_excel_mappings]
# @RELATION CALLS -> [superset_client.get_dataset]
# @RELATION CALLS -> [superset_client.update_dataset]
# [DEF:run_mapping:Function]
# @PURPOSE: Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset.
# @PRE: superset_client должен быть авторизован.
# @PRE: dataset_id должен быть существующим ID в Superset.
# @POST: Если найдены изменения, датасет в Superset обновлен через API.
# @RELATION: CALLS -> self.get_postgres_comments
# @RELATION: CALLS -> self.load_excel_mappings
# @RELATION: CALLS -> superset_client.get_dataset
# @RELATION: CALLS -> superset_client.update_dataset
# @PARAM: superset_client (Any) - Клиент Superset.
# @PARAM: dataset_id (int) - ID датасета для обновления.
# @PARAM: source (str) - Источник данных ('postgres', 'excel', 'both').
@@ -130,20 +125,18 @@ class DatasetMapper:
# @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL.
def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: Optional[Dict] = None, excel_path: Optional[str] = None, table_name: Optional[str] = None, table_schema: Optional[str] = None):
with belief_scope(f"Run dataset mapping for ID {dataset_id}"):
log.reason("Starting dataset mapping", payload={"dataset_id": dataset_id, "source": source})
app_logger.info("[run_mapping][Enter] Starting dataset mapping for ID %d from source '%s'.", dataset_id, source)
mappings: Dict[str, str] = {}
try:
if source in ['postgres', 'both']:
if not (postgres_config and table_name and table_schema):
raise ValueError("Postgres config is required.")
assert postgres_config and table_name and table_schema, "Postgres config is required."
mappings.update(self.get_postgres_comments(postgres_config, table_name, table_schema))
if source in ['excel', 'both']:
if not excel_path:
raise ValueError("Excel path is required.")
assert excel_path, "Excel path is required."
mappings.update(self.load_excel_mappings(excel_path))
if source not in ['postgres', 'excel', 'both']:
log.explore("Invalid mapping source", payload={"source": source}, error=f"Invalid source: {source}")
app_logger.error("[run_mapping][Failure] Invalid source: %s.", source)
return
dataset_response = superset_client.get_dataset(dataset_id)
@@ -228,14 +221,14 @@ class DatasetMapper:
payload_for_update = {k: v for k, v in payload_for_update.items() if v is not None}
superset_client.update_dataset(dataset_id, payload_for_update)
log.reflect("Dataset verbose_name updated", payload={"dataset_id": dataset_id})
app_logger.info("[run_mapping][Success] Dataset %d columns' verbose_name updated.", dataset_id)
else:
log.reflect("No changes in verbose_name, skipping update", payload={"dataset_id": dataset_id})
app_logger.info("[run_mapping][State] No changes in columns' verbose_name, skipping update.")
except (FileNotFoundError, Exception) as e:
log.explore("Dataset mapping failed", payload={"dataset_id": dataset_id, "source": source}, error=str(e))
except (AssertionError, FileNotFoundError, Exception) as e:
app_logger.error("[run_mapping][Failure] %s", e, exc_info=True)
return
# #endregion run_mapping
# [/DEF:run_mapping:Function]
# #endregion DatasetMapper
# #endregion DatasetMapperModule
# #endregion DatasetMapperModule

208
backend/src/core/utils/fileio.py
View File

@@ -1,13 +1,11 @@
# #region FileIO [TYPE Module]
# #region FileIO [TYPE Module] [SEMANTICS file, io, zip, yaml, temp, archive, utility]
#
# @TIER: STANDARD
# @SEMANTICS: file, io, zip, yaml, temp, archive, utility
# @PURPOSE: Предоставляет набор утилит для управления файловыми операциями, включая работу с временными файлами, архивами ZIP, файлами YAML и очистку директорий.
# @BRIEF Предоставляет набор утилит для управления файловыми операциями, включая работу с временными файлами, архивами ZIP, файлами YAML и очистку директорий.
# @LAYER: Infra
# @RELATION: DEPENDS_ON -> [LoggerModule]
# @RELATION DEPENDS_ON -> [LoggerModule]
# @PUBLIC_API: create_temp_file, remove_empty_directories, read_dashboard_from_disk, calculate_crc32, RetentionPolicy, archive_exports, save_and_unpack_dashboard, update_yamls, create_dashboard_export, sanitize_filename, get_filename_from_headers, consolidate_archive_folders
# [SECTION: IMPORTS]
import os
import re
import zipfile
@@ -20,10 +18,6 @@ import shutil
import zlib
from dataclasses import dataclass
from ..logger import logger as app_logger, belief_scope
from ..cot_logger import MarkerLogger
# [/SECTION]
log = MarkerLogger("FileIO")
# #region InvalidZipFormatError [TYPE Class]
# @BRIEF Exception raised when a file is not a valid ZIP archive.
@@ -33,13 +27,9 @@ class InvalidZipFormatError(Exception):
# #region create_temp_file [TYPE Function]
# @BRIEF Контекстный менеджер для создания временного файла или директории с гарантированным удалением.
# @PRE suffix должен быть строкой, определяющей тип ресурса.
# @POST Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста.
# @PARAM: content (Optional[bytes]) - Бинарное содержимое для записи во временный файл.
# @PARAM: suffix (str) - Суффикс ресурса. Если `.dir`, создается директория.
# @PARAM: mode (str) - Режим записи в файл (e.g., 'wb').
# @PRE: suffix должен быть строкой, определяющей тип ресурса.
# @POST: Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста.
# @YIELDS: Path - Путь к временному ресурсу.
# @THROW: IOError - При ошибках создания ресурса.
@contextmanager
def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode: str = 'wb', dry_run = False) -> Generator[Path, None, None]:
with belief_scope("Create temporary resource"):
@@ -49,7 +39,7 @@ def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode
if is_dir:
with tempfile.TemporaryDirectory(suffix=suffix) as temp_dir:
resource_path = Path(temp_dir)
log.reason("Created temporary directory", payload={"path": str(resource_path)})
app_logger.debug("[create_temp_file][State] Created temporary directory: %s", resource_path)
yield resource_path
else:
fd, temp_path_str = tempfile.mkstemp(suffix=suffix)
@@ -57,70 +47,63 @@ def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode
os.close(fd)
if content:
resource_path.write_bytes(content)
log.reason("Created temporary file", payload={"path": str(resource_path)})
app_logger.debug("[create_temp_file][State] Created temporary file: %s", resource_path)
yield resource_path
finally:
if resource_path and resource_path.exists() and not dry_run:
try:
if resource_path.is_dir():
shutil.rmtree(resource_path)
log.reflect("Cleaned up temporary directory", payload={"path": str(resource_path)})
app_logger.debug("[create_temp_file][Cleanup] Removed temporary directory: %s", resource_path)
else:
resource_path.unlink()
log.reflect("Cleaned up temporary file", payload={"path": str(resource_path)})
app_logger.debug("[create_temp_file][Cleanup] Removed temporary file: %s", resource_path)
except OSError as e:
log.explore("Error during temp resource cleanup", payload={"path": str(resource_path)}, error=str(e))
app_logger.error("[create_temp_file][Failure] Error during cleanup of %s: %s", resource_path, e)
# #endregion create_temp_file
# #region remove_empty_directories [TYPE Function]
# @BRIEF Рекурсивно удаляет все пустые поддиректории, начиная с указанного пути.
# @PRE root_dir должен быть путем к существующей директории.
# @POST Все пустые поддиректории удалены, возвращено их количество.
# @PARAM: root_dir (str) - Путь к корневой директории для очистки.
# @RETURN: int - Количество удаленных директорий.
# @PRE: root_dir должен быть путем к существующей директории.
# @POST: Все пустые поддиректории удалены, возвращено их количество.
def remove_empty_directories(root_dir: str) -> int:
with belief_scope(f"Remove empty directories in {root_dir}"):
app_logger.info("[remove_empty_directories][Enter] Starting cleanup of empty directories in %s", root_dir)
removed_count = 0
if not os.path.isdir(root_dir):
log.explore("Empty directory cleanup skipped - directory not found", payload={"root_dir": root_dir}, error=f"Directory not found: {root_dir}")
app_logger.error("[remove_empty_directories][Failure] Directory not found: %s", root_dir)
return 0
for current_dir, _, _ in os.walk(root_dir, topdown=False):
if not os.listdir(current_dir):
try:
os.rmdir(current_dir)
removed_count += 1
log.reason("Removed empty directory", payload={"directory": current_dir})
app_logger.info("[remove_empty_directories][State] Removed empty directory: %s", current_dir)
except OSError as e:
log.explore("Failed to remove empty directory", payload={"directory": current_dir}, error=str(e))
log.reflect("Empty directory cleanup complete", payload={"removed_count": removed_count})
app_logger.error("[remove_empty_directories][Failure] Failed to remove %s: %s", current_dir, e)
app_logger.info("[remove_empty_directories][Exit] Removed %d empty directories.", removed_count)
return removed_count
# #endregion remove_empty_directories
# #region read_dashboard_from_disk [TYPE Function]
# @BRIEF Читает бинарное содержимое файла с диска.
# @PRE file_path должен указывать на существующий файл.
# @POST Возвращает байты содержимого и имя файла.
# @PARAM: file_path (str) - Путь к файлу.
# @RETURN: Tuple[bytes, str] - Кортеж (содержимое, имя файла).
# @THROW: FileNotFoundError - Если файл не найден.
# @PRE: file_path должен указывать на существующий файл.
# @POST: Возвращает байты содержимого и имя файла.
def read_dashboard_from_disk(file_path: str) -> Tuple[bytes, str]:
with belief_scope(f"Read dashboard from {file_path}"):
path = Path(file_path)
assert path.is_file(), f"Файл дашборда не найден: {file_path}"
log.reason("Reading dashboard file from disk", payload={"file_path": file_path})
app_logger.info("[read_dashboard_from_disk][Enter] Reading file: %s", file_path)
content = path.read_bytes()
if not content:
log.explore("Dashboard file is empty", payload={"file_path": file_path}, error="Empty dashboard file")
app_logger.warning("[read_dashboard_from_disk][Warning] File is empty: %s", file_path)
return content, path.name
# #endregion read_dashboard_from_disk
# #region calculate_crc32 [TYPE Function]
# @BRIEF Вычисляет контрольную сумму CRC32 для файла.
# @PRE file_path должен быть объектом Path к существующему файлу.
# @POST Возвращает 8-значную hex-строку CRC32.
# @PARAM: file_path (Path) - Путь к файлу.
# @RETURN: str - 8-значное шестнадцатеричное представление CRC32.
# @THROW: IOError - При ошибках чтения файла.
# @PRE: file_path должен быть объектом Path к существующему файлу.
# @POST: Возвращает 8-значную hex-строку CRC32.
def calculate_crc32(file_path: Path) -> str:
with belief_scope(f"Calculate CRC32 for {file_path}"):
with open(file_path, 'rb') as f:
@@ -128,7 +111,6 @@ def calculate_crc32(file_path: Path) -> str:
return f"{crc32_value:08x}"
# #endregion calculate_crc32
# [SECTION: DATA_CLASSES]
# #region RetentionPolicy [TYPE DataClass]
# @BRIEF Определяет политику хранения для архивов (ежедневные, еженедельные, ежемесячные).
@dataclass
@@ -137,35 +119,31 @@ class RetentionPolicy:
weekly: int = 4
monthly: int = 12
# #endregion RetentionPolicy
# [/SECTION]
# #region archive_exports [TYPE Function]
# @BRIEF Управляет архивом экспортированных файлов, применяя политику хранения и дедупликацию.
# @PRE output_dir должен быть путем к существующей директории.
# @POST Старые или дублирующиеся архивы удалены согласно политике.
# @RELATION CALLS -> [apply_retention_policy]
# @RELATION CALLS -> [calculate_crc32]
# @PARAM: output_dir (str) - Директория с архивами.
# @PARAM: policy (RetentionPolicy) - Политика хранения.
# @PARAM: deduplicate (bool) - Флаг для включения удаления дубликатов по CRC32.
# @PRE: output_dir должен быть путем к существующей директории.
# @POST: Старые или дублирующиеся архивы удалены согласно политике.
# @RELATION CALLS -> apply_retention_policy
# @RELATION CALLS -> calculate_crc32
def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool = False) -> None:
with belief_scope(f"Archive exports in {output_dir}"):
output_path = Path(output_dir)
if not output_path.is_dir():
log.explore("Archive directory not found", payload={"output_dir": output_dir}, error=f"Archive directory not found: {output_dir}")
app_logger.warning("[archive_exports][Skip] Archive directory not found: %s", output_dir)
return
log.reason("Managing archive exports", payload={"output_dir": output_dir})
app_logger.info("[archive_exports][Enter] Managing archive in %s", output_dir)
# 1. Collect all zip files
zip_files = list(output_path.glob("*.zip"))
if not zip_files:
log.reason("No zip files found in archive directory", payload={"output_dir": output_dir})
app_logger.info("[archive_exports][State] No zip files found in %s", output_dir)
return
# 2. Deduplication
if deduplicate:
log.reason("Starting archive deduplication")
app_logger.info("[archive_exports][State] Starting deduplication...")
checksums = {}
files_to_remove = []
@@ -177,19 +155,19 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
crc = calculate_crc32(file_path)
if crc in checksums:
files_to_remove.append(file_path)
log.reason("Duplicate archive found (same checksum)", payload={"file": file_path.name, "original": checksums[crc].name})
app_logger.debug("[archive_exports][State] Duplicate found: %s (same as %s)", file_path.name, checksums[crc].name)
else:
checksums[crc] = file_path
except Exception as e:
log.explore("Failed to calculate CRC32 for archive", payload={"file": str(file_path)}, error=str(e))
app_logger.error("[archive_exports][Failure] Failed to calculate CRC32 for %s: %s", file_path, e)
for f in files_to_remove:
try:
f.unlink()
zip_files.remove(f)
log.reason("Removed duplicate archive", payload={"file": f.name})
app_logger.info("[archive_exports][State] Removed duplicate: %s", f.name)
except OSError as e:
log.explore("Failed to remove duplicate archive", payload={"file": str(f)}, error=str(e))
app_logger.error("[archive_exports][Failure] Failed to remove duplicate %s: %s", f, e)
# 3. Retention Policy
files_with_dates = []
@@ -217,18 +195,15 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
if file_path not in files_to_keep:
try:
file_path.unlink()
log.reason("Removed archive by retention policy", payload={"file": file_path.name})
app_logger.info("[archive_exports][State] Removed by retention policy: %s", file_path.name)
except OSError as e:
log.explore("Failed to remove archive by retention policy", payload={"file": str(file_path)}, error=str(e))
app_logger.error("[archive_exports][Failure] Failed to remove %s: %s", file_path, e)
# #endregion archive_exports
# #region apply_retention_policy [TYPE Function]
# @BRIEF (Helper) Применяет политику хранения к списку файлов, возвращая те, что нужно сохранить.
# @PRE files_with_dates is a list of (Path, date) tuples.
# @POST Returns a set of files to keep.
# @PARAM: files_with_dates (List[Tuple[Path, date]]) - Список файлов с датами.
# @PARAM: policy (RetentionPolicy) - Политика хранения.
# @RETURN: set - Множество путей к файлам, которые должны быть сохранены.
# @PRE: files_with_dates is a list of (Path, date) tuples.
# @POST: Returns a set of files to keep.
def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: RetentionPolicy) -> set:
with belief_scope("Apply retention policy"):
# Сортируем по дате (от новой к старой)
@@ -253,54 +228,43 @@ def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: Re
files_to_keep.update(daily_files)
files_to_keep.update(weekly_files[:policy.weekly])
files_to_keep.update(monthly_files[:policy.monthly])
log.reflect("Retention policy applied", payload={"files_to_keep": len(files_to_keep)})
app_logger.debug("[apply_retention_policy][State] Keeping %d files according to retention policy", len(files_to_keep))
return files_to_keep
# #endregion apply_retention_policy
# #region save_and_unpack_dashboard [TYPE Function]
# @BRIEF Сохраняет бинарное содержимое ZIP-архива на диск и опционально распаковывает его.
# @PRE zip_content должен быть байтами валидного ZIP-архива.
# @POST ZIP-файл сохранен, и если unpack=True, он распакован в output_dir.
# @PARAM: zip_content (bytes) - Содержимое ZIP-архива.
# @PARAM: output_dir (Union[str, Path]) - Директория для сохранения.
# @PARAM: unpack (bool) - Флаг, нужно ли распаковывать архив.
# @PARAM: original_filename (Optional[str]) - Исходное имя файла для сохранения.
# @RETURN: Tuple[Path, Optional[Path]] - Путь к ZIP-файлу и, если применимо, путь к директории с распаковкой.
# @THROW: InvalidZipFormatError - При ошибке формата ZIP.
# @PRE: zip_content должен быть байтами валидного ZIP-архива.
# @POST: ZIP-файл сохранен, и если unpack=True, он распакован в output_dir.
def save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path], unpack: bool = False, original_filename: Optional[str] = None) -> Tuple[Path, Optional[Path]]:
with belief_scope("Save and unpack dashboard"):
log.reason("Processing dashboard ZIP save", payload={"unpack": unpack})
app_logger.info("[save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: %s", unpack)
try:
output_path = Path(output_dir)
output_path.mkdir(parents=True, exist_ok=True)
zip_name = sanitize_filename(original_filename) if original_filename else f"dashboard_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip"
zip_path = output_path / zip_name
zip_path.write_bytes(zip_content)
log.reason("Dashboard ZIP saved to disk", payload={"zip_path": str(zip_path)})
app_logger.info("[save_and_unpack_dashboard][State] Dashboard saved to: %s", zip_path)
if unpack:
with zipfile.ZipFile(zip_path, 'r') as zip_ref:
zip_ref.extractall(output_path)
log.reason("Dashboard unpacked to directory", payload={"output_path": str(output_path)})
app_logger.info("[save_and_unpack_dashboard][State] Dashboard unpacked to: %s", output_path)
return zip_path, output_path
return zip_path, None
except zipfile.BadZipFile as e:
log.explore("Invalid ZIP archive format", payload={"output_dir": str(output_dir)}, error=str(e))
app_logger.error("[save_and_unpack_dashboard][Failure] Invalid ZIP archive: %s", e)
raise InvalidZipFormatError(f"Invalid ZIP file: {e}") from e
# #endregion save_and_unpack_dashboard
# #region update_yamls [TYPE Function]
# @BRIEF Обновляет конфигурации в YAML-файлах, заменяя значения или применяя regex.
# @PRE path должен быть существующей директорией.
# @POST Все YAML файлы в директории обновлены согласно переданным параметрам.
# @RELATION CALLS -> [_update_yaml_file]
# @THROW: FileNotFoundError - Если `path` не существует.
# @PARAM: db_configs (Optional[List[Dict]]) - Список конфигураций для замены.
# @PARAM: path (str) - Путь к директории с YAML файлами.
# @PARAM: regexp_pattern (Optional[LiteralString]) - Паттерн для поиска.
# @PARAM: replace_string (Optional[LiteralString]) - Строка для замены.
# @PRE: path должен быть существующей директорией.
# @POST: Все YAML файлы в директории обновлены согласно переданным параметрам.
# @RELATION CALLS -> _update_yaml_file
def update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str = "dashboards", regexp_pattern: Optional[LiteralString] = None, replace_string: Optional[LiteralString] = None) -> None:
with belief_scope("Update YAML configurations"):
log.reason("Starting YAML configuration update", payload={"path": path})
app_logger.info("[update_yamls][Enter] Starting YAML configuration update.")
dir_path = Path(path)
assert dir_path.is_dir(), f"Путь {path} не существует или не является директорией"
@@ -312,12 +276,8 @@ def update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str =
# #region _update_yaml_file [TYPE Function]
# @BRIEF (Helper) Обновляет один YAML файл.
# @PRE file_path должен быть объектом Path к существующему YAML файлу.
# @POST Файл обновлен согласно переданным конфигурациям или регулярному выражению.
# @PARAM: file_path (Path) - Путь к файлу.
# @PARAM: db_configs (List[Dict]) - Конфигурации.
# @PARAM: regexp_pattern (Optional[str]) - Паттерн.
# @PARAM: replace_string (Optional[str]) - Замена.
# @PRE: file_path должен быть объектом Path к существующему YAML файлу.
# @POST: Файл обновлен согласно переданным конфигурациям или регулярному выражению.
def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_pattern: Optional[str], replace_string: Optional[str]) -> None:
with belief_scope(f"Update YAML file: {file_path}"):
# Читаем содержимое файла
@@ -325,7 +285,7 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_
with open(file_path, 'r', encoding='utf-8') as f:
content = f.read()
except Exception as e:
log.explore("Failed to read YAML file", payload={"file_path": str(file_path)}, error=str(e))
app_logger.error("[_update_yaml_file][Failure] Failed to read %s: %s", file_path, e)
return
# Если задан pattern и replace_string, применяем замену по регулярному выражению
if regexp_pattern and replace_string:
@@ -334,9 +294,9 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_
if new_content != content:
with open(file_path, 'w', encoding='utf-8') as f:
f.write(new_content)
log.reason("Updated YAML file using regex pattern", payload={"file_path": str(file_path)})
app_logger.info("[_update_yaml_file][State] Updated %s using regex pattern", file_path)
except Exception as e:
log.explore("Error applying regex to YAML file", payload={"file_path": str(file_path)}, error=str(e))
app_logger.error("[_update_yaml_file][Failure] Error applying regex to %s: %s", file_path, e)
# Если заданы конфигурации, заменяем значения (поддержка old/new)
if db_configs:
try:
@@ -357,37 +317,33 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_
# Группы: 1=ключ+разделитель, 2=открывающая кавычка (опц), 3=значение, 4=закрывающая кавычка (опц)
pattern = rf'({key_pattern}\s*:\s*)(["\']?)({val_pattern})(["\']?)'
# #region replacer [TYPE Function]
# @BRIEF Функция замены, сохраняющая кавычки если они были.
# @PRE match должен быть объектом совпадения регулярного выражения.
# @POST Возвращает строку с новым значением, сохраняя префикс и кавычки.
# [DEF:replacer:Function]
# @PURPOSE: Функция замены, сохраняющая кавычки если они были.
# @PRE: match должен быть объектом совпадения регулярного выражения.
# @POST: Возвращает строку с новым значением, сохраняя префикс и кавычки.
def replacer(match):
prefix = match.group(1)
quote_open = match.group(2)
quote_close = match.group(4)
return f"{prefix}{quote_open}{new_val}{quote_close}"
# #endregion replacer
# [/DEF:replacer:Function]
modified_content = re.sub(pattern, replacer, modified_content)
log.reason("Replaced YAML config value", payload={"key": key, "file_path": str(file_path)})
app_logger.info("[_update_yaml_file][State] Replaced '%s' with '%s' for key %s in %s", old_val, new_val, key, file_path)
# Записываем обратно изменённый контент без парсинга YAML, сохраняем оригинальное форматирование
with open(file_path, 'w', encoding='utf-8') as f:
f.write(modified_content)
except Exception as e:
log.explore("Error performing raw replacement in YAML", payload={"file_path": str(file_path)}, error=str(e))
app_logger.error("[_update_yaml_file][Failure] Error performing raw replacement in %s: %s", file_path, e)
# #endregion _update_yaml_file
# #region create_dashboard_export [TYPE Function]
# @BRIEF Создает ZIP-архив из указанных исходных путей.
# @PRE source_paths должен содержать существующие пути.
# @POST ZIP-архив создан по пути zip_path.
# @PARAM: zip_path (Union[str, Path]) - Путь для сохранения ZIP архива.
# @PARAM: source_paths (List[Union[str, Path]]) - Список исходных путей для архивации.
# @PARAM: exclude_extensions (Optional[List[str]]) - Список расширений для исключения.
# @RETURN: bool - `True` при успехе, `False` при ошибке.
# @PRE: source_paths должен содержать существующие пути.
# @POST: ZIP-архив создан по пути zip_path.
def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union[str, Path]], exclude_extensions: Optional[List[str]] = None) -> bool:
with belief_scope(f"Create dashboard export: {zip_path}"):
log.reason("Packing dashboard export", payload={"source_paths": [str(p) for p in source_paths], "zip_path": str(zip_path)})
app_logger.info("[create_dashboard_export][Enter] Packing dashboard: %s -> %s", source_paths, zip_path)
try:
exclude_ext = [ext.lower() for ext in exclude_extensions or []]
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
@@ -398,19 +354,17 @@ def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union
if item.is_file() and item.suffix.lower() not in exclude_ext:
arcname = item.relative_to(src_path.parent)
zipf.write(item, arcname)
log.reflect("Dashboard archive created", payload={"zip_path": str(zip_path)})
app_logger.info("[create_dashboard_export][Exit] Archive created: %s", zip_path)
return True
except (IOError, zipfile.BadZipFile, AssertionError) as e:
log.explore("Dashboard archive creation failed", payload={"zip_path": str(zip_path)}, error=str(e))
app_logger.error("[create_dashboard_export][Failure] Error: %s", e, exc_info=True)
return False
# #endregion create_dashboard_export
# #region sanitize_filename [TYPE Function]
# @BRIEF Очищает строку от символов, недопустимых в именах файлов.
# @PRE filename должен быть строкой.
# @POST Возвращает строку без спецсимволов.
# @PARAM: filename (str) - Исходное имя файла.
# @RETURN: str - Очищенная строка.
# @PRE: filename должен быть строкой.
# @POST: Возвращает строку без спецсимволов.
def sanitize_filename(filename: str) -> str:
with belief_scope(f"Sanitize filename: {filename}"):
return re.sub(r'[\\/*?:"<>|]', "_", filename).strip()
@@ -418,10 +372,8 @@ def sanitize_filename(filename: str) -> str:
# #region get_filename_from_headers [TYPE Function]
# @BRIEF Извлекает имя файла из HTTP заголовка 'Content-Disposition'.
# @PRE headers должен быть словарем заголовков.
# @POST Возвращает имя файла или None, если заголовок отсутствует.
# @PARAM: headers (dict) - Словарь HTTP заголовков.
# @RETURN: Optional[str] - Имя файла or `None`.
# @PRE: headers должен быть словарем заголовков.
# @POST: Возвращает имя файла или None, если заголовок отсутствует.
def get_filename_from_headers(headers: dict) -> Optional[str]:
with belief_scope("Get filename from headers"):
content_disposition = headers.get("Content-Disposition", "")
@@ -432,16 +384,14 @@ def get_filename_from_headers(headers: dict) -> Optional[str]:
# #region consolidate_archive_folders [TYPE Function]
# @BRIEF Консолидирует директории архивов на основе общего слага в имени.
# @PRE root_directory должен быть объектом Path к существующей директории.
# @POST Директории с одинаковым префиксом объединены в одну.
# @THROW: TypeError, ValueError - Если `root_directory` невалиден.
# @PARAM: root_directory (Path) - Корневая директория для консолидации.
# @PRE: root_directory должен быть объектом Path к существующей директории.
# @POST: Директории с одинаковым префиксом объединены в одну.
def consolidate_archive_folders(root_directory: Path) -> None:
with belief_scope(f"Consolidate archives in {root_directory}"):
assert isinstance(root_directory, Path), "root_directory must be a Path object."
assert root_directory.is_dir(), "root_directory must be an existing directory."
log.reason("Consolidating archive folders", payload={"root_directory": str(root_directory)})
app_logger.info("[consolidate_archive_folders][Enter] Consolidating archives in %s", root_directory)
# Собираем все директории с архивами
archive_dirs = []
for item in root_directory.iterdir():
@@ -464,7 +414,7 @@ def consolidate_archive_folders(root_directory: Path) -> None:
# Создаем целевую директорию
target_dir = root_directory / slug
target_dir.mkdir(exist_ok=True)
log.reason("Consolidating archive directories", payload={"slug": slug, "dir_count": len(dirs), "target": str(target_dir)})
app_logger.info("[consolidate_archive_folders][State] Consolidating %d directories under %s", len(dirs), target_dir)
# Перемещаем содержимое
for source_dir in dirs:
if source_dir == target_dir:
@@ -477,13 +427,13 @@ def consolidate_archive_folders(root_directory: Path) -> None:
else:
shutil.move(str(item), str(dest_item))
except Exception as e:
log.explore("Failed to move item during consolidation", payload={"source": str(item), "dest": str(dest_item)}, error=str(e))
app_logger.error("[consolidate_archive_folders][Failure] Failed to move %s to %s: %s", item, dest_item, e)
# Удаляем исходную директорию
try:
source_dir.rmdir()
log.reason("Removed source directory after consolidation", payload={"directory": str(source_dir)})
app_logger.info("[consolidate_archive_folders][State] Removed source directory: %s", source_dir)
except Exception as e:
log.explore("Failed to remove source directory after consolidation", payload={"directory": str(source_dir)}, error=str(e))
app_logger.error("[consolidate_archive_folders][Failure] Failed to remove source directory %s: %s", source_dir, e)
# #endregion consolidate_archive_folders
# #endregion FileIO
# #endregion FileIO

18
backend/src/core/utils/matching.py
View File

@@ -1,24 +1,18 @@
# #region FuzzyMatching [TYPE Module] [SEMANTICS fuzzy, matching, rapidfuzz, database, mapping]
#
# @BRIEF Provides utility functions for fuzzy matching database names.
# @LAYER Core
# @INVARIANT Confidence scores are returned as floats between 0.0 and 1.0.
# @RELATION DEPENDS_ON -> [rapidfuzz]
#
# @LAYER: Core
# @RELATION DEPENDS_ON -> rapidfuzz
#
# @INVARIANT: Confidence scores are returned as floats between 0.0 and 1.0.
# [SECTION: IMPORTS]
from rapidfuzz import fuzz, process
from typing import List, Dict
# [/SECTION]
# #region suggest_mappings [TYPE Function]
# @BRIEF Suggests mappings between source and target databases using fuzzy matching.
# @PRE source_databases and target_databases are lists of dictionaries with 'uuid' and 'database_name'.
# @POST Returns a list of suggested mappings with confidence scores.
# @PARAM: source_databases (List[Dict]) - Databases from the source environment.
# @PARAM: target_databases (List[Dict]) - Databases from the target environment.
# @PARAM: threshold (int) - Minimum confidence score (0-100).
# @RETURN: List[Dict] - Suggested mappings.
# @PRE: source_databases and target_databases are lists of dictionaries with 'uuid' and 'database_name'.
# @POST: Returns a list of suggested mappings with confidence scores.
def suggest_mappings(source_databases: List[Dict], target_databases: List[Dict], threshold: int = 60) -> List[Dict]:
"""
Suggest mappings between source and target databases using fuzzy matching.

309
backend/src/core/utils/network.py
View File

@@ -1,11 +1,10 @@
# #region NetworkModule [C:3] [TYPE Module] [SEMANTICS network, http, client, api, requests, session, authentication]
# @BRIEF Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API, включая аутентификацию, управление сессией, retry-логику и обработку ошибок.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [LoggerModule]
#
# @BRIEF Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API, включая аутентификацию, управление сессией, retry-логику и обработку ошибок.
# @LAYER: Infra
# @RELATION DEPENDS_ON -> [LoggerModule]
# @PUBLIC_API: APIClient
# [SECTION: IMPORTS]
from typing import Optional, Dict, Any, List, Union, cast, Tuple
import json
import io
@@ -17,81 +16,81 @@ from requests.adapters import HTTPAdapter
import urllib3
from urllib3.util.retry import Retry
from ..logger import logger as app_logger, belief_scope
from ..cot_logger import MarkerLogger
# [/SECTION]
log = MarkerLogger("Network")
# #region SupersetAPIError [C:1] [TYPE Class]
# @BRIEF Base exception for all Superset API related errors.
class SupersetAPIError(Exception):
# #region __init__ [C:1] [TYPE Function]
# @BRIEF Initializes the exception with a message and context.
# @PRE message is a string, context is a dict.
# @POST Exception is initialized with context.
# [DEF:__init__:Function]
# @COMPLEXITY: 1
# @PURPOSE: Initializes the exception with a message and context.
# @PRE: message is a string, context is a dict.
# @POST: Exception is initialized with context.
def __init__(self, message: str = "Superset API error", **context: Any):
with belief_scope("SupersetAPIError.__init__"):
self.context = context
super().__init__(f"[API_FAILURE] {message} | Context: {self.context}")
# #endregion __init__
# [/DEF:__init__:Function]
# #endregion SupersetAPIError
# #region AuthenticationError [C:1] [TYPE Class]
# @BRIEF Exception raised when authentication fails.
class AuthenticationError(SupersetAPIError):
# #region __init__ [C:1] [TYPE Function]
# @BRIEF Initializes the authentication error.
# @PRE message is a string, context is a dict.
# @POST AuthenticationError is initialized.
# [DEF:__init__:Function]
# @COMPLEXITY: 1
# @PURPOSE: Initializes the authentication error.
# @PRE: message is a string, context is a dict.
# @POST: AuthenticationError is initialized.
def __init__(self, message: str = "Authentication failed", **context: Any):
with belief_scope("AuthenticationError.__init__"):
super().__init__(message, type="authentication", **context)
# #endregion __init__
# [/DEF:__init__:Function]
# #endregion AuthenticationError
# #region PermissionDeniedError [TYPE Class]
# @BRIEF Exception raised when access is denied.
class PermissionDeniedError(AuthenticationError):
# #region __init__ [TYPE Function]
# @BRIEF Initializes the permission denied error.
# @PRE message is a string, context is a dict.
# @POST PermissionDeniedError is initialized.
# [DEF:__init__:Function]
# @PURPOSE: Initializes the permission denied error.
# @PRE: message is a string, context is a dict.
# @POST: PermissionDeniedError is initialized.
def __init__(self, message: str = "Permission denied", **context: Any):
with belief_scope("PermissionDeniedError.__init__"):
super().__init__(message, **context)
# #endregion __init__
# [/DEF:__init__:Function]
# #endregion PermissionDeniedError
# #region DashboardNotFoundError [TYPE Class]
# @BRIEF Exception raised when a dashboard cannot be found.
class DashboardNotFoundError(SupersetAPIError):
# #region __init__ [TYPE Function]
# @BRIEF Initializes the not found error with resource ID.
# @PRE resource_id is provided.
# @POST DashboardNotFoundError is initialized.
# [DEF:__init__:Function]
# @PURPOSE: Initializes the not found error with resource ID.
# @PRE: resource_id is provided.
# @POST: DashboardNotFoundError is initialized.
def __init__(self, resource_id: Union[int, str], message: str = "Dashboard not found", **context: Any):
with belief_scope("DashboardNotFoundError.__init__"):
super().__init__(f"Dashboard '{resource_id}' {message}", subtype="not_found", resource_id=resource_id, **context)
# #endregion __init__
# [/DEF:__init__:Function]
# #endregion DashboardNotFoundError
# #region NetworkError [TYPE Class]
# @BRIEF Exception raised when a network level error occurs.
class NetworkError(Exception):
# #region NetworkError.__init__ [TYPE Function]
# @BRIEF Initializes the network error.
# @PRE message is a string.
# @POST NetworkError is initialized.
# [DEF:NetworkError.__init__:Function]
# @PURPOSE: Initializes the network error.
# @PRE: message is a string.
# @POST: NetworkError is initialized.
def __init__(self, message: str = "Network connection failed", **context: Any):
with belief_scope("NetworkError.__init__"):
self.context = context
super().__init__(f"[NETWORK_FAILURE] {message} | Context: {self.context}")
# #endregion NetworkError.__init__
# [/DEF:NetworkError.__init__:Function]
# #endregion NetworkError
# #region SupersetAuthCache [TYPE Class]
# @BRIEF Process-local cache for Superset access/csrf tokens keyed by environment credentials.
# @PRE base_url and username are stable strings.
# @POST Cached entries expire automatically by TTL and can be reused across requests.
# @PRE: base_url and username are stable strings.
# @POST: Cached entries expire automatically by TTL and can be reused across requests.
class SupersetAuthCache:
TTL_SECONDS = 300
@@ -106,7 +105,7 @@ class SupersetAuthCache:
return (str(base_url or "").strip(), username, bool(verify_ssl))
@classmethod
# #region SupersetAuthCache.get [TYPE Function]
# [DEF:SupersetAuthCache.get:Function]
def get(cls, key: Tuple[str, str, bool]) -> Optional[Dict[str, str]]:
now = time.time()
with cls._lock:
@@ -125,10 +124,10 @@ class SupersetAuthCache:
"access_token": str(tokens.get("access_token") or ""),
"csrf_token": str(tokens.get("csrf_token") or ""),
}
# #endregion SupersetAuthCache.get
# [/DEF:SupersetAuthCache.get:Function]
@classmethod
# #region SupersetAuthCache.set [TYPE Function]
# [DEF:SupersetAuthCache.set:Function]
def set(cls, key: Tuple[str, str, bool], tokens: Dict[str, str], ttl_seconds: Optional[int] = None) -> None:
normalized_ttl = max(int(ttl_seconds or cls.TTL_SECONDS), 1)
with cls._lock:
@@ -139,7 +138,7 @@ class SupersetAuthCache:
},
"expires_at": time.time() + normalized_ttl,
}
# #endregion SupersetAuthCache.set
# [/DEF:SupersetAuthCache.set:Function]
@classmethod
def invalidate(cls, key: Tuple[str, str, bool]) -> None:
@@ -154,8 +153,8 @@ class SupersetAuthCache:
class APIClient:
DEFAULT_TIMEOUT = 30
# #region APIClient.__init__ [TYPE Function]
# @BRIEF Инициализирует API клиент с конфигурацией, сессией и логгером.
# [DEF:APIClient.__init__:Function]
# @PURPOSE: Инициализирует API клиент с конфигурацией, сессией и логгером.
# @PARAM: config (Dict[str, Any]) - Конфигурация.
# @PARAM: verify_ssl (bool) - Проверять ли SSL.
# @PARAM: timeout (int) - Таймаут запросов.
@@ -163,7 +162,7 @@ class APIClient:
# @POST: APIClient instance is initialized with a session.
def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
with belief_scope("__init__"):
log.reason("Initializing APIClient")
app_logger.info("[APIClient.__init__][Entry] Initializing APIClient.")
self.base_url: str = self._normalize_base_url(config.get("base_url", ""))
self.api_base_url: str = f"{self.base_url}/api/v1"
self.auth = config.get("auth")
@@ -176,14 +175,14 @@ class APIClient:
verify_ssl,
)
self._authenticated = False
log.reflect("APIClient initialized")
# #endregion APIClient.__init__
app_logger.info("[APIClient.__init__][Exit] APIClient initialized.")
# [/DEF:APIClient.__init__:Function]
# #region _init_session [TYPE Function]
# @BRIEF Создает и настраивает `requests.Session` с retry-логикой.
# @PRE self.request_settings must be initialized.
# @POST Returns a configured requests.Session instance.
# @RETURN requests.Session - Настроенная сессия.
# [DEF:_init_session:Function]
# @PURPOSE: Создает и настраивает `requests.Session` с retry-логикой.
# @PRE: self.request_settings must be initialized.
# @POST: Returns a configured requests.Session instance.
# @RETURN: requests.Session - Настроенная сессия.
def _init_session(self) -> requests.Session:
with belief_scope("_init_session"):
session = requests.Session()
@@ -216,32 +215,32 @@ class APIClient:
if not self.request_settings["verify_ssl"]:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
log.reflect("SSL verification disabled for session", payload={"base_url": self.base_url})
app_logger.warning("[_init_session][State] SSL verification disabled.")
# When verify_ssl is false, we should also disable hostname verification
session.verify = False
else:
session.verify = True
return session
# #endregion _init_session
# [/DEF:_init_session:Function]
# #region _normalize_base_url [TYPE Function]
# @BRIEF Normalize Superset environment URL to base host/path without trailing slash and /api/v1 suffix.
# @PRE raw_url can be empty.
# @POST Returns canonical base URL suitable for building API endpoints.
# @RETURN str
# [DEF:_normalize_base_url:Function]
# @PURPOSE: Normalize Superset environment URL to base host/path without trailing slash and /api/v1 suffix.
# @PRE: raw_url can be empty.
# @POST: Returns canonical base URL suitable for building API endpoints.
# @RETURN: str
def _normalize_base_url(self, raw_url: str) -> str:
normalized = str(raw_url or "").strip().rstrip("/")
if normalized.lower().endswith("/api/v1"):
normalized = normalized[:-len("/api/v1")]
return normalized.rstrip("/")
# #endregion _normalize_base_url
# [/DEF:_normalize_base_url:Function]
# #region _build_api_url [TYPE Function]
# @BRIEF Build absolute Superset API URL for endpoint using canonical /api/v1 base.
# @PRE endpoint is relative path or absolute URL.
# @POST Returns full URL without accidental duplicate slashes.
# @RETURN str
# [DEF:_build_api_url:Function]
# @PURPOSE: Build absolute Superset API URL for endpoint using canonical /api/v1 base.
# @PRE: endpoint is relative path or absolute URL.
# @POST: Returns full URL without accidental duplicate slashes.
# @RETURN: str
def _build_api_url(self, endpoint: str) -> str:
normalized_endpoint = str(endpoint or "").strip()
if normalized_endpoint.startswith("http://") or normalized_endpoint.startswith("https://"):
@@ -251,97 +250,65 @@ class APIClient:
if normalized_endpoint.startswith("/api/v1/") or normalized_endpoint == "/api/v1":
return f"{self.base_url}{normalized_endpoint}"
return f"{self.api_base_url}{normalized_endpoint}"
# #endregion _build_api_url
# [/DEF:_build_api_url:Function]
# #region APIClient.authenticate [TYPE Function]
# @BRIEF Выполняет аутентификацию в Superset API и получает access и CSRF токены.
# @PRE self.auth and self.base_url must be valid.
# @POST `self._tokens` заполнен, `self._authenticated` установлен в `True`.
# @RETURN Dict[str, str] - Словарь с токенами.
# [DEF:APIClient.authenticate:Function]
# @PURPOSE: Выполняет аутентификацию в Superset API и получает access и CSRF токены.
# @PRE: self.auth and self.base_url must be valid.
# @POST: `self._tokens` заполнен, `self._authenticated` установлен в `True`.
# @RETURN: Dict[str, str] - Словарь с токенами.
# @THROW: AuthenticationError, NetworkError - при ошибках.
# @RELATION: [CALLS] ->[SupersetAuthCache.get]
# @RELATION: [CALLS] ->[SupersetAuthCache.set]
# @RATIONALE: Auth cache stores access_token and csrf_token but not the session cookie.
# When a new APIClient instance reuses cached tokens, its fresh requests.Session
# has no session cookie — causing superset CSRF check to fail with
# "CSRF session token is missing". The fix is to always fetch a fresh CSRF token
# (GET /security/csrf_token/) even on cache hit, which establishes a session cookie
# in the new session while avoiding a full POST login.
# @REJECTED: Skipping csrf_token GET on cache hit — session cookie would be missing, causing
# all POST/PUT/DELETE requests to fail with CSRF error on new APIClient instances.
def authenticate(self) -> Dict[str, str]:
with belief_scope("authenticate"):
log.reason("Authenticating to Superset", payload={"base_url": self.base_url})
app_logger.info("[authenticate][Enter] Authenticating to %s", self.base_url)
cached_tokens = SupersetAuthCache.get(self._auth_cache_key)
if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"):
self._tokens = cached_tokens
# Always fetch a fresh CSRF token to establish session cookie in this session.
# The cached access_token may still be valid; we avoid a full POST login.
try:
csrf_url = f"{self.api_base_url}/security/csrf_token/"
csrf_response = self.session.get(
csrf_url,
headers={"Authorization": f"Bearer {self._tokens['access_token']}"},
timeout=self.request_settings["timeout"],
)
csrf_response.raise_for_status()
# Update CSRF token (may have rotated)
self._tokens["csrf_token"] = csrf_response.json()["result"]
SupersetAuthCache.set(self._auth_cache_key, self._tokens)
log.reason("Reused cached tokens + refreshed CSRF token", payload={"base_url": self.base_url})
except Exception as csrf_err:
# CSRF refresh failed — likely access_token expired, do full re-auth
log.explore("CSRF refresh failed, performing full re-auth", error=str(csrf_err))
SupersetAuthCache.invalidate(self._auth_cache_key)
# Fall through to full login below
self._tokens = {}
self._authenticated = False
if not self._authenticated:
try:
login_url = f"{self.api_base_url}/security/login"
# Log the payload keys and values (masking password)
masked_auth = {k: ("******" if k == "password" else v) for k, v in self.auth.items()}
log.reason("Performing full Superset login", payload={"login_url": login_url, "auth_payload": masked_auth})
response = self.session.post(login_url, json=self.auth, timeout=self.request_settings["timeout"])
if response.status_code != 200:
log.explore("Superset login returned non-200 status", payload={"status_code": response.status_code, "response": response.text}, error=f"HTTP {response.status_code}: {response.text[:200]}")
response.raise_for_status()
access_token = response.json()["access_token"]
csrf_url = f"{self.api_base_url}/security/csrf_token/"
csrf_response = self.session.get(csrf_url, headers={"Authorization": f"Bearer {access_token}"}, timeout=self.request_settings["timeout"])
csrf_response.raise_for_status()
self._tokens = {"access_token": access_token, "csrf_token": csrf_response.json()["result"]}
self._authenticated = True
SupersetAuthCache.set(self._auth_cache_key, self._tokens)
log.reflect("Authenticated successfully")
return self._tokens
except requests.exceptions.HTTPError as e:
SupersetAuthCache.invalidate(self._auth_cache_key)
status_code = e.response.status_code if e.response is not None else None
if status_code in [502, 503, 504]:
raise NetworkError(f"Environment unavailable during authentication (Status {status_code})", status_code=status_code) from e
raise AuthenticationError(f"Authentication failed: {e}") from e
except (requests.exceptions.RequestException, KeyError) as e:
SupersetAuthCache.invalidate(self._auth_cache_key)
raise NetworkError(f"Network or parsing error during authentication: {e}") from e
self._authenticated = True
log.reflect("Authenticated successfully (from cached tokens)")
return self._tokens
# #endregion APIClient.authenticate
self._authenticated = True
app_logger.info("[authenticate][CacheHit] Reusing cached Superset auth tokens for %s", self.base_url)
return self._tokens
try:
login_url = f"{self.api_base_url}/security/login"
# Log the payload keys and values (masking password)
masked_auth = {k: ("******" if k == "password" else v) for k, v in self.auth.items()}
app_logger.info(f"[authenticate][Debug] Login URL: {login_url}")
app_logger.info(f"[authenticate][Debug] Auth payload: {masked_auth}")
response = self.session.post(login_url, json=self.auth, timeout=self.request_settings["timeout"])
if response.status_code != 200:
app_logger.error(f"[authenticate][Error] Status: {response.status_code}, Response: {response.text}")
response.raise_for_status()
access_token = response.json()["access_token"]
csrf_url = f"{self.api_base_url}/security/csrf_token/"
csrf_response = self.session.get(csrf_url, headers={"Authorization": f"Bearer {access_token}"}, timeout=self.request_settings["timeout"])
csrf_response.raise_for_status()
self._tokens = {"access_token": access_token, "csrf_token": csrf_response.json()["result"]}
self._authenticated = True
SupersetAuthCache.set(self._auth_cache_key, self._tokens)
app_logger.info("[authenticate][Exit] Authenticated successfully.")
return self._tokens
except requests.exceptions.HTTPError as e:
SupersetAuthCache.invalidate(self._auth_cache_key)
status_code = e.response.status_code if e.response is not None else None
if status_code in [502, 503, 504]:
raise NetworkError(f"Environment unavailable during authentication (Status {status_code})", status_code=status_code) from e
raise AuthenticationError(f"Authentication failed: {e}") from e
except (requests.exceptions.RequestException, KeyError) as e:
SupersetAuthCache.invalidate(self._auth_cache_key)
raise NetworkError(f"Network or parsing error during authentication: {e}") from e
# [/DEF:APIClient.authenticate:Function]
@property
# #region headers [TYPE Function]
# @BRIEF Возвращает HTTP-заголовки для аутентифицированных запросов.
# @PRE APIClient is initialized and authenticated or can be authenticated.
# @POST Returns headers including auth tokens.
# [DEF:headers:Function]
# @PURPOSE: Возвращает HTTP-заголовки для аутентифицированных запросов.
# @PRE: APIClient is initialized and authenticated or can be authenticated.
# @POST: Returns headers including auth tokens.
def headers(self) -> Dict[str, str]:
if not self._authenticated:
self.authenticate()
@@ -351,10 +318,10 @@ class APIClient:
"Referer": self.base_url,
"Content-Type": "application/json"
}
# #endregion headers
# [/DEF:headers:Function]
# #region request [TYPE Function]
# @BRIEF Выполняет универсальный HTTP-запрос к API.
# [DEF:request:Function]
# @PURPOSE: Выполняет универсальный HTTP-запрос к API.
# @PARAM: method (str) - HTTP метод.
# @PARAM: endpoint (str) - API эндпоинт.
# @PARAM: headers (Optional[Dict]) - Дополнительные заголовки.
@@ -381,10 +348,10 @@ class APIClient:
self._handle_http_error(e, endpoint)
except requests.exceptions.RequestException as e:
self._handle_network_error(e, full_url)
# #endregion request
# [/DEF:request:Function]
# #region _handle_http_error [TYPE Function]
# @BRIEF (Helper) Преобразует HTTP ошибки в кастомные исключения.
# [DEF:_handle_http_error:Function]
# @PURPOSE: (Helper) Преобразует HTTP ошибки в кастомные исключения.
# @PARAM: e (requests.exceptions.HTTPError) - Ошибка.
# @PARAM: endpoint (str) - Эндпоинт.
# @PRE: e must be a valid HTTPError with a response.
@@ -408,12 +375,12 @@ class APIClient:
if status_code == 401:
raise AuthenticationError() from e
raise SupersetAPIError(f"API Error {status_code}: {e.response.text}") from e
# #endregion _handle_http_error
# [/DEF:_handle_http_error:Function]
# #region _is_dashboard_endpoint [TYPE Function]
# @BRIEF Determine whether an API endpoint represents a dashboard resource for 404 translation.
# @PRE endpoint may be relative or absolute.
# @POST Returns true only for dashboard-specific endpoints.
# [DEF:_is_dashboard_endpoint:Function]
# @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.
# @PRE: endpoint may be relative or absolute.
# @POST: Returns true only for dashboard-specific endpoints.
def _is_dashboard_endpoint(self, endpoint: str) -> bool:
normalized_endpoint = str(endpoint or "").strip().lower()
if not normalized_endpoint:
@@ -426,10 +393,10 @@ class APIClient:
if normalized_endpoint.startswith("/api/v1/"):
normalized_endpoint = normalized_endpoint[len("/api/v1"):]
return normalized_endpoint.startswith("/dashboard/") or normalized_endpoint == "/dashboard"
# #endregion _is_dashboard_endpoint
# [/DEF:_is_dashboard_endpoint:Function]
# #region _handle_network_error [TYPE Function]
# @BRIEF (Helper) Преобразует сетевые ошибки в `NetworkError`.
# [DEF:_handle_network_error:Function]
# @PURPOSE: (Helper) Преобразует сетевые ошибки в `NetworkError`.
# @PARAM: e (requests.exceptions.RequestException) - Ошибка.
# @PARAM: url (str) - URL.
# @PRE: e must be a RequestException.
@@ -443,10 +410,10 @@ class APIClient:
else:
msg = f"Unknown network error: {e}"
raise NetworkError(msg, url=url) from e
# #endregion _handle_network_error
# [/DEF:_handle_network_error:Function]
# #region upload_file [TYPE Function]
# @BRIEF Загружает файл на сервер через multipart/form-data.
# [DEF:upload_file:Function]
# @PURPOSE: Загружает файл на сервер через multipart/form-data.
# @PARAM: endpoint (str) - Эндпоинт.
# @PARAM: file_info (Dict[str, Any]) - Информация о файле.
# @PARAM: extra_data (Optional[Dict]) - Дополнительные данные.
@@ -474,10 +441,10 @@ class APIClient:
raise TypeError(f"Unsupported file_obj type: {type(file_obj)}")
return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout)
# #endregion upload_file
# [/DEF:upload_file:Function]
# #region _perform_upload [TYPE Function]
# @BRIEF (Helper) Выполняет POST запрос с файлом.
# [DEF:_perform_upload:Function]
# @PURPOSE: (Helper) Выполняет POST запрос с файлом.
# @PARAM: url (str) - URL.
# @PARAM: files (Dict) - Файлы.
# @PARAM: data (Optional[Dict]) - Данные.
@@ -495,17 +462,17 @@ class APIClient:
try:
return response.json()
except Exception as json_e:
log.explore("Upload response not valid JSON", payload={"preview": response.text[:200]}, error=str(json_e))
app_logger.debug(f"[_perform_upload][Debug] Response is not valid JSON: {response.text[:200]}...")
raise SupersetAPIError(f"API error during upload: Response is not valid JSON: {json_e}") from json_e
return response.json()
except requests.exceptions.HTTPError as e:
raise SupersetAPIError(f"API error during upload: {e.response.text}") from e
except requests.exceptions.RequestException as e:
raise NetworkError(f"Network error during upload: {e}", url=url) from e
# #endregion _perform_upload
# [/DEF:_perform_upload:Function]
# #region fetch_paginated_count [TYPE Function]
# @BRIEF Получает общее количество элементов для пагинации.
# [DEF:fetch_paginated_count:Function]
# @PURPOSE: Получает общее количество элементов для пагинации.
# @PARAM: endpoint (str) - Эндпоинт.
# @PARAM: query_params (Dict) - Параметры запроса.
# @PARAM: count_field (str) - Поле с количеством.
@@ -516,10 +483,10 @@ class APIClient:
with belief_scope("fetch_paginated_count"):
response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query_params)}))
return response_json.get(count_field, 0)
# #endregion fetch_paginated_count
# [/DEF:fetch_paginated_count:Function]
# #region fetch_paginated_data [TYPE Function]
# @BRIEF Автоматически собирает данные со всех страниц пагинированного эндпоинта.
# [DEF:fetch_paginated_data:Function]
# @PURPOSE: Автоматически собирает данные со всех страниц пагинированного эндпоинта.
# @PARAM: endpoint (str) - Эндпоинт.
# @PARAM: pagination_options (Dict[str, Any]) - Опции пагинации.
# @PRE: pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional.
@@ -547,7 +514,7 @@ class APIClient:
if total_count is None:
total_count = response_json.get(count_field, len(first_page_results))
log.reason("Total count resolved from first page", payload={"total_count": total_count})
app_logger.debug(f"[fetch_paginated_data][State] Total count resolved from first page: {total_count}")
# Fetch remaining pages
total_pages = (total_count + page_size - 1) // page_size
@@ -557,7 +524,7 @@ class APIClient:
results.extend(response_json.get(results_field, []))
return results
# #endregion fetch_paginated_data
# [/DEF:fetch_paginated_data:Function]
# #endregion APIClient

127
backend/src/core/utils/superset_compilation_adapter.py
View File

@@ -1,12 +1,12 @@
# #region SupersetCompilationAdapter [C:4] [TYPE Module] [SEMANTICS dataset_review, superset, compilation_preview, sql_lab_launch, execution_truth]
# @BRIEF Interact with Superset preview compilation and SQL Lab execution endpoints using the current approved execution context.
# @LAYER Infra
# @PRE effective template params and dataset execution reference are available.
# @POST preview and launch calls return Superset-originated artifacts or explicit errors.
# @SIDE_EFFECT performs upstream Superset preview and SQL Lab calls.
# @INVARIANT The adapter never fabricates compiled SQL locally; preview truth is delegated to Superset only.
# @LAYER: Infra
# @RELATION CALLS -> [SupersetClient]
# @RELATION DEPENDS_ON -> [CompiledPreview]
# @PRE: effective template params and dataset execution reference are available.
# @POST: preview and launch calls return Superset-originated artifacts or explicit errors.
# @SIDE_EFFECT: performs upstream Superset preview and SQL Lab calls.
# @INVARIANT: The adapter never fabricates compiled SQL locally; preview truth is delegated to Superset only.
from __future__ import annotations
@@ -52,38 +52,41 @@ class SqlLabLaunchPayload:
# #region SupersetCompilationAdapter [C:4] [TYPE Class]
# @BRIEF Delegate preview compilation and SQL Lab launch to Superset without local SQL fabrication.
# @PRE environment is configured and Superset is reachable for the target session.
# @POST adapter can return explicit ready/failed preview artifacts and canonical SQL Lab references.
# @SIDE_EFFECT issues network requests to Superset API surfaces.
# @RELATION CALLS -> [SupersetClient]
# @PRE: environment is configured and Superset is reachable for the target session.
# @POST: adapter can return explicit ready/failed preview artifacts and canonical SQL Lab references.
# @SIDE_EFFECT: issues network requests to Superset API surfaces.
class SupersetCompilationAdapter:
# #region SupersetCompilationAdapter.__init__ [C:2] [TYPE Function]
# @BRIEF Bind adapter to one Superset environment and client instance.
# [DEF:SupersetCompilationAdapter.__init__:Function]
# @COMPLEXITY: 2
# @PURPOSE: Bind adapter to one Superset environment and client instance.
def __init__(
self, environment: Environment, client: Optional[SupersetClient] = None
) -> None:
self.environment = environment
self.client = client or SupersetClient(environment)
# #endregion SupersetCompilationAdapter.__init__
# [/DEF:SupersetCompilationAdapter.__init__:Function]
# #region SupersetCompilationAdapter._supports_client_method [C:2] [TYPE Function]
# @BRIEF Detect explicitly implemented client capabilities without treating loose mocks as real methods.
# [DEF:SupersetCompilationAdapter._supports_client_method:Function]
# @COMPLEXITY: 2
# @PURPOSE: Detect explicitly implemented client capabilities without treating loose mocks as real methods.
def _supports_client_method(self, method_name: str) -> bool:
client_dict = getattr(self.client, "__dict__", {})
if method_name in client_dict:
return callable(client_dict[method_name])
return callable(getattr(type(self.client), method_name, None))
# #endregion SupersetCompilationAdapter._supports_client_method
# [/DEF:SupersetCompilationAdapter._supports_client_method:Function]
# #region SupersetCompilationAdapter.compile_preview [C:4] [TYPE Function]
# @BRIEF Request Superset-side compiled SQL preview for the current effective inputs.
# @PRE dataset_id and effective inputs are available for the current session.
# @POST returns a ready or failed preview artifact backed only by Superset-originated SQL or diagnostics.
# @SIDE_EFFECT performs upstream preview requests.
# @DATA_CONTRACT Input[PreviewCompilationPayload] -> Output[CompiledPreview]
# @RELATION CALLS -> [SupersetCompilationAdapter._request_superset_preview]
# [DEF:SupersetCompilationAdapter.compile_preview:Function]
# @COMPLEXITY: 4
# @PURPOSE: Request Superset-side compiled SQL preview for the current effective inputs.
# @RELATION: [CALLS] ->[SupersetCompilationAdapter._request_superset_preview]
# @PRE: dataset_id and effective inputs are available for the current session.
# @POST: returns a ready or failed preview artifact backed only by Superset-originated SQL or diagnostics.
# @SIDE_EFFECT: performs upstream preview requests.
# @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[CompiledPreview]
def compile_preview(self, payload: PreviewCompilationPayload) -> CompiledPreview:
with belief_scope("SupersetCompilationAdapter.compile_preview"):
if payload.dataset_id <= 0:
@@ -169,25 +172,27 @@ class SupersetCompilationAdapter:
)
return preview
# #endregion SupersetCompilationAdapter.compile_preview
# [/DEF:SupersetCompilationAdapter.compile_preview:Function]
# #region SupersetCompilationAdapter.mark_preview_stale [C:2] [TYPE Function]
# @BRIEF Invalidate previous preview after mapping or value changes.
# @PRE preview is a persisted preview artifact or current in-memory snapshot.
# @POST preview status becomes stale without fabricating a replacement artifact.
# [DEF:SupersetCompilationAdapter.mark_preview_stale:Function]
# @COMPLEXITY: 2
# @PURPOSE: Invalidate previous preview after mapping or value changes.
# @PRE: preview is a persisted preview artifact or current in-memory snapshot.
# @POST: preview status becomes stale without fabricating a replacement artifact.
def mark_preview_stale(self, preview: CompiledPreview) -> CompiledPreview:
preview.preview_status = PreviewStatus.STALE
return preview
# #endregion SupersetCompilationAdapter.mark_preview_stale
# [/DEF:SupersetCompilationAdapter.mark_preview_stale:Function]
# #region SupersetCompilationAdapter.create_sql_lab_session [C:4] [TYPE Function]
# @BRIEF Create the canonical audited execution session after all launch gates pass.
# @PRE compiled_sql is Superset-originated and launch gates are already satisfied.
# @POST returns one canonical SQL Lab session reference from Superset.
# @SIDE_EFFECT performs upstream SQL Lab execution/session creation.
# @DATA_CONTRACT Input[SqlLabLaunchPayload] -> Output[str]
# @RELATION CALLS -> [SupersetCompilationAdapter._request_sql_lab_session]
# [DEF:SupersetCompilationAdapter.create_sql_lab_session:Function]
# @COMPLEXITY: 4
# @PURPOSE: Create the canonical audited execution session after all launch gates pass.
# @RELATION: [CALLS] ->[SupersetCompilationAdapter._request_sql_lab_session]
# @PRE: compiled_sql is Superset-originated and launch gates are already satisfied.
# @POST: returns one canonical SQL Lab session reference from Superset.
# @SIDE_EFFECT: performs upstream SQL Lab execution/session creation.
# @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[str]
def create_sql_lab_session(self, payload: SqlLabLaunchPayload) -> str:
with belief_scope("SupersetCompilationAdapter.create_sql_lab_session"):
compiled_sql = str(payload.compiled_sql or "").strip()
@@ -239,15 +244,16 @@ class SupersetCompilationAdapter:
)
return sql_lab_session_ref
# #endregion SupersetCompilationAdapter.create_sql_lab_session
# [/DEF:SupersetCompilationAdapter.create_sql_lab_session:Function]
# #region SupersetCompilationAdapter._request_superset_preview [C:4] [TYPE Function]
# @BRIEF Request preview compilation through explicit client support backed by real Superset endpoints only.
# @PRE payload contains a valid dataset identifier and deterministic execution inputs for one preview attempt.
# @POST returns one normalized upstream compilation response including the chosen strategy metadata.
# @SIDE_EFFECT issues one or more Superset preview requests through the client fallback chain.
# @DATA_CONTRACT Input[PreviewCompilationPayload] -> Output[Dict[str,Any]]
# @RELATION CALLS -> [SupersetClient.compile_dataset_preview]
# [DEF:SupersetCompilationAdapter._request_superset_preview:Function]
# @COMPLEXITY: 4
# @PURPOSE: Request preview compilation through explicit client support backed by real Superset endpoints only.
# @RELATION: [CALLS] ->[SupersetClient.compile_dataset_preview]
# @PRE: payload contains a valid dataset identifier and deterministic execution inputs for one preview attempt.
# @POST: returns one normalized upstream compilation response including the chosen strategy metadata.
# @SIDE_EFFECT: issues one or more Superset preview requests through the client fallback chain.
# @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[Dict[str,Any]]
def _request_superset_preview(
self, payload: PreviewCompilationPayload
) -> Dict[str, Any]:
@@ -384,15 +390,16 @@ class SupersetCompilationAdapter:
)
raise RuntimeError(str(exc)) from exc
# #endregion SupersetCompilationAdapter._request_superset_preview
# [/DEF:SupersetCompilationAdapter._request_superset_preview:Function]
# #region SupersetCompilationAdapter._request_sql_lab_session [C:4] [TYPE Function]
# @BRIEF Probe supported SQL Lab execution surfaces and return the first successful response.
# @PRE payload carries non-empty Superset-originated SQL and a preview identifier for the current launch.
# @POST returns the first successful SQL Lab execution response from Superset.
# @SIDE_EFFECT issues Superset dataset lookup and SQL Lab execution requests.
# @DATA_CONTRACT Input[SqlLabLaunchPayload] -> Output[Dict[str,Any]]
# @RELATION CALLS -> [SupersetClient.get_dataset]
# [DEF:SupersetCompilationAdapter._request_sql_lab_session:Function]
# @COMPLEXITY: 4
# @PURPOSE: Probe supported SQL Lab execution surfaces and return the first successful response.
# @RELATION: [CALLS] ->[SupersetClient.get_dataset]
# @PRE: payload carries non-empty Superset-originated SQL and a preview identifier for the current launch.
# @POST: returns the first successful SQL Lab execution response from Superset.
# @SIDE_EFFECT: issues Superset dataset lookup and SQL Lab execution requests.
# @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[Dict[str,Any]]
def _request_sql_lab_session(self, payload: SqlLabLaunchPayload) -> Dict[str, Any]:
dataset_raw = self.client.get_dataset(payload.dataset_id)
dataset_record = (
@@ -444,11 +451,12 @@ class SupersetCompilationAdapter:
"; ".join(errors) or "No Superset SQL Lab surface accepted the request"
)
# #endregion SupersetCompilationAdapter._request_sql_lab_session
# [/DEF:SupersetCompilationAdapter._request_sql_lab_session:Function]
# #region SupersetCompilationAdapter._normalize_preview_response [C:3] [TYPE Function]
# @BRIEF Normalize candidate Superset preview responses into one compiled-sql structure.
# @RELATION DEPENDS_ON -> [CompiledPreview]
# [DEF:SupersetCompilationAdapter._normalize_preview_response:Function]
# @COMPLEXITY: 3
# @PURPOSE: Normalize candidate Superset preview responses into one compiled-sql structure.
# @RELATION: [DEPENDS_ON] ->[CompiledPreview]
def _normalize_preview_response(self, response: Any) -> Optional[Dict[str, Any]]:
if not isinstance(response, dict):
return None
@@ -477,16 +485,19 @@ class SupersetCompilationAdapter:
}
return None
# #endregion SupersetCompilationAdapter._normalize_preview_response
# [/DEF:SupersetCompilationAdapter._normalize_preview_response:Function]
# #region SupersetCompilationAdapter._dump_json [C:1] [TYPE Function]
# @BRIEF Serialize Superset request payload deterministically for network transport.
# [DEF:SupersetCompilationAdapter._dump_json:Function]
# @COMPLEXITY: 1
# @PURPOSE: Serialize Superset request payload deterministically for network transport.
def _dump_json(self, payload: Dict[str, Any]) -> str:
import json
return json.dumps(payload, sort_keys=True, default=str)
# #endregion SupersetCompilationAdapter._dump_json
# [/DEF:SupersetCompilationAdapter._dump_json:Function]
# #endregion SupersetCompilationAdapter
# #endregion SupersetCompilationAdapter

18
backend/src/core/utils/superset_context_extractor/__init__.py
View File

@@ -1,15 +1,15 @@
# #region SupersetContextExtractorPackage [C:4] [TYPE Module] [SEMANTICS dataset_review, superset, link_parsing, context_recovery, partial_recovery]
# @LAYER: Infra
# @BRIEF Recover dataset and dashboard context from Superset links while preserving explicit partial-recovery markers.
# @LAYER Infra
# @PRE Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
# @POST Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
# @SIDE_EFFECT Performs upstream Superset API reads.
# @INVARIANT Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context.
# @RELATION DEPENDS_ON -> [ImportedFilter]
# @RELATION DEPENDS_ON -> [TemplateVariable]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RATIONALE Decomposed from monolithic superset_context_extractor.py (1397 lines) into
# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
# @POST: Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
# @SIDE_EFFECT: Performs upstream Superset API reads.
# @INVARIANT: Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context.
#
# @RATIONALE: Decomposed from monolithic superset_context_extractor.py (1397 lines) into
# domain-scoped mixins to satisfy INV_7 (module < 400 lines). The composed class
# preserves the original public API surface — all consumers continue to import
# from `src.core.utils.superset_context_extractor` without changes.
@@ -26,15 +26,15 @@ from ._pii import mask_pii_value, sanitize_imported_filter_for_assistant, _mask_
# #region SupersetContextExtractor [C:4] [TYPE Class]
# @BRIEF Parse supported Superset URLs and recover canonical dataset/dashboard references for review-session intake.
# @PRE constructor receives a configured environment with a usable Superset base URL.
# @POST extractor instance is ready to parse links against one Superset environment.
# @SIDE_EFFECT downstream parse operations may call Superset APIs through SupersetClient.
# @RELATION DEPENDS_ON -> [Environment]
# @RELATION INHERITS -> [SupersetContextExtractorBase]
# @RELATION INHERITS -> [SupersetContextParsingMixin]
# @RELATION INHERITS -> [SupersetContextRecoveryMixin]
# @RELATION INHERITS -> [SupersetContextTemplatesMixin]
# @RELATION INHERITS -> [SupersetContextFiltersExtractMixin]
# @PRE: constructor receives a configured environment with a usable Superset base URL.
# @POST: extractor instance is ready to parse links against one Superset environment.
# @SIDE_EFFECT: downstream parse operations may call Superset APIs through SupersetClient.
class SupersetContextExtractor(
SupersetContextFiltersExtractMixin,
SupersetContextRecoveryMixin,

81
backend/src/core/utils/superset_context_extractor/_base.py
View File

@@ -1,13 +1,13 @@
# #region SupersetContextExtractorBase [C:4] [TYPE Module] [SEMANTICS dataset_review, superset, link_parsing, context_recovery, partial_recovery]
# @LAYER: Infra
# @BRIEF Base class and helpers for recovering dataset/dashboard context from Superset links with explicit partial-recovery markers.
# @LAYER Infra
# @PRE Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
# @POST Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
# @SIDE_EFFECT Performs upstream Superset API reads.
# @INVARIANT Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context.
# @RELATION DEPENDS_ON -> [ImportedFilter]
# @RELATION DEPENDS_ON -> [TemplateVariable]
# @RELATION CALLS -> [SupersetClient]
# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
# @POST: Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
# @SIDE_EFFECT: Performs upstream Superset API reads.
# @INVARIANT: Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context.
# #endregion SupersetContextExtractorBase
# #region _base_imports [TYPE Block]
@@ -50,23 +50,25 @@ class SupersetParsedContext:
# #region SupersetContextExtractorBase [C:4] [TYPE Class]
# @BRIEF Bind extractor to one Superset environment and client instance; provide shared URL-parsing helpers.
# @PRE constructor receives a configured environment with a usable Superset base URL.
# @POST extractor instance is ready to parse links against one Superset environment.
# @SIDE_EFFECT downstream parse operations may call Superset APIs through SupersetClient.
# @RELATION DEPENDS_ON -> [Environment]
# @PRE: constructor receives a configured environment with a usable Superset base URL.
# @POST: extractor instance is ready to parse links against one Superset environment.
# @SIDE_EFFECT: downstream parse operations may call Superset APIs through SupersetClient.
class SupersetContextExtractorBase:
# #region SupersetContextExtractorBase.__init__ [C:2] [TYPE Function]
# @BRIEF Bind extractor to one Superset environment and client instance.
# [DEF:SupersetContextExtractorBase.__init__:Function]
# @COMPLEXITY: 2
# @PURPOSE: Bind extractor to one Superset environment and client instance.
def __init__(
self, environment: Environment, client: Optional[SupersetClient] = None
) -> None:
self.environment = environment
self.client = client or SupersetClient(environment)
# #endregion SupersetContextExtractorBase.__init__
# [/DEF:SupersetContextExtractorBase.__init__:Function]
# #region SupersetContextExtractorBase.build_recovery_summary [C:2] [TYPE Function]
# @BRIEF Summarize recovered, partial, and unresolved context for session state and UX.
# [DEF:SupersetContextExtractorBase.build_recovery_summary:Function]
# @COMPLEXITY: 2
# @PURPOSE: Summarize recovered, partial, and unresolved context for session state and UX.
def build_recovery_summary(
self, parsed_context: SupersetParsedContext
) -> Dict[str, Any]:
@@ -80,10 +82,11 @@ class SupersetContextExtractorBase:
"imported_filter_count": len(parsed_context.imported_filters),
}
# #endregion SupersetContextExtractorBase.build_recovery_summary
# [/DEF:SupersetContextExtractorBase.build_recovery_summary:Function]
# #region SupersetContextExtractorBase._extract_numeric_identifier [C:2] [TYPE Function]
# @BRIEF Extract a numeric identifier from a REST-like Superset URL path.
# [DEF:SupersetContextExtractorBase._extract_numeric_identifier:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a numeric identifier from a REST-like Superset URL path.
def _extract_numeric_identifier(
self, path_parts: List[str], resource_name: str
) -> Optional[int]:
@@ -102,10 +105,11 @@ class SupersetContextExtractorBase:
return None
return int(candidate)
# #endregion SupersetContextExtractorBase._extract_numeric_identifier
# [/DEF:SupersetContextExtractorBase._extract_numeric_identifier:Function]
# #region SupersetContextExtractorBase._extract_dashboard_reference [C:2] [TYPE Function]
# @BRIEF Extract a dashboard id-or-slug reference from a Superset URL path.
# [DEF:SupersetContextExtractorBase._extract_dashboard_reference:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a dashboard id-or-slug reference from a Superset URL path.
def _extract_dashboard_reference(self, path_parts: List[str]) -> Optional[str]:
if "dashboard" not in path_parts:
return None
@@ -122,10 +126,11 @@ class SupersetContextExtractorBase:
return None
return candidate
# #endregion SupersetContextExtractorBase._extract_dashboard_reference
# [/DEF:SupersetContextExtractorBase._extract_dashboard_reference:Function]
# #region SupersetContextExtractorBase._extract_dashboard_permalink_key [C:2] [TYPE Function]
# @BRIEF Extract a dashboard permalink key from a Superset URL path.
# [DEF:SupersetContextExtractorBase._extract_dashboard_permalink_key:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a dashboard permalink key from a Superset URL path.
def _extract_dashboard_permalink_key(self, path_parts: List[str]) -> Optional[str]:
if "dashboard" not in path_parts:
return None
@@ -143,31 +148,34 @@ class SupersetContextExtractorBase:
return None
return permalink_key
# #endregion SupersetContextExtractorBase._extract_dashboard_permalink_key
# [/DEF:SupersetContextExtractorBase._extract_dashboard_permalink_key:Function]
# #region SupersetContextExtractorBase._extract_dashboard_id_from_state [C:2] [TYPE Function]
# @BRIEF Extract a dashboard identifier from returned permalink state when present.
# [DEF:SupersetContextExtractorBase._extract_dashboard_id_from_state:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a dashboard identifier from returned permalink state when present.
def _extract_dashboard_id_from_state(self, state: Dict[str, Any]) -> Optional[int]:
return self._search_nested_numeric_key(
payload=state,
candidate_keys={"dashboardId", "dashboard_id", "dashboard_id_value"},
)
# #endregion SupersetContextExtractorBase._extract_dashboard_id_from_state
# [/DEF:SupersetContextExtractorBase._extract_dashboard_id_from_state:Function]
# #region SupersetContextExtractorBase._extract_chart_id_from_state [C:2] [TYPE Function]
# @BRIEF Extract a chart identifier from returned permalink state when dashboard id is absent.
# [DEF:SupersetContextExtractorBase._extract_chart_id_from_state:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a chart identifier from returned permalink state when dashboard id is absent.
def _extract_chart_id_from_state(self, state: Dict[str, Any]) -> Optional[int]:
return self._search_nested_numeric_key(
payload=state,
candidate_keys={"slice_id", "sliceId", "chartId", "chart_id"},
)
# #endregion SupersetContextExtractorBase._extract_chart_id_from_state
# [/DEF:SupersetContextExtractorBase._extract_chart_id_from_state:Function]
# #region SupersetContextExtractorBase._search_nested_numeric_key [C:3] [TYPE Function]
# @BRIEF Recursively search nested dict/list payloads for the first numeric value under a candidate key set.
# @RELATION DEPENDS_ON -> [SupersetContextExtractorBase]
# [DEF:SupersetContextExtractorBase._search_nested_numeric_key:Function]
# @COMPLEXITY: 3
# @PURPOSE: Recursively search nested dict/list payloads for the first numeric value under a candidate key set.
# @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase]
def _search_nested_numeric_key(
self, payload: Any, candidate_keys: Set[str]
) -> Optional[int]:
@@ -189,10 +197,11 @@ class SupersetContextExtractorBase:
return found
return None
# #endregion SupersetContextExtractorBase._search_nested_numeric_key
# [/DEF:SupersetContextExtractorBase._search_nested_numeric_key:Function]
# #region SupersetContextExtractorBase._decode_query_state [C:2] [TYPE Function]
# @BRIEF Decode query-string structures used by Superset URL state transport.
# [DEF:SupersetContextExtractorBase._decode_query_state:Function]
# @COMPLEXITY: 2
# @PURPOSE: Decode query-string structures used by Superset URL state transport.
def _decode_query_state(self, query_params: Dict[str, List[str]]) -> Dict[str, Any]:
query_state: Dict[str, Any] = {}
for key, values in query_params.items():
@@ -212,7 +221,7 @@ class SupersetContextExtractorBase:
query_state[key] = decoded_value
return query_state
# #endregion SupersetContextExtractorBase._decode_query_state
# [/DEF:SupersetContextExtractorBase._decode_query_state:Function]
# #endregion SupersetContextExtractorBase

9
backend/src/core/utils/superset_context_extractor/_filters.py
View File

@@ -1,6 +1,6 @@
# #region SupersetContextFiltersExtractMixin [C:3] [TYPE Module] [SEMANTICS superset, filters, extraction, query_state, dataMask, native_filters]
# @LAYER: Infra
# @BRIEF Extract normalized imported filters from decoded Superset query state (native_filters, dataMask, native_filter_state, form_data).
# @LAYER Infra
# @RELATION DEPENDS_ON -> [SupersetContextExtractorBase]
# #endregion SupersetContextFiltersExtractMixin
@@ -19,8 +19,9 @@ app_logger = cast(Any, app_logger)
# #region SupersetContextFiltersExtractMixin [C:3] [TYPE Class]
# @BRIEF Mixin providing query-state filter extraction for the composed SupersetContextExtractor.
class SupersetContextFiltersExtractMixin:
# #region SupersetContextFiltersExtractMixin._extract_imported_filters [C:2] [TYPE Function]
# @BRIEF Normalize imported filters from decoded query state without fabricating missing values.
# [DEF:SupersetContextFiltersExtractMixin._extract_imported_filters:Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize imported filters from decoded query state without fabricating missing values.
def _extract_imported_filters(
self, query_state: Dict[str, Any]
) -> List[Dict[str, Any]]:
@@ -251,7 +252,7 @@ class SupersetContextFiltersExtractMixin:
return imported_filters
# #endregion SupersetContextFiltersExtractMixin._extract_imported_filters
# [/DEF:SupersetContextFiltersExtractMixin._extract_imported_filters:Function]
# #endregion SupersetContextFiltersExtractMixin

28
backend/src/core/utils/superset_context_extractor/_parsing.py
View File

@@ -1,6 +1,6 @@
# #region SupersetContextParsingMixin [C:4] [TYPE Module] [SEMANTICS superset, link_parsing, url, permalink, recovery]
# @LAYER: Infra
# @BRIEF Parse supported Superset URLs and recover canonical dataset/dashboard references for review-session intake.
# @LAYER Infra
# @RELATION CALLS -> [SupersetClient]
# @RELATION CALLS -> [SupersetContextExtractorBase]
# #endregion SupersetContextParsingMixin
@@ -21,13 +21,14 @@ logger = cast(Any, logger)
# #region SupersetContextParsingMixin [C:4] [TYPE Class]
# @BRIEF Mixin providing Superset URL parsing logic for the composed SupersetContextExtractor.
class SupersetContextParsingMixin:
# #region SupersetContextParsingMixin.parse_superset_link [C:4] [TYPE Function]
# @BRIEF Extract candidate identifiers and query state from supported Superset URLs.
# @PRE link is a non-empty Superset URL compatible with the configured environment.
# @POST returns resolved dataset/dashboard context, preserving explicit partial-recovery state if some identifiers cannot be confirmed.
# @SIDE_EFFECT may issue Superset API reads to resolve dataset references from dashboard or chart URLs.
# @DATA_CONTRACT Input[link:str] -> Output[SupersetParsedContext]
# @RELATION CALLS -> [SupersetClient.get_dashboard_detail]
# [DEF:SupersetContextParsingMixin.parse_superset_link:Function]
# @COMPLEXITY: 4
# @PURPOSE: Extract candidate identifiers and query state from supported Superset URLs.
# @RELATION: CALLS -> [SupersetClient.get_dashboard_detail]
# @PRE: link is a non-empty Superset URL compatible with the configured environment.
# @POST: returns resolved dataset/dashboard context, preserving explicit partial-recovery state if some identifiers cannot be confirmed.
# @SIDE_EFFECT: may issue Superset API reads to resolve dataset references from dashboard or chart URLs.
# @DATA_CONTRACT: Input[link:str] -> Output[SupersetParsedContext]
def parse_superset_link(self, link: str) -> SupersetParsedContext:
with belief_scope("SupersetContextExtractor.parse_superset_link"):
normalized_link = str(link or "").strip()
@@ -328,11 +329,12 @@ class SupersetContextParsingMixin:
)
return result
# #endregion SupersetContextParsingMixin.parse_superset_link
# [/DEF:SupersetContextParsingMixin.parse_superset_link:Function]
# #region SupersetContextParsingMixin._recover_dataset_binding_from_dashboard [C:3] [TYPE Function]
# @BRIEF Recover a dataset binding from resolved dashboard context while preserving explicit unresolved markers.
# @RELATION CALLS -> [SupersetClient.get_dashboard_detail]
# [DEF:SupersetContextParsingMixin._recover_dataset_binding_from_dashboard:Function]
# @COMPLEXITY: 3
# @PURPOSE: Recover a dataset binding from resolved dashboard context while preserving explicit unresolved markers.
# @RELATION: CALLS -> [SupersetClient.get_dashboard_detail]
def _recover_dataset_binding_from_dashboard(
self,
dashboard_id: int,
@@ -369,7 +371,7 @@ class SupersetContextParsingMixin:
unresolved_references.append("dashboard_dataset_binding_missing")
return None, unresolved_references
# #endregion SupersetContextParsingMixin._recover_dataset_binding_from_dashboard
# [/DEF:SupersetContextParsingMixin._recover_dataset_binding_from_dashboard:Function]
# #endregion SupersetContextParsingMixin

2
backend/src/core/utils/superset_context_extractor/_pii.py
View File

@@ -1,6 +1,6 @@
# #region SupersetContextExtractorPII [C:3] [TYPE Module] [SEMANTICS superset, pii, masking, sanitization, privacy]
# @LAYER: Infra
# @BRIEF PII redaction helpers for assistant-facing dataset-review context — mask emails, UUIDs, long digit strings, and mixed identifiers.
# @LAYER Infra
# #endregion SupersetContextExtractorPII
# #region _pii_imports [TYPE Block]

26
backend/src/core/utils/superset_context_extractor/_recovery.py
View File

@@ -1,6 +1,6 @@
# #region SupersetContextRecoveryMixin [C:4] [TYPE Module] [SEMANTICS superset, filters, recovery, imported_filters]
# @LAYER: Infra
# @BRIEF Recover imported filters from Superset parsed context and dashboard metadata.
# @LAYER Infra
# @RELATION CALLS -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetContextExtractorBase]
# #endregion SupersetContextRecoveryMixin
@@ -22,13 +22,14 @@ logger = cast(Any, logger)
# #region SupersetContextRecoveryMixin [C:4] [TYPE Class]
# @BRIEF Mixin providing filter recovery for the composed SupersetContextExtractor.
class SupersetContextRecoveryMixin:
# #region SupersetContextRecoveryMixin.recover_imported_filters [C:4] [TYPE Function]
# @BRIEF Build imported filter entries from URL state and Superset-side saved context.
# @PRE parsed_context comes from a successful Superset link parse for one environment.
# @POST returns explicit recovered and partial filter entries with preserved provenance and confirmation requirements.
# @SIDE_EFFECT may issue Superset reads for dashboard metadata enrichment.
# @DATA_CONTRACT Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]]
# @RELATION CALLS -> [SupersetClient.get_dashboard]
# [DEF:SupersetContextRecoveryMixin.recover_imported_filters:Function]
# @COMPLEXITY: 4
# @PURPOSE: Build imported filter entries from URL state and Superset-side saved context.
# @RELATION: CALLS -> [SupersetClient.get_dashboard]
# @PRE: parsed_context comes from a successful Superset link parse for one environment.
# @POST: returns explicit recovered and partial filter entries with preserved provenance and confirmation requirements.
# @SIDE_EFFECT: may issue Superset reads for dashboard metadata enrichment.
# @DATA_CONTRACT: Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]]
def recover_imported_filters(
self, parsed_context: SupersetParsedContext
) -> List[Dict[str, Any]]:
@@ -259,10 +260,11 @@ class SupersetContextRecoveryMixin:
)
return recovered_filters
# #endregion SupersetContextRecoveryMixin.recover_imported_filters
# [/DEF:SupersetContextRecoveryMixin.recover_imported_filters:Function]
# #region SupersetContextRecoveryMixin._normalize_imported_filter_payload [C:2] [TYPE Function]
# @BRIEF Normalize one imported-filter payload with explicit provenance and confirmation state.
# [DEF:SupersetContextRecoveryMixin._normalize_imported_filter_payload:Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize one imported-filter payload with explicit provenance and confirmation state.
def _normalize_imported_filter_payload(
self,
payload: Dict[str, Any],
@@ -303,7 +305,7 @@ class SupersetContextRecoveryMixin:
"notes": str(payload.get("notes") or default_note),
}
# #endregion SupersetContextRecoveryMixin._normalize_imported_filter_payload
# [/DEF:SupersetContextRecoveryMixin._normalize_imported_filter_payload:Function]
# #endregion SupersetContextRecoveryMixin

47
backend/src/core/utils/superset_context_extractor/_templates.py
View File

@@ -1,6 +1,6 @@
# #region SupersetContextTemplatesMixin [C:3] [TYPE Module] [SEMANTICS superset, template_variables, jinja, discovery, deterministic]
# @LAYER: Infra
# @BRIEF Deterministically detect runtime variables and Jinja references from dataset query-bearing fields without execution.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [TemplateVariable]
# @RELATION DEPENDS_ON -> [SupersetContextExtractorBase]
# #endregion SupersetContextTemplatesMixin
@@ -20,12 +20,13 @@ logger = cast(Any, logger)
# #region SupersetContextTemplatesMixin [C:3] [TYPE Class]
# @BRIEF Mixin providing template variable discovery for the composed SupersetContextExtractor.
class SupersetContextTemplatesMixin:
# #region SupersetContextTemplatesMixin.discover_template_variables [C:4] [TYPE Function]
# @BRIEF Detect runtime variables and Jinja references from dataset query-bearing fields.
# @PRE dataset_payload is a Superset dataset-detail style payload with query-bearing fields when available.
# @POST returns deduplicated explicit variable records without executing Jinja or fabricating runtime values.
# @SIDE_EFFECT none.
# @DATA_CONTRACT Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]]
# [DEF:SupersetContextTemplatesMixin.discover_template_variables:Function]
# @COMPLEXITY: 4
# @PURPOSE: Detect runtime variables and Jinja references from dataset query-bearing fields.
# @PRE: dataset_payload is a Superset dataset-detail style payload with query-bearing fields when available.
# @POST: returns deduplicated explicit variable records without executing Jinja or fabricating runtime values.
# @SIDE_EFFECT: none.
# @DATA_CONTRACT: Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]]
def discover_template_variables(
self, dataset_payload: Dict[str, Any]
) -> List[Dict[str, Any]]:
@@ -117,11 +118,12 @@ class SupersetContextTemplatesMixin:
)
return discovered
# #endregion SupersetContextTemplatesMixin.discover_template_variables
# [/DEF:SupersetContextTemplatesMixin.discover_template_variables:Function]
# #region SupersetContextTemplatesMixin._collect_query_bearing_expressions [C:3] [TYPE Function]
# @BRIEF Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery.
# @RELATION DEPENDS_ON -> [SupersetContextTemplatesMixin.discover_template_variables]
# [DEF:SupersetContextTemplatesMixin._collect_query_bearing_expressions:Function]
# @COMPLEXITY: 3
# @PURPOSE: Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery.
# @RELATION: DEPENDS_ON -> [SupersetContextTemplatesMixin.discover_template_variables]
def _collect_query_bearing_expressions(
self, dataset_payload: Dict[str, Any]
) -> List[str]:
@@ -160,10 +162,11 @@ class SupersetContextTemplatesMixin:
return expressions
# #endregion SupersetContextTemplatesMixin._collect_query_bearing_expressions
# [/DEF:SupersetContextTemplatesMixin._collect_query_bearing_expressions:Function]
# #region SupersetContextTemplatesMixin._append_template_variable [C:2] [TYPE Function]
# @BRIEF Append one deduplicated template-variable descriptor.
# [DEF:SupersetContextTemplatesMixin._append_template_variable:Function]
# @COMPLEXITY: 2
# @PURPOSE: Append one deduplicated template-variable descriptor.
def _append_template_variable(
self,
discovered: List[Dict[str, Any]],
@@ -192,10 +195,11 @@ class SupersetContextTemplatesMixin:
}
)
# #endregion SupersetContextTemplatesMixin._append_template_variable
# [/DEF:SupersetContextTemplatesMixin._append_template_variable:Function]
# #region SupersetContextTemplatesMixin._extract_primary_jinja_identifier [C:2] [TYPE Function]
# @BRIEF Extract a deterministic primary identifier from a Jinja expression without executing it.
# [DEF:SupersetContextTemplatesMixin._extract_primary_jinja_identifier:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a deterministic primary identifier from a Jinja expression without executing it.
def _extract_primary_jinja_identifier(self, expression: str) -> Optional[str]:
matched = re.match(r"([A-Za-z_][A-Za-z0-9_]*)", expression.strip())
if matched is None:
@@ -214,10 +218,11 @@ class SupersetContextTemplatesMixin:
return None
return candidate
# #endregion SupersetContextTemplatesMixin._extract_primary_jinja_identifier
# [/DEF:SupersetContextTemplatesMixin._extract_primary_jinja_identifier:Function]
# #region SupersetContextTemplatesMixin._normalize_default_literal [C:2] [TYPE Function]
# @BRIEF Normalize literal default fragments from template helper calls into JSON-safe values.
# [DEF:SupersetContextTemplatesMixin._normalize_default_literal:Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize literal default fragments from template helper calls into JSON-safe values.
def _normalize_default_literal(self, literal: Optional[str]) -> Any:
normalized_literal = str(literal or "").strip()
if not normalized_literal:
@@ -241,7 +246,7 @@ class SupersetContextTemplatesMixin:
except ValueError:
return normalized_literal
# #endregion SupersetContextTemplatesMixin._normalize_default_literal
# [/DEF:SupersetContextTemplatesMixin._normalize_default_literal:Function]
# #endregion SupersetContextTemplatesMixin