chore(lint): apply ruff --fix (4443 auto-fixes)
Auto-fixed categories: - F401: unused imports removed - I001: import blocks sorted - W293: trailing whitespace stripped - UP035: deprecated typing imports replaced - SIM: simplify suggestions applied - ARG: unused args prefixed with underscore - T201: print statements removed - F841: unused variables removed - RUF059: unpacked variables prefixed Remaining ~1500 unfixable errors (C901, B904, N806, E402) require manual work. Backend smoke tests: 13/13 passed.
This commit is contained in:
@@ -8,10 +8,13 @@
|
||||
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions]
|
||||
# @RELATION DEPENDS_ON -> [SupersetAuthCache]
|
||||
# @INVARIANT: Async client reuses cached auth tokens per environment credentials and invalidates on 401.
|
||||
from typing import Optional, Dict, Any, Union
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from ..logger import logger as app_logger, belief_scope
|
||||
|
||||
from ..logger import belief_scope
|
||||
from ..logger import logger as app_logger
|
||||
from .network import (
|
||||
AuthenticationError,
|
||||
DashboardNotFoundError,
|
||||
@@ -20,6 +23,8 @@ from .network import (
|
||||
SupersetAPIError,
|
||||
SupersetAuthCache,
|
||||
)
|
||||
|
||||
|
||||
# #region AsyncAPIClient [C:3] [TYPE Class]
|
||||
# @BRIEF Async Superset API client backed by httpx.AsyncClient with shared auth cache.
|
||||
# @RELATION DEPENDS_ON -> [SupersetAuthCache]
|
||||
@@ -27,7 +32,7 @@ from .network import (
|
||||
# @RELATION CALLS -> [SupersetAuthCache.set]
|
||||
class AsyncAPIClient:
|
||||
DEFAULT_TIMEOUT = 30
|
||||
_auth_locks: Dict[tuple[str, str, bool], asyncio.Lock] = {}
|
||||
_auth_locks: dict[tuple[str, str, bool], asyncio.Lock] = {}
|
||||
# #region AsyncAPIClient.__init__ [TYPE Function] [C:3]
|
||||
# @PURPOSE: Initialize async API client for one environment.
|
||||
# @PRE: config contains base_url and auth payload.
|
||||
@@ -35,7 +40,7 @@ class AsyncAPIClient:
|
||||
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> self._auth_cache_key[str]
|
||||
# @RELATION: [CALLS] ->[AsyncAPIClient._normalize_base_url]
|
||||
# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]
|
||||
def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
|
||||
def __init__(self, config: dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
|
||||
self.base_url: str = self._normalize_base_url(config.get("base_url", ""))
|
||||
self.api_base_url: str = f"{self.base_url}/api/v1"
|
||||
self.auth = config.get("auth")
|
||||
@@ -45,7 +50,7 @@ class AsyncAPIClient:
|
||||
timeout=httpx.Timeout(timeout),
|
||||
follow_redirects=True,
|
||||
)
|
||||
self._tokens: Dict[str, str] = {}
|
||||
self._tokens: dict[str, str] = {}
|
||||
self._authenticated = False
|
||||
self._auth_cache_key = SupersetAuthCache.build_key(
|
||||
self.base_url,
|
||||
@@ -95,7 +100,7 @@ class AsyncAPIClient:
|
||||
# @RELATION: [CALLS] ->[SupersetAuthCache.get]
|
||||
# @RELATION: [CALLS] ->[SupersetAuthCache.set]
|
||||
# @RELATION: [CALLS] ->[AsyncAPIClient._get_auth_lock]
|
||||
async def authenticate(self) -> Dict[str, str]:
|
||||
async def authenticate(self) -> dict[str, str]:
|
||||
cached_tokens = SupersetAuthCache.get(self._auth_cache_key)
|
||||
if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"):
|
||||
self._tokens = cached_tokens
|
||||
@@ -148,7 +153,7 @@ class AsyncAPIClient:
|
||||
# @PURPOSE: Return authenticated Superset headers for async requests.
|
||||
# @POST: Headers include Authorization and CSRF tokens.
|
||||
# @RELATION: [CALLS] ->[AsyncAPIClient.authenticate]
|
||||
async def get_headers(self) -> Dict[str, str]:
|
||||
async def get_headers(self) -> dict[str, str]:
|
||||
if not self._authenticated:
|
||||
await self.authenticate()
|
||||
return {
|
||||
@@ -169,10 +174,10 @@ class AsyncAPIClient:
|
||||
self,
|
||||
method: str,
|
||||
endpoint: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
raw_response: bool = False,
|
||||
**kwargs,
|
||||
) -> Union[httpx.Response, Dict[str, Any]]:
|
||||
) -> httpx.Response | dict[str, Any]:
|
||||
full_url = self._build_api_url(endpoint)
|
||||
request_headers = await self.get_headers()
|
||||
if headers:
|
||||
@@ -262,4 +267,4 @@ class AsyncAPIClient:
|
||||
await self._client.aclose()
|
||||
# #endregion AsyncAPIClient.aclose
|
||||
# #endregion AsyncAPIClient
|
||||
# #endregion AsyncNetworkModule
|
||||
# #endregion AsyncNetworkModule
|
||||
|
||||
@@ -6,10 +6,15 @@
|
||||
# @RELATION DEPENDS_ON -> pandas
|
||||
# @RELATION DEPENDS_ON -> psycopg2
|
||||
# @PUBLIC_API: DatasetMapper
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd # type: ignore
|
||||
import psycopg2 # type: ignore
|
||||
from typing import Dict, Optional, Any
|
||||
from ..logger import logger as app_logger, belief_scope
|
||||
|
||||
from ..logger import belief_scope
|
||||
from ..logger import logger as app_logger
|
||||
|
||||
|
||||
# #region DatasetMapper [TYPE Class]
|
||||
# @BRIEF Класс для меппинга и обновления verbose_map в датасетах Superset.
|
||||
class DatasetMapper:
|
||||
@@ -29,7 +34,7 @@ class DatasetMapper:
|
||||
# @PARAM: table_name (str) - Имя таблицы.
|
||||
# @PARAM: table_schema (str) - Схема таблицы.
|
||||
# @RETURN: Dict[str, str] - Словарь с комментариями к колонкам.
|
||||
def get_postgres_comments(self, db_config: Dict, table_name: str, table_schema: str) -> Dict[str, str]:
|
||||
def get_postgres_comments(self, db_config: dict, table_name: str, table_schema: str) -> dict[str, str]:
|
||||
with belief_scope("Fetch comments from PostgreSQL"):
|
||||
app_logger.info("[get_postgres_comments][Enter] Fetching comments from PostgreSQL for %s.%s.", table_schema, table_name)
|
||||
query = f"""
|
||||
@@ -90,7 +95,7 @@ class DatasetMapper:
|
||||
# @THROW: Exception - При ошибках чтения файла или парсинга.
|
||||
# @PARAM: file_path (str) - Путь к XLSX файлу.
|
||||
# @RETURN: Dict[str, str] - Словарь с меппингами.
|
||||
def load_excel_mappings(self, file_path: str) -> Dict[str, str]:
|
||||
def load_excel_mappings(self, file_path: str) -> dict[str, str]:
|
||||
with belief_scope("Load mappings from Excel"):
|
||||
app_logger.info("[load_excel_mappings][Enter] Loading mappings from %s.", file_path)
|
||||
try:
|
||||
@@ -118,11 +123,11 @@ class DatasetMapper:
|
||||
# @PARAM: excel_path (Optional[str]) - Путь к XLSX файлу.
|
||||
# @PARAM: table_name (Optional[str]) - Имя таблицы в PostgreSQL.
|
||||
# @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL.
|
||||
def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: Optional[Dict] = None, excel_path: Optional[str] = None, table_name: Optional[str] = None, table_schema: Optional[str] = None):
|
||||
def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: dict | None = None, excel_path: str | None = None, table_name: str | None = None, table_schema: str | None = None):
|
||||
with belief_scope(f"Run dataset mapping for ID {dataset_id}"):
|
||||
app_logger.info("[run_mapping][Enter] Starting dataset mapping for ID %d from source '%s'.", dataset_id, source)
|
||||
mappings: Dict[str, str] = {}
|
||||
|
||||
mappings: dict[str, str] = {}
|
||||
|
||||
try:
|
||||
if source in ['postgres', 'both']:
|
||||
assert postgres_config and table_name and table_schema, "Postgres config is required."
|
||||
@@ -135,13 +140,13 @@ class DatasetMapper:
|
||||
return
|
||||
dataset_response = superset_client.get_dataset(dataset_id)
|
||||
dataset_data = dataset_response['result']
|
||||
|
||||
|
||||
original_columns = dataset_data.get('columns', [])
|
||||
updated_columns = []
|
||||
changes_made = False
|
||||
for column in original_columns:
|
||||
col_name = column.get('column_name')
|
||||
|
||||
|
||||
new_column = {
|
||||
"column_name": col_name,
|
||||
"id": column.get("id"),
|
||||
@@ -158,14 +163,14 @@ class DatasetMapper:
|
||||
"uuid": column.get("uuid"),
|
||||
"verbose_name": column.get("verbose_name"),
|
||||
}
|
||||
|
||||
|
||||
new_column = {k: v for k, v in new_column.items() if v is not None}
|
||||
if col_name in mappings:
|
||||
mapping_value = mappings[col_name]
|
||||
if isinstance(mapping_value, str) and new_column.get('verbose_name') != mapping_value:
|
||||
new_column['verbose_name'] = mapping_value
|
||||
changes_made = True
|
||||
|
||||
|
||||
updated_columns.append(new_column)
|
||||
updated_metrics = []
|
||||
for metric in dataset_data.get("metrics", []):
|
||||
@@ -207,7 +212,7 @@ class DatasetMapper:
|
||||
"offset": dataset_data.get("offset"),
|
||||
"template_params": dataset_data.get("template_params"),
|
||||
}
|
||||
|
||||
|
||||
payload_for_update = {k: v for k, v in payload_for_update.items() if v is not None}
|
||||
superset_client.update_dataset(dataset_id, payload_for_update)
|
||||
app_logger.info("[run_mapping][Success] Dataset %d columns' verbose_name updated.", dataset_id)
|
||||
@@ -218,4 +223,4 @@ class DatasetMapper:
|
||||
return
|
||||
# #endregion run_mapping
|
||||
# #endregion DatasetMapper
|
||||
# #endregion DatasetMapperModule
|
||||
# #endregion DatasetMapperModule
|
||||
|
||||
@@ -7,16 +7,21 @@
|
||||
# @PUBLIC_API: create_temp_file, remove_empty_directories, read_dashboard_from_disk, calculate_crc32, RetentionPolicy, archive_exports, save_and_unpack_dashboard, update_yamls, create_dashboard_export, sanitize_filename, get_filename_from_headers, consolidate_archive_folders
|
||||
import os
|
||||
import re
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Tuple, Dict, List, Union, LiteralString, Generator
|
||||
from contextlib import contextmanager
|
||||
import tempfile
|
||||
from datetime import date, datetime
|
||||
import shutil
|
||||
import tempfile
|
||||
import zipfile
|
||||
import zlib
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from ..logger import logger as app_logger, belief_scope
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, LiteralString
|
||||
|
||||
from ..logger import belief_scope
|
||||
from ..logger import logger as app_logger
|
||||
|
||||
|
||||
# #region InvalidZipFormatError [TYPE Class]
|
||||
# @BRIEF Exception raised when a file is not a valid ZIP archive.
|
||||
class InvalidZipFormatError(Exception):
|
||||
@@ -28,7 +33,7 @@ class InvalidZipFormatError(Exception):
|
||||
# @POST: Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста.
|
||||
# @YIELDS: Path - Путь к временному ресурсу.
|
||||
@contextmanager
|
||||
def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode: str = 'wb', dry_run = False) -> Generator[Path, None, None]:
|
||||
def create_temp_file(content: bytes | None = None, suffix: str = ".zip", mode: str = 'wb', dry_run = False) -> Generator[Path]:
|
||||
with belief_scope("Create temporary resource"):
|
||||
resource_path = None
|
||||
is_dir = suffix.startswith('.dir')
|
||||
@@ -84,7 +89,7 @@ def remove_empty_directories(root_dir: str) -> int:
|
||||
# @BRIEF Читает бинарное содержимое файла с диска.
|
||||
# @PRE: file_path должен указывать на существующий файл.
|
||||
# @POST: Возвращает байты содержимого и имя файла.
|
||||
def read_dashboard_from_disk(file_path: str) -> Tuple[bytes, str]:
|
||||
def read_dashboard_from_disk(file_path: str) -> tuple[bytes, str]:
|
||||
with belief_scope(f"Read dashboard from {file_path}"):
|
||||
path = Path(file_path)
|
||||
assert path.is_file(), f"Файл дашборда не найден: {file_path}"
|
||||
@@ -99,10 +104,9 @@ def read_dashboard_from_disk(file_path: str) -> Tuple[bytes, str]:
|
||||
# @PRE: file_path должен быть объектом Path к существующему файлу.
|
||||
# @POST: Возвращает 8-значную hex-строку CRC32.
|
||||
def calculate_crc32(file_path: Path) -> str:
|
||||
with belief_scope(f"Calculate CRC32 for {file_path}"):
|
||||
with open(file_path, 'rb') as f:
|
||||
crc32_value = zlib.crc32(f.read())
|
||||
return f"{crc32_value:08x}"
|
||||
with belief_scope(f"Calculate CRC32 for {file_path}"), open(file_path, 'rb') as f:
|
||||
crc32_value = zlib.crc32(f.read())
|
||||
return f"{crc32_value:08x}"
|
||||
# #endregion calculate_crc32
|
||||
# #region RetentionPolicy [TYPE DataClass]
|
||||
# @BRIEF Определяет политику хранения для архивов (ежедневные, еженедельные, ежемесячные).
|
||||
@@ -125,7 +129,7 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
|
||||
app_logger.warning("[archive_exports][Skip] Archive directory not found: %s", output_dir)
|
||||
return
|
||||
app_logger.info("[archive_exports][Enter] Managing archive in %s", output_dir)
|
||||
|
||||
|
||||
# 1. Collect all zip files
|
||||
zip_files = list(output_path.glob("*.zip"))
|
||||
if not zip_files:
|
||||
@@ -136,10 +140,10 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
|
||||
app_logger.info("[archive_exports][State] Starting deduplication...")
|
||||
checksums = {}
|
||||
files_to_remove = []
|
||||
|
||||
|
||||
# Sort by modification time (newest first) to keep the latest version
|
||||
zip_files.sort(key=lambda f: f.stat().st_mtime, reverse=True)
|
||||
|
||||
|
||||
for file_path in zip_files:
|
||||
try:
|
||||
crc = calculate_crc32(file_path)
|
||||
@@ -150,7 +154,7 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
|
||||
checksums[crc] = file_path
|
||||
except Exception as e:
|
||||
app_logger.error("[archive_exports][Failure] Failed to calculate CRC32 for %s: %s", file_path, e)
|
||||
|
||||
|
||||
for f in files_to_remove:
|
||||
try:
|
||||
f.unlink()
|
||||
@@ -171,14 +175,14 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
|
||||
file_date = datetime.strptime(date_str, "%Y%m%d").date()
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
|
||||
if not file_date:
|
||||
# Fallback to modification time
|
||||
file_date = datetime.fromtimestamp(file_path.stat().st_mtime).date()
|
||||
|
||||
|
||||
files_with_dates.append((file_path, file_date))
|
||||
files_to_keep = apply_retention_policy(files_with_dates, policy)
|
||||
|
||||
|
||||
for file_path, _ in files_with_dates:
|
||||
if file_path not in files_to_keep:
|
||||
try:
|
||||
@@ -191,30 +195,30 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
|
||||
# @BRIEF (Helper) Применяет политику хранения к списку файлов, возвращая те, что нужно сохранить.
|
||||
# @PRE: files_with_dates is a list of (Path, date) tuples.
|
||||
# @POST: Returns a set of files to keep.
|
||||
def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: RetentionPolicy) -> set:
|
||||
def apply_retention_policy(files_with_dates: list[tuple[Path, date]], policy: RetentionPolicy) -> set:
|
||||
with belief_scope("Apply retention policy"):
|
||||
# Сортируем по дате (от новой к старой)
|
||||
sorted_files = sorted(files_with_dates, key=lambda x: x[1], reverse=True)
|
||||
# Словарь для хранения файлов по категориям
|
||||
daily_files = []
|
||||
weekly_files = []
|
||||
monthly_files = []
|
||||
today = date.today()
|
||||
for file_path, file_date in sorted_files:
|
||||
# Ежедневные
|
||||
if (today - file_date).days < policy.daily:
|
||||
daily_files.append(file_path)
|
||||
# Еженедельные
|
||||
elif (today - file_date).days < policy.weekly * 7:
|
||||
weekly_files.append(file_path)
|
||||
# Ежемесячные
|
||||
elif (today - file_date).days < policy.monthly * 30:
|
||||
monthly_files.append(file_path)
|
||||
# Возвращаем множество файлов, которые нужно сохранить
|
||||
files_to_keep = set()
|
||||
files_to_keep.update(daily_files)
|
||||
files_to_keep.update(weekly_files[:policy.weekly])
|
||||
files_to_keep.update(monthly_files[:policy.monthly])
|
||||
sorted_files = sorted(files_with_dates, key=lambda x: x[1], reverse=True)
|
||||
# Словарь для хранения файлов по категориям
|
||||
daily_files = []
|
||||
weekly_files = []
|
||||
monthly_files = []
|
||||
today = date.today()
|
||||
for file_path, file_date in sorted_files:
|
||||
# Ежедневные
|
||||
if (today - file_date).days < policy.daily:
|
||||
daily_files.append(file_path)
|
||||
# Еженедельные
|
||||
elif (today - file_date).days < policy.weekly * 7:
|
||||
weekly_files.append(file_path)
|
||||
# Ежемесячные
|
||||
elif (today - file_date).days < policy.monthly * 30:
|
||||
monthly_files.append(file_path)
|
||||
# Возвращаем множество файлов, которые нужно сохранить
|
||||
files_to_keep = set()
|
||||
files_to_keep.update(daily_files)
|
||||
files_to_keep.update(weekly_files[:policy.weekly])
|
||||
files_to_keep.update(monthly_files[:policy.monthly])
|
||||
app_logger.debug("[apply_retention_policy][State] Keeping %d files according to retention policy", len(files_to_keep))
|
||||
return files_to_keep
|
||||
# #endregion apply_retention_policy
|
||||
@@ -222,7 +226,7 @@ def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: Re
|
||||
# @BRIEF Сохраняет бинарное содержимое ZIP-архива на диск и опционально распаковывает его.
|
||||
# @PRE: zip_content должен быть байтами валидного ZIP-архива.
|
||||
# @POST: ZIP-файл сохранен, и если unpack=True, он распакован в output_dir.
|
||||
def save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path], unpack: bool = False, original_filename: Optional[str] = None) -> Tuple[Path, Optional[Path]]:
|
||||
def save_and_unpack_dashboard(zip_content: bytes, output_dir: str | Path, unpack: bool = False, original_filename: str | None = None) -> tuple[Path, Path | None]:
|
||||
with belief_scope("Save and unpack dashboard"):
|
||||
app_logger.info("[save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: %s", unpack)
|
||||
try:
|
||||
@@ -247,14 +251,14 @@ def save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path],
|
||||
# @PRE: path должен быть существующей директорией.
|
||||
# @POST: Все YAML файлы в директории обновлены согласно переданным параметрам.
|
||||
# @RELATION CALLS -> _update_yaml_file
|
||||
def update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str = "dashboards", regexp_pattern: Optional[LiteralString] = None, replace_string: Optional[LiteralString] = None) -> None:
|
||||
def update_yamls(db_configs: list[dict[str, Any]] | None = None, path: str = "dashboards", regexp_pattern: LiteralString | None = None, replace_string: LiteralString | None = None) -> None:
|
||||
with belief_scope("Update YAML configurations"):
|
||||
app_logger.info("[update_yamls][Enter] Starting YAML configuration update.")
|
||||
dir_path = Path(path)
|
||||
assert dir_path.is_dir(), f"Путь {path} не существует или не является директорией"
|
||||
|
||||
configs: List[Dict[str, Any]] = db_configs or []
|
||||
|
||||
|
||||
configs: list[dict[str, Any]] = db_configs or []
|
||||
|
||||
for file_path in dir_path.rglob("*.yaml"):
|
||||
_update_yaml_file(file_path, configs, regexp_pattern, replace_string)
|
||||
# #endregion update_yamls
|
||||
@@ -262,11 +266,11 @@ def update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str =
|
||||
# @BRIEF (Helper) Обновляет один YAML файл.
|
||||
# @PRE: file_path должен быть объектом Path к существующему YAML файлу.
|
||||
# @POST: Файл обновлен согласно переданным конфигурациям или регулярному выражению.
|
||||
def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_pattern: Optional[str], replace_string: Optional[str]) -> None:
|
||||
def _update_yaml_file(file_path: Path, db_configs: list[dict[str, Any]], regexp_pattern: str | None, replace_string: str | None) -> None:
|
||||
with belief_scope(f"Update YAML file: {file_path}"):
|
||||
# Читаем содержимое файла
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
with open(file_path, encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
except Exception as e:
|
||||
app_logger.error("[_update_yaml_file][Failure] Failed to read %s: %s", file_path, e)
|
||||
@@ -300,7 +304,7 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_
|
||||
val_pattern = re.escape(old_val)
|
||||
# Группы: 1=ключ+разделитель, 2=открывающая кавычка (опц), 3=значение, 4=закрывающая кавычка (опц)
|
||||
pattern = rf'({key_pattern}\s*:\s*)(["\']?)({val_pattern})(["\']?)'
|
||||
|
||||
|
||||
# #region replacer [TYPE Function]
|
||||
# @PURPOSE: Функция замены, сохраняющая кавычки если они были.
|
||||
# @PRE: match должен быть объектом совпадения регулярного выражения.
|
||||
@@ -323,7 +327,7 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_
|
||||
# @BRIEF Создает ZIP-архив из указанных исходных путей.
|
||||
# @PRE: source_paths должен содержать существующие пути.
|
||||
# @POST: ZIP-архив создан по пути zip_path.
|
||||
def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union[str, Path]], exclude_extensions: Optional[List[str]] = None) -> bool:
|
||||
def create_dashboard_export(zip_path: str | Path, source_paths: list[str | Path], exclude_extensions: list[str] | None = None) -> bool:
|
||||
with belief_scope(f"Create dashboard export: {zip_path}"):
|
||||
app_logger.info("[create_dashboard_export][Enter] Packing dashboard: %s -> %s", source_paths, zip_path)
|
||||
try:
|
||||
@@ -338,7 +342,7 @@ def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union
|
||||
zipf.write(item, arcname)
|
||||
app_logger.reflect(f"Archive created: {zip_path}", extra={"src": "create_dashboard_export"})
|
||||
return True
|
||||
except (IOError, zipfile.BadZipFile, AssertionError) as e:
|
||||
except (OSError, zipfile.BadZipFile, AssertionError) as e:
|
||||
app_logger.error("[create_dashboard_export][Failure] Error: %s", e, exc_info=True)
|
||||
return False
|
||||
# #endregion create_dashboard_export
|
||||
@@ -354,7 +358,7 @@ def sanitize_filename(filename: str) -> str:
|
||||
# @BRIEF Извлекает имя файла из HTTP заголовка 'Content-Disposition'.
|
||||
# @PRE: headers должен быть словарем заголовков.
|
||||
# @POST: Возвращает имя файла или None, если заголовок отсутствует.
|
||||
def get_filename_from_headers(headers: dict) -> Optional[str]:
|
||||
def get_filename_from_headers(headers: dict) -> str | None:
|
||||
with belief_scope("Get filename from headers"):
|
||||
content_disposition = headers.get("Content-Disposition", "")
|
||||
if match := re.search(r'filename="?([^"]+)"?', content_disposition):
|
||||
@@ -367,51 +371,51 @@ def get_filename_from_headers(headers: dict) -> Optional[str]:
|
||||
# @POST: Директории с одинаковым префиксом объединены в одну.
|
||||
def consolidate_archive_folders(root_directory: Path) -> None:
|
||||
with belief_scope(f"Consolidate archives in {root_directory}"):
|
||||
assert isinstance(root_directory, Path), "root_directory must be a Path object."
|
||||
assert root_directory.is_dir(), "root_directory must be an existing directory."
|
||||
|
||||
app_logger.info("[consolidate_archive_folders][Enter] Consolidating archives in %s", root_directory)
|
||||
# Собираем все директории с архивами
|
||||
archive_dirs = []
|
||||
for item in root_directory.iterdir():
|
||||
if item.is_dir():
|
||||
# Проверяем, есть ли в директории ZIP-архивы
|
||||
if any(item.glob("*.zip")):
|
||||
archive_dirs.append(item)
|
||||
# Группируем по слагу (части имени до первого '_')
|
||||
slug_groups = {}
|
||||
for dir_path in archive_dirs:
|
||||
dir_name = dir_path.name
|
||||
slug = dir_name.split('_')[0] if '_' in dir_name else dir_name
|
||||
if slug not in slug_groups:
|
||||
slug_groups[slug] = []
|
||||
slug_groups[slug].append(dir_path)
|
||||
# Для каждой группы консолидируем
|
||||
for slug, dirs in slug_groups.items():
|
||||
if len(dirs) <= 1:
|
||||
continue
|
||||
# Создаем целевую директорию
|
||||
target_dir = root_directory / slug
|
||||
target_dir.mkdir(exist_ok=True)
|
||||
app_logger.info("[consolidate_archive_folders][State] Consolidating %d directories under %s", len(dirs), target_dir)
|
||||
# Перемещаем содержимое
|
||||
for source_dir in dirs:
|
||||
if source_dir == target_dir:
|
||||
continue
|
||||
for item in source_dir.iterdir():
|
||||
dest_item = target_dir / item.name
|
||||
try:
|
||||
if item.is_dir():
|
||||
shutil.move(str(item), str(dest_item))
|
||||
else:
|
||||
shutil.move(str(item), str(dest_item))
|
||||
except Exception as e:
|
||||
app_logger.error("[consolidate_archive_folders][Failure] Failed to move %s to %s: %s", item, dest_item, e)
|
||||
# Удаляем исходную директорию
|
||||
try:
|
||||
source_dir.rmdir()
|
||||
app_logger.info("[consolidate_archive_folders][State] Removed source directory: %s", source_dir)
|
||||
except Exception as e:
|
||||
app_logger.error("[consolidate_archive_folders][Failure] Failed to remove source directory %s: %s", source_dir, e)
|
||||
assert isinstance(root_directory, Path), "root_directory must be a Path object."
|
||||
assert root_directory.is_dir(), "root_directory must be an existing directory."
|
||||
|
||||
app_logger.info("[consolidate_archive_folders][Enter] Consolidating archives in %s", root_directory)
|
||||
# Собираем все директории с архивами
|
||||
archive_dirs = []
|
||||
for item in root_directory.iterdir():
|
||||
if item.is_dir():
|
||||
# Проверяем, есть ли в директории ZIP-архивы
|
||||
if any(item.glob("*.zip")):
|
||||
archive_dirs.append(item)
|
||||
# Группируем по слагу (части имени до первого '_')
|
||||
slug_groups = {}
|
||||
for dir_path in archive_dirs:
|
||||
dir_name = dir_path.name
|
||||
slug = dir_name.split('_')[0] if '_' in dir_name else dir_name
|
||||
if slug not in slug_groups:
|
||||
slug_groups[slug] = []
|
||||
slug_groups[slug].append(dir_path)
|
||||
# Для каждой группы консолидируем
|
||||
for slug, dirs in slug_groups.items():
|
||||
if len(dirs) <= 1:
|
||||
continue
|
||||
# Создаем целевую директорию
|
||||
target_dir = root_directory / slug
|
||||
target_dir.mkdir(exist_ok=True)
|
||||
app_logger.info("[consolidate_archive_folders][State] Consolidating %d directories under %s", len(dirs), target_dir)
|
||||
# Перемещаем содержимое
|
||||
for source_dir in dirs:
|
||||
if source_dir == target_dir:
|
||||
continue
|
||||
for item in source_dir.iterdir():
|
||||
dest_item = target_dir / item.name
|
||||
try:
|
||||
if item.is_dir():
|
||||
shutil.move(str(item), str(dest_item))
|
||||
else:
|
||||
shutil.move(str(item), str(dest_item))
|
||||
except Exception as e:
|
||||
app_logger.error("[consolidate_archive_folders][Failure] Failed to move %s to %s: %s", item, dest_item, e)
|
||||
# Удаляем исходную директорию
|
||||
try:
|
||||
source_dir.rmdir()
|
||||
app_logger.info("[consolidate_archive_folders][State] Removed source directory: %s", source_dir)
|
||||
except Exception as e:
|
||||
app_logger.error("[consolidate_archive_folders][Failure] Failed to remove source directory %s: %s", source_dir, e)
|
||||
# #endregion consolidate_archive_folders
|
||||
# #endregion FileIO
|
||||
# #endregion FileIO
|
||||
|
||||
@@ -6,14 +6,15 @@
|
||||
#
|
||||
# @INVARIANT: Confidence scores are returned as floats between 0.0 and 1.0.
|
||||
|
||||
|
||||
from rapidfuzz import fuzz, process
|
||||
from typing import List, Dict
|
||||
|
||||
|
||||
# #region suggest_mappings [TYPE Function]
|
||||
# @BRIEF Suggests mappings between source and target databases using fuzzy matching.
|
||||
# @PRE: source_databases and target_databases are lists of dictionaries with 'uuid' and 'database_name'.
|
||||
# @POST: Returns a list of suggested mappings with confidence scores.
|
||||
def suggest_mappings(source_databases: List[Dict], target_databases: List[Dict], threshold: int = 60) -> List[Dict]:
|
||||
def suggest_mappings(source_databases: list[dict], target_databases: list[dict], threshold: int = 60) -> list[dict]:
|
||||
"""
|
||||
Suggest mappings between source and target databases using fuzzy matching.
|
||||
"""
|
||||
@@ -22,15 +23,15 @@ def suggest_mappings(source_databases: List[Dict], target_databases: List[Dict],
|
||||
return suggestions
|
||||
|
||||
target_names = [db['database_name'] for db in target_databases]
|
||||
|
||||
|
||||
for s_db in source_databases:
|
||||
# Use token_sort_ratio as decided in research.md
|
||||
match = process.extractOne(
|
||||
s_db['database_name'],
|
||||
target_names,
|
||||
s_db['database_name'],
|
||||
target_names,
|
||||
scorer=fuzz.token_sort_ratio
|
||||
)
|
||||
|
||||
|
||||
if match:
|
||||
name, score, index = match
|
||||
if score >= threshold:
|
||||
@@ -41,7 +42,7 @@ def suggest_mappings(source_databases: List[Dict], target_databases: List[Dict],
|
||||
"target_db_uuid": target_databases[index]['uuid'],
|
||||
"confidence": score / 100.0
|
||||
})
|
||||
|
||||
|
||||
return suggestions
|
||||
# #endregion suggest_mappings
|
||||
|
||||
|
||||
@@ -4,17 +4,22 @@
|
||||
# @LAYER: Infra
|
||||
# @RELATION DEPENDS_ON -> [LoggerModule]
|
||||
# @PUBLIC_API: APIClient
|
||||
from typing import Optional, Dict, Any, List, Union, cast, Tuple
|
||||
import json
|
||||
import io
|
||||
from pathlib import Path
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
import urllib3
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
from ..logger import logger as app_logger, belief_scope
|
||||
|
||||
from ..logger import belief_scope
|
||||
from ..logger import logger as app_logger
|
||||
|
||||
|
||||
# #region SupersetAPIError [C:1] [TYPE Class]
|
||||
# @BRIEF Base exception for all Superset API related errors.
|
||||
class SupersetAPIError(Exception):
|
||||
@@ -59,7 +64,7 @@ class DashboardNotFoundError(SupersetAPIError):
|
||||
# @PURPOSE: Initializes the not found error with resource ID.
|
||||
# @PRE: resource_id is provided.
|
||||
# @POST: DashboardNotFoundError is initialized.
|
||||
def __init__(self, resource_id: Union[int, str], message: str = "Dashboard not found", **context: Any):
|
||||
def __init__(self, resource_id: int | str, message: str = "Dashboard not found", **context: Any):
|
||||
with belief_scope("DashboardNotFoundError.__init__"):
|
||||
super().__init__(f"Dashboard '{resource_id}' {message}", subtype="not_found", resource_id=resource_id, **context)
|
||||
# #endregion __init__
|
||||
@@ -84,16 +89,16 @@ class NetworkError(Exception):
|
||||
class SupersetAuthCache:
|
||||
TTL_SECONDS = 300
|
||||
_lock = threading.Lock()
|
||||
_entries: Dict[Tuple[str, str, bool], Dict[str, Any]] = {}
|
||||
_entries: dict[tuple[str, str, bool], dict[str, Any]] = {}
|
||||
@classmethod
|
||||
def build_key(cls, base_url: str, auth: Optional[Dict[str, Any]], verify_ssl: bool) -> Tuple[str, str, bool]:
|
||||
def build_key(cls, base_url: str, auth: dict[str, Any] | None, verify_ssl: bool) -> tuple[str, str, bool]:
|
||||
username = ""
|
||||
if isinstance(auth, dict):
|
||||
username = str(auth.get("username") or "").strip()
|
||||
return (str(base_url or "").strip(), username, bool(verify_ssl))
|
||||
@classmethod
|
||||
# #region SupersetAuthCache.get [TYPE Function]
|
||||
def get(cls, key: Tuple[str, str, bool]) -> Optional[Dict[str, str]]:
|
||||
def get(cls, key: tuple[str, str, bool]) -> dict[str, str] | None:
|
||||
now = time.time()
|
||||
with cls._lock:
|
||||
payload = cls._entries.get(key)
|
||||
@@ -114,7 +119,7 @@ class SupersetAuthCache:
|
||||
# #endregion SupersetAuthCache.get
|
||||
@classmethod
|
||||
# #region SupersetAuthCache.set [TYPE Function]
|
||||
def set(cls, key: Tuple[str, str, bool], tokens: Dict[str, str], ttl_seconds: Optional[int] = None) -> None:
|
||||
def set(cls, key: tuple[str, str, bool], tokens: dict[str, str], ttl_seconds: int | None = None) -> None:
|
||||
normalized_ttl = max(int(ttl_seconds or cls.TTL_SECONDS), 1)
|
||||
with cls._lock:
|
||||
cls._entries[key] = {
|
||||
@@ -126,7 +131,7 @@ class SupersetAuthCache:
|
||||
}
|
||||
# #endregion SupersetAuthCache.set
|
||||
@classmethod
|
||||
def invalidate(cls, key: Tuple[str, str, bool]) -> None:
|
||||
def invalidate(cls, key: tuple[str, str, bool]) -> None:
|
||||
with cls._lock:
|
||||
cls._entries.pop(key, None)
|
||||
# #endregion SupersetAuthCache
|
||||
@@ -143,7 +148,7 @@ class APIClient:
|
||||
# @PARAM: timeout (int) - Таймаут запросов.
|
||||
# @PRE: config must contain 'base_url' and 'auth'.
|
||||
# @POST: APIClient instance is initialized with a session.
|
||||
def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
|
||||
def __init__(self, config: dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
|
||||
with belief_scope("__init__"):
|
||||
app_logger.reason("Initializing APIClient.", extra={"src": "APIClient.__init__"})
|
||||
self.base_url: str = self._normalize_base_url(config.get("base_url", ""))
|
||||
@@ -151,7 +156,7 @@ class APIClient:
|
||||
self.auth = config.get("auth")
|
||||
self.request_settings = {"verify_ssl": verify_ssl, "timeout": timeout}
|
||||
self.session = self._init_session()
|
||||
self._tokens: Dict[str, str] = {}
|
||||
self._tokens: dict[str, str] = {}
|
||||
self._auth_cache_key = SupersetAuthCache.build_key(
|
||||
self.base_url,
|
||||
self.auth,
|
||||
@@ -168,33 +173,34 @@ class APIClient:
|
||||
def _init_session(self) -> requests.Session:
|
||||
with belief_scope("_init_session"):
|
||||
session = requests.Session()
|
||||
|
||||
|
||||
# Create a custom adapter that handles TLS issues
|
||||
class TLSAdapter(HTTPAdapter):
|
||||
def init_poolmanager(self, connections, maxsize, block=False):
|
||||
from urllib3.poolmanager import PoolManager
|
||||
import ssl
|
||||
|
||||
|
||||
from urllib3.poolmanager import PoolManager
|
||||
|
||||
# Create an SSL context that ignores TLSv1 unrecognized name errors
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.set_ciphers('HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!SRP:!CAMELLIA')
|
||||
|
||||
|
||||
# Ignore TLSV1_UNRECOGNIZED_NAME errors by disabling hostname verification
|
||||
# This is safe when verify_ssl is false (we're already not verifying the certificate)
|
||||
ctx.check_hostname = False
|
||||
|
||||
|
||||
self.poolmanager = PoolManager(
|
||||
num_pools=connections,
|
||||
maxsize=maxsize,
|
||||
block=block,
|
||||
ssl_context=ctx
|
||||
)
|
||||
|
||||
|
||||
retries = Retry(total=3, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504])
|
||||
adapter = TLSAdapter(max_retries=retries)
|
||||
session.mount('http://', adapter)
|
||||
session.mount('https://', adapter)
|
||||
|
||||
|
||||
if not self.request_settings["verify_ssl"]:
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
app_logger.warning("[_init_session][State] SSL verification disabled.")
|
||||
@@ -202,7 +208,7 @@ class APIClient:
|
||||
session.verify = False
|
||||
else:
|
||||
session.verify = True
|
||||
|
||||
|
||||
return session
|
||||
# #endregion _init_session
|
||||
# #region _normalize_base_url [TYPE Function]
|
||||
@@ -239,7 +245,7 @@ class APIClient:
|
||||
# @THROW: AuthenticationError, NetworkError - при ошибках.
|
||||
# @RELATION: [CALLS] ->[SupersetAuthCache.get]
|
||||
# @RELATION: [CALLS] ->[SupersetAuthCache.set]
|
||||
def authenticate(self) -> Dict[str, str]:
|
||||
def authenticate(self) -> dict[str, str]:
|
||||
with belief_scope("authenticate"):
|
||||
app_logger.info("[authenticate][Enter] Authenticating to %s", self.base_url)
|
||||
cached_tokens = SupersetAuthCache.get(self._auth_cache_key)
|
||||
@@ -288,19 +294,19 @@ class APIClient:
|
||||
masked_auth = {k: ("******" if k == "password" else v) for k, v in self.auth.items()}
|
||||
app_logger.info(f"[authenticate][Debug] Login URL: {login_url}")
|
||||
app_logger.info(f"[authenticate][Debug] Auth payload: {masked_auth}")
|
||||
|
||||
|
||||
response = self.session.post(login_url, json=self.auth, timeout=self.request_settings["timeout"])
|
||||
|
||||
|
||||
if response.status_code != 200:
|
||||
app_logger.error(f"[authenticate][Error] Status: {response.status_code}, Response: {response.text}")
|
||||
|
||||
|
||||
response.raise_for_status()
|
||||
access_token = response.json()["access_token"]
|
||||
|
||||
|
||||
csrf_url = f"{self.api_base_url}/security/csrf_token/"
|
||||
csrf_response = self.session.get(csrf_url, headers={"Authorization": f"Bearer {access_token}"}, timeout=self.request_settings["timeout"])
|
||||
csrf_response.raise_for_status()
|
||||
|
||||
|
||||
self._tokens = {"access_token": access_token, "csrf_token": csrf_response.json()["result"]}
|
||||
self._authenticated = True
|
||||
SupersetAuthCache.set(self._auth_cache_key, self._tokens)
|
||||
@@ -321,7 +327,7 @@ class APIClient:
|
||||
# @PURPOSE: Возвращает HTTP-заголовки для аутентифицированных запросов.
|
||||
# @PRE: APIClient is initialized and authenticated or can be authenticated.
|
||||
# @POST: Returns headers including auth tokens.
|
||||
def headers(self) -> Dict[str, str]:
|
||||
def headers(self) -> dict[str, str]:
|
||||
if not self._authenticated:
|
||||
self.authenticate()
|
||||
return {
|
||||
@@ -341,12 +347,12 @@ class APIClient:
|
||||
# @POST: Returns response content or raw Response object.
|
||||
# @RETURN: `requests.Response` если `raw_response=True`, иначе `dict`.
|
||||
# @THROW: SupersetAPIError, NetworkError и их подклассы.
|
||||
def request(self, method: str, endpoint: str, headers: Optional[Dict] = None, raw_response: bool = False, **kwargs) -> Union[requests.Response, Dict[str, Any]]:
|
||||
def request(self, method: str, endpoint: str, headers: dict | None = None, raw_response: bool = False, **kwargs) -> requests.Response | dict[str, Any]:
|
||||
full_url = self._build_api_url(endpoint)
|
||||
_headers = self.headers.copy()
|
||||
if headers:
|
||||
_headers.update(headers)
|
||||
|
||||
|
||||
try:
|
||||
response = self.session.request(method, full_url, headers=_headers, **kwargs)
|
||||
response.raise_for_status()
|
||||
@@ -429,15 +435,15 @@ class APIClient:
|
||||
# @POST: File is uploaded and response returned.
|
||||
# @RETURN: Ответ API в виде словаря.
|
||||
# @THROW: SupersetAPIError, NetworkError, TypeError.
|
||||
def upload_file(self, endpoint: str, file_info: Dict[str, Any], extra_data: Optional[Dict] = None, timeout: Optional[int] = None) -> Dict:
|
||||
def upload_file(self, endpoint: str, file_info: dict[str, Any], extra_data: dict | None = None, timeout: int | None = None) -> dict:
|
||||
with belief_scope("upload_file"):
|
||||
full_url = self._build_api_url(endpoint)
|
||||
_headers = self.headers.copy()
|
||||
_headers.pop('Content-Type', None)
|
||||
|
||||
|
||||
|
||||
|
||||
file_obj, file_name, form_field = file_info.get("file_obj"), file_info.get("file_name"), file_info.get("form_field", "file")
|
||||
|
||||
|
||||
files_payload = {}
|
||||
if isinstance(file_obj, (str, Path)):
|
||||
with open(file_obj, 'rb') as f:
|
||||
@@ -446,7 +452,7 @@ class APIClient:
|
||||
files_payload = {form_field: (file_name, file_obj.getvalue(), 'application/x-zip-compressed')}
|
||||
else:
|
||||
raise TypeError(f"Unsupported file_obj type: {type(file_obj)}")
|
||||
|
||||
|
||||
return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout)
|
||||
# #endregion upload_file
|
||||
# #region _perform_upload [TYPE Function]
|
||||
@@ -459,7 +465,7 @@ class APIClient:
|
||||
# @PRE: url, files, and headers must be provided.
|
||||
# @POST: POST request is performed and JSON response returned.
|
||||
# @RETURN: Dict - Ответ.
|
||||
def _perform_upload(self, url: str, files: Dict, data: Optional[Dict], headers: Dict, timeout: Optional[int]) -> Dict:
|
||||
def _perform_upload(self, url: str, files: dict, data: dict | None, headers: dict, timeout: int | None) -> dict:
|
||||
with belief_scope("_perform_upload"):
|
||||
try:
|
||||
response = self.session.post(url, files=files, data=data or {}, headers=headers, timeout=timeout or self.request_settings["timeout"])
|
||||
@@ -484,9 +490,9 @@ class APIClient:
|
||||
# @PRE: query_params must be a dictionary.
|
||||
# @POST: Returns total count of items.
|
||||
# @RETURN: int - Количество.
|
||||
def fetch_paginated_count(self, endpoint: str, query_params: Dict, count_field: str = "count") -> int:
|
||||
def fetch_paginated_count(self, endpoint: str, query_params: dict, count_field: str = "count") -> int:
|
||||
with belief_scope("fetch_paginated_count"):
|
||||
response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query_params)}))
|
||||
response_json = cast(dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query_params)}))
|
||||
return response_json.get(count_field, 0)
|
||||
# #endregion fetch_paginated_count
|
||||
# #region fetch_paginated_data [TYPE Function]
|
||||
@@ -496,26 +502,26 @@ class APIClient:
|
||||
# @PRE: pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional.
|
||||
# @POST: Returns all items across all pages.
|
||||
# @RETURN: List[Any] - Список данных.
|
||||
def fetch_paginated_data(self, endpoint: str, pagination_options: Dict[str, Any]) -> List[Any]:
|
||||
def fetch_paginated_data(self, endpoint: str, pagination_options: dict[str, Any]) -> list[Any]:
|
||||
with belief_scope("fetch_paginated_data"):
|
||||
base_query = pagination_options["base_query"]
|
||||
total_count = pagination_options.get("total_count")
|
||||
|
||||
|
||||
results_field = pagination_options["results_field"]
|
||||
count_field = pagination_options.get("count_field", "count")
|
||||
page_size = base_query.get('page_size', 1000)
|
||||
assert page_size > 0, "'page_size' must be a positive number."
|
||||
|
||||
|
||||
results = []
|
||||
page = 0
|
||||
|
||||
|
||||
# Fetch first page to get data and total count if not provided
|
||||
query = {**base_query, 'page': page}
|
||||
response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)}))
|
||||
|
||||
response_json = cast(dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)}))
|
||||
|
||||
first_page_results = response_json.get(results_field, [])
|
||||
results.extend(first_page_results)
|
||||
|
||||
|
||||
if total_count is None:
|
||||
total_count = response_json.get(count_field, len(first_page_results))
|
||||
app_logger.debug(f"[fetch_paginated_data][State] Total count resolved from first page: {total_count}")
|
||||
@@ -523,10 +529,10 @@ class APIClient:
|
||||
total_pages = (total_count + page_size - 1) // page_size
|
||||
for page in range(1, total_pages):
|
||||
query = {**base_query, 'page': page}
|
||||
response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)}))
|
||||
response_json = cast(dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)}))
|
||||
results.extend(response_json.get(results_field, []))
|
||||
|
||||
|
||||
return results
|
||||
# #endregion fetch_paginated_data
|
||||
# #endregion APIClient
|
||||
# #endregion NetworkModule
|
||||
# #endregion NetworkModule
|
||||
|
||||
@@ -8,14 +8,18 @@
|
||||
# @SIDE_EFFECT: performs upstream Superset preview and SQL Lab calls.
|
||||
# @INVARIANT: The adapter never fabricates compiled SQL locally; preview truth is delegated to Superset only.
|
||||
from __future__ import annotations
|
||||
|
||||
# #region SupersetCompilationAdapter.imports [TYPE Block]
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any
|
||||
|
||||
from src.core.config_models import Environment
|
||||
from src.core.logger import belief_scope, logger
|
||||
from src.core.superset_client import SupersetClient
|
||||
from src.models.dataset_review import CompiledPreview, PreviewStatus
|
||||
|
||||
|
||||
# #endregion SupersetCompilationAdapter.imports
|
||||
# #region PreviewCompilationPayload [C:2] [TYPE Class]
|
||||
# @BRIEF Typed preview payload for Superset-side compilation.
|
||||
@@ -24,8 +28,8 @@ class PreviewCompilationPayload:
|
||||
session_id: str
|
||||
dataset_id: int
|
||||
preview_fingerprint: str
|
||||
template_params: Dict[str, Any]
|
||||
effective_filters: List[Dict[str, Any]]
|
||||
template_params: dict[str, Any]
|
||||
effective_filters: list[dict[str, Any]]
|
||||
# #endregion PreviewCompilationPayload
|
||||
# #region SqlLabLaunchPayload [C:2] [TYPE Class]
|
||||
# @BRIEF Typed SQL Lab payload for audited launch handoff.
|
||||
@@ -35,7 +39,7 @@ class SqlLabLaunchPayload:
|
||||
dataset_id: int
|
||||
preview_id: str
|
||||
compiled_sql: str
|
||||
template_params: Dict[str, Any]
|
||||
template_params: dict[str, Any]
|
||||
# #endregion SqlLabLaunchPayload
|
||||
# #region SupersetCompilationAdapter [C:4] [TYPE Class]
|
||||
# @BRIEF Delegate preview compilation and SQL Lab launch to Superset without local SQL fabrication.
|
||||
@@ -47,7 +51,7 @@ class SupersetCompilationAdapter:
|
||||
# #region SupersetCompilationAdapter.__init__ [TYPE Function] [C:2]
|
||||
# @PURPOSE: Bind adapter to one Superset environment and client instance.
|
||||
def __init__(
|
||||
self, environment: Environment, client: Optional[SupersetClient] = None
|
||||
self, environment: Environment, client: SupersetClient | None = None
|
||||
) -> None:
|
||||
self.environment = environment
|
||||
self.client = client or SupersetClient(environment)
|
||||
@@ -221,7 +225,7 @@ class SupersetCompilationAdapter:
|
||||
# @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[Dict[str,Any]]
|
||||
def _request_superset_preview(
|
||||
self, payload: PreviewCompilationPayload
|
||||
) -> Dict[str, Any]:
|
||||
) -> dict[str, Any]:
|
||||
direct_compile_preview = getattr(self.client, "compile_preview", None)
|
||||
if self._supports_client_method("compile_preview") and callable(
|
||||
direct_compile_preview
|
||||
@@ -313,7 +317,7 @@ class SupersetCompilationAdapter:
|
||||
"Superset preview compilation response could not be normalized"
|
||||
)
|
||||
return normalized
|
||||
errors: List[str] = []
|
||||
errors: list[str] = []
|
||||
for endpoint in (
|
||||
f"/dataset/{payload.dataset_id}/preview",
|
||||
f"/dataset/{payload.dataset_id}/sql",
|
||||
@@ -357,7 +361,7 @@ class SupersetCompilationAdapter:
|
||||
# @POST: returns the first successful SQL Lab execution response from Superset.
|
||||
# @SIDE_EFFECT: issues Superset dataset lookup and SQL Lab execution requests.
|
||||
# @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[Dict[str,Any]]
|
||||
def _request_sql_lab_session(self, payload: SqlLabLaunchPayload) -> Dict[str, Any]:
|
||||
def _request_sql_lab_session(self, payload: SqlLabLaunchPayload) -> dict[str, Any]:
|
||||
dataset_raw = self.client.get_dataset(payload.dataset_id)
|
||||
dataset_record = (
|
||||
dataset_raw.get("result", dataset_raw)
|
||||
@@ -384,7 +388,7 @@ class SupersetCompilationAdapter:
|
||||
{"kind": "network", "target": "/sqllab/execute/", "http_method": "POST"},
|
||||
{"kind": "network", "target": "/sql_lab/execute/", "http_method": "POST"},
|
||||
]
|
||||
errors: List[str] = []
|
||||
errors: list[str] = []
|
||||
for candidate in candidate_calls:
|
||||
try:
|
||||
response = self.client.network.request(
|
||||
@@ -408,7 +412,7 @@ class SupersetCompilationAdapter:
|
||||
# #region SupersetCompilationAdapter._normalize_preview_response [TYPE Function] [C:3]
|
||||
# @PURPOSE: Normalize candidate Superset preview responses into one compiled-sql structure.
|
||||
# @RELATION: [DEPENDS_ON] ->[CompiledPreview]
|
||||
def _normalize_preview_response(self, response: Any) -> Optional[Dict[str, Any]]:
|
||||
def _normalize_preview_response(self, response: Any) -> dict[str, Any] | None:
|
||||
if not isinstance(response, dict):
|
||||
return None
|
||||
compiled_sql_candidates = [
|
||||
@@ -436,9 +440,9 @@ class SupersetCompilationAdapter:
|
||||
# #endregion SupersetCompilationAdapter._normalize_preview_response
|
||||
# #region SupersetCompilationAdapter._dump_json [TYPE Function] [C:1]
|
||||
# @PURPOSE: Serialize Superset request payload deterministically for network transport.
|
||||
def _dump_json(self, payload: Dict[str, Any]) -> str:
|
||||
def _dump_json(self, payload: dict[str, Any]) -> str:
|
||||
import json
|
||||
return json.dumps(payload, sort_keys=True, default=str)
|
||||
# #endregion SupersetCompilationAdapter._dump_json
|
||||
# #endregion SupersetCompilationAdapter
|
||||
# #endregion SupersetCompilationAdapter
|
||||
# #endregion SupersetCompilationAdapter
|
||||
|
||||
@@ -17,11 +17,11 @@
|
||||
# #endregion SupersetContextExtractorPackage
|
||||
|
||||
from ._base import SupersetContextExtractorBase, SupersetParsedContext
|
||||
from ._filters import SupersetContextFiltersExtractMixin
|
||||
from ._parsing import SupersetContextParsingMixin
|
||||
from ._pii import _mask_sensitive_text, mask_pii_value, sanitize_imported_filter_for_assistant
|
||||
from ._recovery import SupersetContextRecoveryMixin
|
||||
from ._templates import SupersetContextTemplatesMixin
|
||||
from ._filters import SupersetContextFiltersExtractMixin
|
||||
from ._pii import mask_pii_value, sanitize_imported_filter_for_assistant, _mask_sensitive_text
|
||||
|
||||
|
||||
# #region SupersetContextExtractor [C:4] [TYPE Class]
|
||||
@@ -57,7 +57,7 @@ class SupersetContextExtractor(
|
||||
__all__ = [
|
||||
"SupersetContextExtractor",
|
||||
"SupersetParsedContext",
|
||||
"_mask_sensitive_text",
|
||||
"mask_pii_value",
|
||||
"sanitize_imported_filter_for_assistant",
|
||||
"_mask_sensitive_text",
|
||||
]
|
||||
|
||||
@@ -11,15 +11,16 @@
|
||||
# #endregion SupersetContextExtractorBase
|
||||
# #region _base_imports [TYPE Block]
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Set, cast
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
from typing import Any, cast
|
||||
from urllib.parse import unquote
|
||||
|
||||
from ...config_models import Environment
|
||||
from ...logger import belief_scope, logger
|
||||
from ...logger import logger
|
||||
from ...superset_client import SupersetClient
|
||||
|
||||
# #endregion _base_imports
|
||||
logger = cast(Any, logger)
|
||||
# #region SupersetParsedContext [C:2] [TYPE Class]
|
||||
@@ -28,15 +29,15 @@ logger = cast(Any, logger)
|
||||
class SupersetParsedContext:
|
||||
source_url: str
|
||||
dataset_ref: str
|
||||
dataset_id: Optional[int] = None
|
||||
dashboard_id: Optional[int] = None
|
||||
chart_id: Optional[int] = None
|
||||
dataset_id: int | None = None
|
||||
dashboard_id: int | None = None
|
||||
chart_id: int | None = None
|
||||
resource_type: str = "unknown"
|
||||
query_state: Dict[str, Any] = field(default_factory=dict)
|
||||
imported_filters: List[Dict[str, Any]] = field(default_factory=list)
|
||||
unresolved_references: List[str] = field(default_factory=list)
|
||||
query_state: dict[str, Any] = field(default_factory=dict)
|
||||
imported_filters: list[dict[str, Any]] = field(default_factory=list)
|
||||
unresolved_references: list[str] = field(default_factory=list)
|
||||
partial_recovery: bool = False
|
||||
dataset_payload: Optional[Dict[str, Any]] = None
|
||||
dataset_payload: dict[str, Any] | None = None
|
||||
# #endregion SupersetParsedContext
|
||||
# #region SupersetContextExtractorBase [C:4] [TYPE Class]
|
||||
# @BRIEF Bind extractor to one Superset environment and client instance; provide shared URL-parsing helpers.
|
||||
@@ -48,7 +49,7 @@ class SupersetContextExtractorBase:
|
||||
# #region SupersetContextExtractorBase.__init__ [TYPE Function] [C:2]
|
||||
# @PURPOSE: Bind extractor to one Superset environment and client instance.
|
||||
def __init__(
|
||||
self, environment: Environment, client: Optional[SupersetClient] = None
|
||||
self, environment: Environment, client: SupersetClient | None = None
|
||||
) -> None:
|
||||
self.environment = environment
|
||||
self.client = client or SupersetClient(environment)
|
||||
@@ -57,7 +58,7 @@ class SupersetContextExtractorBase:
|
||||
# @PURPOSE: Summarize recovered, partial, and unresolved context for session state and UX.
|
||||
def build_recovery_summary(
|
||||
self, parsed_context: SupersetParsedContext
|
||||
) -> Dict[str, Any]:
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"dataset_ref": parsed_context.dataset_ref,
|
||||
"dataset_id": parsed_context.dataset_id,
|
||||
@@ -71,8 +72,8 @@ class SupersetContextExtractorBase:
|
||||
# #region SupersetContextExtractorBase._extract_numeric_identifier [TYPE Function] [C:2]
|
||||
# @PURPOSE: Extract a numeric identifier from a REST-like Superset URL path.
|
||||
def _extract_numeric_identifier(
|
||||
self, path_parts: List[str], resource_name: str
|
||||
) -> Optional[int]:
|
||||
self, path_parts: list[str], resource_name: str
|
||||
) -> int | None:
|
||||
if resource_name not in path_parts:
|
||||
return None
|
||||
try:
|
||||
@@ -88,7 +89,7 @@ class SupersetContextExtractorBase:
|
||||
# #endregion SupersetContextExtractorBase._extract_numeric_identifier
|
||||
# #region SupersetContextExtractorBase._extract_dashboard_reference [TYPE Function] [C:2]
|
||||
# @PURPOSE: Extract a dashboard id-or-slug reference from a Superset URL path.
|
||||
def _extract_dashboard_reference(self, path_parts: List[str]) -> Optional[str]:
|
||||
def _extract_dashboard_reference(self, path_parts: list[str]) -> str | None:
|
||||
if "dashboard" not in path_parts:
|
||||
return None
|
||||
try:
|
||||
@@ -104,7 +105,7 @@ class SupersetContextExtractorBase:
|
||||
# #endregion SupersetContextExtractorBase._extract_dashboard_reference
|
||||
# #region SupersetContextExtractorBase._extract_dashboard_permalink_key [TYPE Function] [C:2]
|
||||
# @PURPOSE: Extract a dashboard permalink key from a Superset URL path.
|
||||
def _extract_dashboard_permalink_key(self, path_parts: List[str]) -> Optional[str]:
|
||||
def _extract_dashboard_permalink_key(self, path_parts: list[str]) -> str | None:
|
||||
if "dashboard" not in path_parts:
|
||||
return None
|
||||
try:
|
||||
@@ -121,7 +122,7 @@ class SupersetContextExtractorBase:
|
||||
# #endregion SupersetContextExtractorBase._extract_dashboard_permalink_key
|
||||
# #region SupersetContextExtractorBase._extract_dashboard_id_from_state [TYPE Function] [C:2]
|
||||
# @PURPOSE: Extract a dashboard identifier from returned permalink state when present.
|
||||
def _extract_dashboard_id_from_state(self, state: Dict[str, Any]) -> Optional[int]:
|
||||
def _extract_dashboard_id_from_state(self, state: dict[str, Any]) -> int | None:
|
||||
return self._search_nested_numeric_key(
|
||||
payload=state,
|
||||
candidate_keys={"dashboardId", "dashboard_id", "dashboard_id_value"},
|
||||
@@ -129,7 +130,7 @@ class SupersetContextExtractorBase:
|
||||
# #endregion SupersetContextExtractorBase._extract_dashboard_id_from_state
|
||||
# #region SupersetContextExtractorBase._extract_chart_id_from_state [TYPE Function] [C:2]
|
||||
# @PURPOSE: Extract a chart identifier from returned permalink state when dashboard id is absent.
|
||||
def _extract_chart_id_from_state(self, state: Dict[str, Any]) -> Optional[int]:
|
||||
def _extract_chart_id_from_state(self, state: dict[str, Any]) -> int | None:
|
||||
return self._search_nested_numeric_key(
|
||||
payload=state,
|
||||
candidate_keys={"slice_id", "sliceId", "chartId", "chart_id"},
|
||||
@@ -139,8 +140,8 @@ class SupersetContextExtractorBase:
|
||||
# @PURPOSE: Recursively search nested dict/list payloads for the first numeric value under a candidate key set.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase]
|
||||
def _search_nested_numeric_key(
|
||||
self, payload: Any, candidate_keys: Set[str]
|
||||
) -> Optional[int]:
|
||||
self, payload: Any, candidate_keys: set[str]
|
||||
) -> int | None:
|
||||
if isinstance(payload, dict):
|
||||
for key, value in payload.items():
|
||||
if key in candidate_keys:
|
||||
@@ -161,8 +162,8 @@ class SupersetContextExtractorBase:
|
||||
# #endregion SupersetContextExtractorBase._search_nested_numeric_key
|
||||
# #region SupersetContextExtractorBase._decode_query_state [TYPE Function] [C:2]
|
||||
# @PURPOSE: Decode query-string structures used by Superset URL state transport.
|
||||
def _decode_query_state(self, query_params: Dict[str, List[str]]) -> Dict[str, Any]:
|
||||
query_state: Dict[str, Any] = {}
|
||||
def _decode_query_state(self, query_params: dict[str, list[str]]) -> dict[str, Any]:
|
||||
query_state: dict[str, Any] = {}
|
||||
for key, values in query_params.items():
|
||||
if not values:
|
||||
continue
|
||||
@@ -180,4 +181,4 @@ class SupersetContextExtractorBase:
|
||||
query_state[key] = decoded_value
|
||||
return query_state
|
||||
# #endregion SupersetContextExtractorBase._decode_query_state
|
||||
# #endregion SupersetContextExtractorBase
|
||||
# #endregion SupersetContextExtractorBase
|
||||
|
||||
@@ -5,9 +5,12 @@
|
||||
# #endregion SupersetContextFiltersExtractMixin
|
||||
# #region _filters_imports [TYPE Block]
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, List, cast
|
||||
from typing import Any, cast
|
||||
|
||||
from ...logger import logger as app_logger
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
# #endregion _filters_imports
|
||||
# #region SupersetContextFiltersExtractMixin [C:3] [TYPE Class]
|
||||
@@ -16,9 +19,9 @@ class SupersetContextFiltersExtractMixin:
|
||||
# #region SupersetContextFiltersExtractMixin._extract_imported_filters [TYPE Function] [C:2]
|
||||
# @PURPOSE: Normalize imported filters from decoded query state without fabricating missing values.
|
||||
def _extract_imported_filters(
|
||||
self, query_state: Dict[str, Any]
|
||||
) -> List[Dict[str, Any]]:
|
||||
imported_filters: List[Dict[str, Any]] = []
|
||||
self, query_state: dict[str, Any]
|
||||
) -> list[dict[str, Any]]:
|
||||
imported_filters: list[dict[str, Any]] = []
|
||||
native_filters_payload = query_state.get("native_filters")
|
||||
if isinstance(native_filters_payload, list):
|
||||
for index, item in enumerate(native_filters_payload):
|
||||
@@ -230,4 +233,4 @@ class SupersetContextFiltersExtractMixin:
|
||||
)
|
||||
return imported_filters
|
||||
# #endregion SupersetContextFiltersExtractMixin._extract_imported_filters
|
||||
# #endregion SupersetContextFiltersExtractMixin
|
||||
# #endregion SupersetContextFiltersExtractMixin
|
||||
|
||||
@@ -6,10 +6,13 @@
|
||||
# #endregion SupersetContextParsingMixin
|
||||
# #region _parsing_imports [TYPE Block]
|
||||
from __future__ import annotations
|
||||
from typing import Any, Dict, List, Optional, cast
|
||||
|
||||
from typing import Any, cast
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from ...logger import belief_scope, logger
|
||||
from ._base import SupersetParsedContext
|
||||
|
||||
logger = cast(Any, logger)
|
||||
# #endregion _parsing_imports
|
||||
# #region SupersetContextParsingMixin [C:4] [TYPE Class]
|
||||
@@ -48,9 +51,9 @@ class SupersetContextParsingMixin:
|
||||
dashboard_permalink_key = self._extract_dashboard_permalink_key(path_parts)
|
||||
chart_id = self._extract_numeric_identifier(path_parts, "chart")
|
||||
resource_type = "unknown"
|
||||
dataset_ref: Optional[str] = None
|
||||
dataset_ref: str | None = None
|
||||
partial_recovery = False
|
||||
unresolved_references: List[str] = []
|
||||
unresolved_references: list[str] = []
|
||||
if dataset_id is not None:
|
||||
resource_type = "dataset"
|
||||
dataset_ref = f"dataset:{dataset_id}"
|
||||
@@ -253,7 +256,7 @@ class SupersetContextParsingMixin:
|
||||
extra={"path": parsed_url.path},
|
||||
)
|
||||
raise ValueError("Unsupported Superset link shape")
|
||||
dataset_payload: Optional[Dict[str, Any]] = None
|
||||
dataset_payload: dict[str, Any] | None = None
|
||||
if dataset_id is not None:
|
||||
try:
|
||||
dataset_payload = self.client.get_dataset_detail(dataset_id)
|
||||
@@ -317,9 +320,9 @@ class SupersetContextParsingMixin:
|
||||
def _recover_dataset_binding_from_dashboard(
|
||||
self,
|
||||
dashboard_id: int,
|
||||
dataset_ref: Optional[str],
|
||||
unresolved_references: List[str],
|
||||
) -> tuple[Optional[int], List[str]]:
|
||||
dataset_ref: str | None,
|
||||
unresolved_references: list[str],
|
||||
) -> tuple[int | None, list[str]]:
|
||||
dashboard_detail = self.client.get_dashboard_detail(dashboard_id)
|
||||
datasets = dashboard_detail.get("datasets") or []
|
||||
if datasets:
|
||||
@@ -349,4 +352,4 @@ class SupersetContextParsingMixin:
|
||||
unresolved_references.append("dashboard_dataset_binding_missing")
|
||||
return None, unresolved_references
|
||||
# #endregion SupersetContextParsingMixin._recover_dataset_binding_from_dashboard
|
||||
# #endregion SupersetContextParsingMixin
|
||||
# #endregion SupersetContextParsingMixin
|
||||
|
||||
@@ -6,7 +6,8 @@
|
||||
# #region _pii_imports [TYPE Block]
|
||||
import re
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, List, Tuple
|
||||
from typing import Any
|
||||
|
||||
# #endregion _pii_imports
|
||||
|
||||
_EMAIL_PATTERN = re.compile(
|
||||
@@ -23,12 +24,12 @@ _MIXED_IDENTIFIER_PATTERN = re.compile(
|
||||
|
||||
# #region mask_pii_value [C:2] [TYPE Function]
|
||||
# @BRIEF Redact likely sensitive identifiers while preserving structural hints for assistant-facing dataset-review context.
|
||||
def mask_pii_value(value: Any) -> Tuple[Any, bool]:
|
||||
def mask_pii_value(value: Any) -> tuple[Any, bool]:
|
||||
if isinstance(value, str):
|
||||
return _mask_sensitive_text(value)
|
||||
if isinstance(value, list):
|
||||
masked_any = False
|
||||
masked_list: List[Any] = []
|
||||
masked_list: list[Any] = []
|
||||
for item in value:
|
||||
masked_item, item_masked = mask_pii_value(item)
|
||||
masked_list.append(masked_item)
|
||||
@@ -36,7 +37,7 @@ def mask_pii_value(value: Any) -> Tuple[Any, bool]:
|
||||
return masked_list, masked_any
|
||||
if isinstance(value, dict):
|
||||
masked_any = False
|
||||
masked_dict: Dict[str, Any] = {}
|
||||
masked_dict: dict[str, Any] = {}
|
||||
for key, item in value.items():
|
||||
masked_item, item_masked = mask_pii_value(item)
|
||||
masked_dict[key] = masked_item
|
||||
@@ -51,8 +52,8 @@ def mask_pii_value(value: Any) -> Tuple[Any, bool]:
|
||||
# #region sanitize_imported_filter_for_assistant [C:2] [TYPE Function]
|
||||
# @BRIEF Build assistant-safe imported-filter payload with raw-value redaction metadata when sensitive tokens are detected.
|
||||
def sanitize_imported_filter_for_assistant(
|
||||
filter_payload: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
filter_payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
sanitized = deepcopy(filter_payload)
|
||||
masked_raw_value, was_masked = mask_pii_value(filter_payload.get("raw_value"))
|
||||
sanitized["raw_value"] = masked_raw_value
|
||||
@@ -67,7 +68,7 @@ def sanitize_imported_filter_for_assistant(
|
||||
|
||||
# #region _mask_sensitive_text [C:2] [TYPE Function]
|
||||
# @BRIEF Redact likely PII substrings from one string while preserving non-sensitive delimiters and token shape.
|
||||
def _mask_sensitive_text(value: str) -> Tuple[str, bool]:
|
||||
def _mask_sensitive_text(value: str) -> tuple[str, bool]:
|
||||
masked = value
|
||||
masked_any = False
|
||||
|
||||
|
||||
@@ -6,11 +6,14 @@
|
||||
# #endregion SupersetContextRecoveryMixin
|
||||
# #region _recovery_imports [TYPE Block]
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, List, Set, cast
|
||||
from typing import Any, cast
|
||||
|
||||
from ...logger import belief_scope, logger
|
||||
from ._base import SupersetParsedContext
|
||||
|
||||
logger = cast(Any, logger)
|
||||
# #endregion _recovery_imports
|
||||
# #region SupersetContextRecoveryMixin [C:4] [TYPE Class]
|
||||
@@ -25,13 +28,13 @@ class SupersetContextRecoveryMixin:
|
||||
# @DATA_CONTRACT: Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]]
|
||||
def recover_imported_filters(
|
||||
self, parsed_context: SupersetParsedContext
|
||||
) -> List[Dict[str, Any]]:
|
||||
) -> list[dict[str, Any]]:
|
||||
with belief_scope("SupersetContextExtractor.recover_imported_filters"):
|
||||
recovered_filters: List[Dict[str, Any]] = []
|
||||
seen_filter_keys: Set[str] = set()
|
||||
metadata_filters: List[Dict[str, Any]] = []
|
||||
metadata_filters_by_id: Dict[str, Dict[str, Any]] = {}
|
||||
def merge_recovered_filter(candidate: Dict[str, Any]) -> None:
|
||||
recovered_filters: list[dict[str, Any]] = []
|
||||
seen_filter_keys: set[str] = set()
|
||||
metadata_filters: list[dict[str, Any]] = []
|
||||
metadata_filters_by_id: dict[str, dict[str, Any]] = {}
|
||||
def merge_recovered_filter(candidate: dict[str, Any]) -> None:
|
||||
filter_key = candidate["filter_name"].strip().lower()
|
||||
existing_index = next(
|
||||
(
|
||||
@@ -242,10 +245,10 @@ class SupersetContextRecoveryMixin:
|
||||
# @PURPOSE: Normalize one imported-filter payload with explicit provenance and confirmation state.
|
||||
def _normalize_imported_filter_payload(
|
||||
self,
|
||||
payload: Dict[str, Any],
|
||||
payload: dict[str, Any],
|
||||
default_source: str,
|
||||
default_note: str,
|
||||
) -> Dict[str, Any]:
|
||||
) -> dict[str, Any]:
|
||||
raw_value = payload.get("raw_value")
|
||||
if "raw_value" not in payload and "value" in payload:
|
||||
raw_value = payload.get("value")
|
||||
@@ -279,4 +282,4 @@ class SupersetContextRecoveryMixin:
|
||||
"notes": str(payload.get("notes") or default_note),
|
||||
}
|
||||
# #endregion SupersetContextRecoveryMixin._normalize_imported_filter_payload
|
||||
# #endregion SupersetContextRecoveryMixin
|
||||
# #endregion SupersetContextRecoveryMixin
|
||||
|
||||
@@ -6,9 +6,12 @@
|
||||
# #endregion SupersetContextTemplatesMixin
|
||||
# #region _templates_imports [TYPE Block]
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional, Set, cast
|
||||
from typing import Any, cast
|
||||
|
||||
from ...logger import belief_scope, logger
|
||||
|
||||
logger = cast(Any, logger)
|
||||
# #endregion _templates_imports
|
||||
# #region SupersetContextTemplatesMixin [C:3] [TYPE Class]
|
||||
@@ -21,11 +24,11 @@ class SupersetContextTemplatesMixin:
|
||||
# @SIDE_EFFECT: none.
|
||||
# @DATA_CONTRACT: Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]]
|
||||
def discover_template_variables(
|
||||
self, dataset_payload: Dict[str, Any]
|
||||
) -> List[Dict[str, Any]]:
|
||||
self, dataset_payload: dict[str, Any]
|
||||
) -> list[dict[str, Any]]:
|
||||
with belief_scope("SupersetContextExtractor.discover_template_variables"):
|
||||
discovered: List[Dict[str, Any]] = []
|
||||
seen_variable_names: Set[str] = set()
|
||||
discovered: list[dict[str, Any]] = []
|
||||
seen_variable_names: set[str] = set()
|
||||
for expression_source in self._collect_query_bearing_expressions(
|
||||
dataset_payload
|
||||
):
|
||||
@@ -111,9 +114,9 @@ class SupersetContextTemplatesMixin:
|
||||
# @PURPOSE: Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery.
|
||||
# @RELATION: DEPENDS_ON -> [SupersetContextTemplatesMixin.discover_template_variables]
|
||||
def _collect_query_bearing_expressions(
|
||||
self, dataset_payload: Dict[str, Any]
|
||||
) -> List[str]:
|
||||
expressions: List[str] = []
|
||||
self, dataset_payload: dict[str, Any]
|
||||
) -> list[str]:
|
||||
expressions: list[str] = []
|
||||
def append_expression(candidate: Any) -> None:
|
||||
if not isinstance(candidate, str):
|
||||
return
|
||||
@@ -147,8 +150,8 @@ class SupersetContextTemplatesMixin:
|
||||
# @PURPOSE: Append one deduplicated template-variable descriptor.
|
||||
def _append_template_variable(
|
||||
self,
|
||||
discovered: List[Dict[str, Any]],
|
||||
seen_variable_names: Set[str],
|
||||
discovered: list[dict[str, Any]],
|
||||
seen_variable_names: set[str],
|
||||
variable_name: str,
|
||||
expression_source: str,
|
||||
variable_kind: str,
|
||||
@@ -175,7 +178,7 @@ class SupersetContextTemplatesMixin:
|
||||
# #endregion SupersetContextTemplatesMixin._append_template_variable
|
||||
# #region SupersetContextTemplatesMixin._extract_primary_jinja_identifier [TYPE Function] [C:2]
|
||||
# @PURPOSE: Extract a deterministic primary identifier from a Jinja expression without executing it.
|
||||
def _extract_primary_jinja_identifier(self, expression: str) -> Optional[str]:
|
||||
def _extract_primary_jinja_identifier(self, expression: str) -> str | None:
|
||||
matched = re.match(r"([A-Za-z_][A-Za-z0-9_]*)", expression.strip())
|
||||
if matched is None:
|
||||
return None
|
||||
@@ -195,7 +198,7 @@ class SupersetContextTemplatesMixin:
|
||||
# #endregion SupersetContextTemplatesMixin._extract_primary_jinja_identifier
|
||||
# #region SupersetContextTemplatesMixin._normalize_default_literal [TYPE Function] [C:2]
|
||||
# @PURPOSE: Normalize literal default fragments from template helper calls into JSON-safe values.
|
||||
def _normalize_default_literal(self, literal: Optional[str]) -> Any:
|
||||
def _normalize_default_literal(self, literal: str | None) -> Any:
|
||||
normalized_literal = str(literal or "").strip()
|
||||
if not normalized_literal:
|
||||
return None
|
||||
@@ -218,4 +221,4 @@ class SupersetContextTemplatesMixin:
|
||||
except ValueError:
|
||||
return normalized_literal
|
||||
# #endregion SupersetContextTemplatesMixin._normalize_default_literal
|
||||
# #endregion SupersetContextTemplatesMixin
|
||||
# #endregion SupersetContextTemplatesMixin
|
||||
|
||||
Reference in New Issue
Block a user