chore(lint): apply ruff --fix (4443 auto-fixes)

Auto-fixed categories:
- F401: unused imports removed
- I001: import blocks sorted
- W293: trailing whitespace stripped
- UP035: deprecated typing imports replaced
- SIM: simplify suggestions applied
- ARG: unused args prefixed with underscore
- T201: print statements removed
- F841: unused variables removed
- RUF059: unpacked variables prefixed

Remaining ~1500 unfixable errors (C901, B904, N806, E402) require manual work.

Backend smoke tests: 13/13 passed.
This commit is contained in:
2026-05-14 11:20:17 +03:00
parent a9a5eff518
commit c6189876b3
337 changed files with 4677 additions and 4515 deletions

View File

@@ -8,10 +8,13 @@
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions]
# @RELATION DEPENDS_ON -> [SupersetAuthCache]
# @INVARIANT: Async client reuses cached auth tokens per environment credentials and invalidates on 401.
from typing import Optional, Dict, Any, Union
import asyncio
from typing import Any
import httpx
from ..logger import logger as app_logger, belief_scope
from ..logger import belief_scope
from ..logger import logger as app_logger
from .network import (
AuthenticationError,
DashboardNotFoundError,
@@ -20,6 +23,8 @@ from .network import (
SupersetAPIError,
SupersetAuthCache,
)
# #region AsyncAPIClient [C:3] [TYPE Class]
# @BRIEF Async Superset API client backed by httpx.AsyncClient with shared auth cache.
# @RELATION DEPENDS_ON -> [SupersetAuthCache]
@@ -27,7 +32,7 @@ from .network import (
# @RELATION CALLS -> [SupersetAuthCache.set]
class AsyncAPIClient:
DEFAULT_TIMEOUT = 30
_auth_locks: Dict[tuple[str, str, bool], asyncio.Lock] = {}
_auth_locks: dict[tuple[str, str, bool], asyncio.Lock] = {}
# #region AsyncAPIClient.__init__ [TYPE Function] [C:3]
# @PURPOSE: Initialize async API client for one environment.
# @PRE: config contains base_url and auth payload.
@@ -35,7 +40,7 @@ class AsyncAPIClient:
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> self._auth_cache_key[str]
# @RELATION: [CALLS] ->[AsyncAPIClient._normalize_base_url]
# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]
def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
def __init__(self, config: dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
self.base_url: str = self._normalize_base_url(config.get("base_url", ""))
self.api_base_url: str = f"{self.base_url}/api/v1"
self.auth = config.get("auth")
@@ -45,7 +50,7 @@ class AsyncAPIClient:
timeout=httpx.Timeout(timeout),
follow_redirects=True,
)
self._tokens: Dict[str, str] = {}
self._tokens: dict[str, str] = {}
self._authenticated = False
self._auth_cache_key = SupersetAuthCache.build_key(
self.base_url,
@@ -95,7 +100,7 @@ class AsyncAPIClient:
# @RELATION: [CALLS] ->[SupersetAuthCache.get]
# @RELATION: [CALLS] ->[SupersetAuthCache.set]
# @RELATION: [CALLS] ->[AsyncAPIClient._get_auth_lock]
async def authenticate(self) -> Dict[str, str]:
async def authenticate(self) -> dict[str, str]:
cached_tokens = SupersetAuthCache.get(self._auth_cache_key)
if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"):
self._tokens = cached_tokens
@@ -148,7 +153,7 @@ class AsyncAPIClient:
# @PURPOSE: Return authenticated Superset headers for async requests.
# @POST: Headers include Authorization and CSRF tokens.
# @RELATION: [CALLS] ->[AsyncAPIClient.authenticate]
async def get_headers(self) -> Dict[str, str]:
async def get_headers(self) -> dict[str, str]:
if not self._authenticated:
await self.authenticate()
return {
@@ -169,10 +174,10 @@ class AsyncAPIClient:
self,
method: str,
endpoint: str,
headers: Optional[Dict[str, str]] = None,
headers: dict[str, str] | None = None,
raw_response: bool = False,
**kwargs,
) -> Union[httpx.Response, Dict[str, Any]]:
) -> httpx.Response | dict[str, Any]:
full_url = self._build_api_url(endpoint)
request_headers = await self.get_headers()
if headers:
@@ -262,4 +267,4 @@ class AsyncAPIClient:
await self._client.aclose()
# #endregion AsyncAPIClient.aclose
# #endregion AsyncAPIClient
# #endregion AsyncNetworkModule
# #endregion AsyncNetworkModule

View File

@@ -6,10 +6,15 @@
# @RELATION DEPENDS_ON -> pandas
# @RELATION DEPENDS_ON -> psycopg2
# @PUBLIC_API: DatasetMapper
from typing import Any
import pandas as pd # type: ignore
import psycopg2 # type: ignore
from typing import Dict, Optional, Any
from ..logger import logger as app_logger, belief_scope
from ..logger import belief_scope
from ..logger import logger as app_logger
# #region DatasetMapper [TYPE Class]
# @BRIEF Класс для меппинга и обновления verbose_map в датасетах Superset.
class DatasetMapper:
@@ -29,7 +34,7 @@ class DatasetMapper:
# @PARAM: table_name (str) - Имя таблицы.
# @PARAM: table_schema (str) - Схема таблицы.
# @RETURN: Dict[str, str] - Словарь с комментариями к колонкам.
def get_postgres_comments(self, db_config: Dict, table_name: str, table_schema: str) -> Dict[str, str]:
def get_postgres_comments(self, db_config: dict, table_name: str, table_schema: str) -> dict[str, str]:
with belief_scope("Fetch comments from PostgreSQL"):
app_logger.info("[get_postgres_comments][Enter] Fetching comments from PostgreSQL for %s.%s.", table_schema, table_name)
query = f"""
@@ -90,7 +95,7 @@ class DatasetMapper:
# @THROW: Exception - При ошибках чтения файла или парсинга.
# @PARAM: file_path (str) - Путь к XLSX файлу.
# @RETURN: Dict[str, str] - Словарь с меппингами.
def load_excel_mappings(self, file_path: str) -> Dict[str, str]:
def load_excel_mappings(self, file_path: str) -> dict[str, str]:
with belief_scope("Load mappings from Excel"):
app_logger.info("[load_excel_mappings][Enter] Loading mappings from %s.", file_path)
try:
@@ -118,11 +123,11 @@ class DatasetMapper:
# @PARAM: excel_path (Optional[str]) - Путь к XLSX файлу.
# @PARAM: table_name (Optional[str]) - Имя таблицы в PostgreSQL.
# @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL.
def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: Optional[Dict] = None, excel_path: Optional[str] = None, table_name: Optional[str] = None, table_schema: Optional[str] = None):
def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: dict | None = None, excel_path: str | None = None, table_name: str | None = None, table_schema: str | None = None):
with belief_scope(f"Run dataset mapping for ID {dataset_id}"):
app_logger.info("[run_mapping][Enter] Starting dataset mapping for ID %d from source '%s'.", dataset_id, source)
mappings: Dict[str, str] = {}
mappings: dict[str, str] = {}
try:
if source in ['postgres', 'both']:
assert postgres_config and table_name and table_schema, "Postgres config is required."
@@ -135,13 +140,13 @@ class DatasetMapper:
return
dataset_response = superset_client.get_dataset(dataset_id)
dataset_data = dataset_response['result']
original_columns = dataset_data.get('columns', [])
updated_columns = []
changes_made = False
for column in original_columns:
col_name = column.get('column_name')
new_column = {
"column_name": col_name,
"id": column.get("id"),
@@ -158,14 +163,14 @@ class DatasetMapper:
"uuid": column.get("uuid"),
"verbose_name": column.get("verbose_name"),
}
new_column = {k: v for k, v in new_column.items() if v is not None}
if col_name in mappings:
mapping_value = mappings[col_name]
if isinstance(mapping_value, str) and new_column.get('verbose_name') != mapping_value:
new_column['verbose_name'] = mapping_value
changes_made = True
updated_columns.append(new_column)
updated_metrics = []
for metric in dataset_data.get("metrics", []):
@@ -207,7 +212,7 @@ class DatasetMapper:
"offset": dataset_data.get("offset"),
"template_params": dataset_data.get("template_params"),
}
payload_for_update = {k: v for k, v in payload_for_update.items() if v is not None}
superset_client.update_dataset(dataset_id, payload_for_update)
app_logger.info("[run_mapping][Success] Dataset %d columns' verbose_name updated.", dataset_id)
@@ -218,4 +223,4 @@ class DatasetMapper:
return
# #endregion run_mapping
# #endregion DatasetMapper
# #endregion DatasetMapperModule
# #endregion DatasetMapperModule

View File

@@ -7,16 +7,21 @@
# @PUBLIC_API: create_temp_file, remove_empty_directories, read_dashboard_from_disk, calculate_crc32, RetentionPolicy, archive_exports, save_and_unpack_dashboard, update_yamls, create_dashboard_export, sanitize_filename, get_filename_from_headers, consolidate_archive_folders
import os
import re
import zipfile
from pathlib import Path
from typing import Any, Optional, Tuple, Dict, List, Union, LiteralString, Generator
from contextlib import contextmanager
import tempfile
from datetime import date, datetime
import shutil
import tempfile
import zipfile
import zlib
from collections.abc import Generator
from contextlib import contextmanager
from dataclasses import dataclass
from ..logger import logger as app_logger, belief_scope
from datetime import date, datetime
from pathlib import Path
from typing import Any, LiteralString
from ..logger import belief_scope
from ..logger import logger as app_logger
# #region InvalidZipFormatError [TYPE Class]
# @BRIEF Exception raised when a file is not a valid ZIP archive.
class InvalidZipFormatError(Exception):
@@ -28,7 +33,7 @@ class InvalidZipFormatError(Exception):
# @POST: Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста.
# @YIELDS: Path - Путь к временному ресурсу.
@contextmanager
def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode: str = 'wb', dry_run = False) -> Generator[Path, None, None]:
def create_temp_file(content: bytes | None = None, suffix: str = ".zip", mode: str = 'wb', dry_run = False) -> Generator[Path]:
with belief_scope("Create temporary resource"):
resource_path = None
is_dir = suffix.startswith('.dir')
@@ -84,7 +89,7 @@ def remove_empty_directories(root_dir: str) -> int:
# @BRIEF Читает бинарное содержимое файла с диска.
# @PRE: file_path должен указывать на существующий файл.
# @POST: Возвращает байты содержимого и имя файла.
def read_dashboard_from_disk(file_path: str) -> Tuple[bytes, str]:
def read_dashboard_from_disk(file_path: str) -> tuple[bytes, str]:
with belief_scope(f"Read dashboard from {file_path}"):
path = Path(file_path)
assert path.is_file(), f"Файл дашборда не найден: {file_path}"
@@ -99,10 +104,9 @@ def read_dashboard_from_disk(file_path: str) -> Tuple[bytes, str]:
# @PRE: file_path должен быть объектом Path к существующему файлу.
# @POST: Возвращает 8-значную hex-строку CRC32.
def calculate_crc32(file_path: Path) -> str:
with belief_scope(f"Calculate CRC32 for {file_path}"):
with open(file_path, 'rb') as f:
crc32_value = zlib.crc32(f.read())
return f"{crc32_value:08x}"
with belief_scope(f"Calculate CRC32 for {file_path}"), open(file_path, 'rb') as f:
crc32_value = zlib.crc32(f.read())
return f"{crc32_value:08x}"
# #endregion calculate_crc32
# #region RetentionPolicy [TYPE DataClass]
# @BRIEF Определяет политику хранения для архивов (ежедневные, еженедельные, ежемесячные).
@@ -125,7 +129,7 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
app_logger.warning("[archive_exports][Skip] Archive directory not found: %s", output_dir)
return
app_logger.info("[archive_exports][Enter] Managing archive in %s", output_dir)
# 1. Collect all zip files
zip_files = list(output_path.glob("*.zip"))
if not zip_files:
@@ -136,10 +140,10 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
app_logger.info("[archive_exports][State] Starting deduplication...")
checksums = {}
files_to_remove = []
# Sort by modification time (newest first) to keep the latest version
zip_files.sort(key=lambda f: f.stat().st_mtime, reverse=True)
for file_path in zip_files:
try:
crc = calculate_crc32(file_path)
@@ -150,7 +154,7 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
checksums[crc] = file_path
except Exception as e:
app_logger.error("[archive_exports][Failure] Failed to calculate CRC32 for %s: %s", file_path, e)
for f in files_to_remove:
try:
f.unlink()
@@ -171,14 +175,14 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
file_date = datetime.strptime(date_str, "%Y%m%d").date()
except ValueError:
pass
if not file_date:
# Fallback to modification time
file_date = datetime.fromtimestamp(file_path.stat().st_mtime).date()
files_with_dates.append((file_path, file_date))
files_to_keep = apply_retention_policy(files_with_dates, policy)
for file_path, _ in files_with_dates:
if file_path not in files_to_keep:
try:
@@ -191,30 +195,30 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
# @BRIEF (Helper) Применяет политику хранения к списку файлов, возвращая те, что нужно сохранить.
# @PRE: files_with_dates is a list of (Path, date) tuples.
# @POST: Returns a set of files to keep.
def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: RetentionPolicy) -> set:
def apply_retention_policy(files_with_dates: list[tuple[Path, date]], policy: RetentionPolicy) -> set:
with belief_scope("Apply retention policy"):
# Сортируем по дате (от новой к старой)
sorted_files = sorted(files_with_dates, key=lambda x: x[1], reverse=True)
# Словарь для хранения файлов по категориям
daily_files = []
weekly_files = []
monthly_files = []
today = date.today()
for file_path, file_date in sorted_files:
# Ежедневные
if (today - file_date).days < policy.daily:
daily_files.append(file_path)
# Еженедельные
elif (today - file_date).days < policy.weekly * 7:
weekly_files.append(file_path)
# Ежемесячные
elif (today - file_date).days < policy.monthly * 30:
monthly_files.append(file_path)
# Возвращаем множество файлов, которые нужно сохранить
files_to_keep = set()
files_to_keep.update(daily_files)
files_to_keep.update(weekly_files[:policy.weekly])
files_to_keep.update(monthly_files[:policy.monthly])
sorted_files = sorted(files_with_dates, key=lambda x: x[1], reverse=True)
# Словарь для хранения файлов по категориям
daily_files = []
weekly_files = []
monthly_files = []
today = date.today()
for file_path, file_date in sorted_files:
# Ежедневные
if (today - file_date).days < policy.daily:
daily_files.append(file_path)
# Еженедельные
elif (today - file_date).days < policy.weekly * 7:
weekly_files.append(file_path)
# Ежемесячные
elif (today - file_date).days < policy.monthly * 30:
monthly_files.append(file_path)
# Возвращаем множество файлов, которые нужно сохранить
files_to_keep = set()
files_to_keep.update(daily_files)
files_to_keep.update(weekly_files[:policy.weekly])
files_to_keep.update(monthly_files[:policy.monthly])
app_logger.debug("[apply_retention_policy][State] Keeping %d files according to retention policy", len(files_to_keep))
return files_to_keep
# #endregion apply_retention_policy
@@ -222,7 +226,7 @@ def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: Re
# @BRIEF Сохраняет бинарное содержимое ZIP-архива на диск и опционально распаковывает его.
# @PRE: zip_content должен быть байтами валидного ZIP-архива.
# @POST: ZIP-файл сохранен, и если unpack=True, он распакован в output_dir.
def save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path], unpack: bool = False, original_filename: Optional[str] = None) -> Tuple[Path, Optional[Path]]:
def save_and_unpack_dashboard(zip_content: bytes, output_dir: str | Path, unpack: bool = False, original_filename: str | None = None) -> tuple[Path, Path | None]:
with belief_scope("Save and unpack dashboard"):
app_logger.info("[save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: %s", unpack)
try:
@@ -247,14 +251,14 @@ def save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path],
# @PRE: path должен быть существующей директорией.
# @POST: Все YAML файлы в директории обновлены согласно переданным параметрам.
# @RELATION CALLS -> _update_yaml_file
def update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str = "dashboards", regexp_pattern: Optional[LiteralString] = None, replace_string: Optional[LiteralString] = None) -> None:
def update_yamls(db_configs: list[dict[str, Any]] | None = None, path: str = "dashboards", regexp_pattern: LiteralString | None = None, replace_string: LiteralString | None = None) -> None:
with belief_scope("Update YAML configurations"):
app_logger.info("[update_yamls][Enter] Starting YAML configuration update.")
dir_path = Path(path)
assert dir_path.is_dir(), f"Путь {path} не существует или не является директорией"
configs: List[Dict[str, Any]] = db_configs or []
configs: list[dict[str, Any]] = db_configs or []
for file_path in dir_path.rglob("*.yaml"):
_update_yaml_file(file_path, configs, regexp_pattern, replace_string)
# #endregion update_yamls
@@ -262,11 +266,11 @@ def update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str =
# @BRIEF (Helper) Обновляет один YAML файл.
# @PRE: file_path должен быть объектом Path к существующему YAML файлу.
# @POST: Файл обновлен согласно переданным конфигурациям или регулярному выражению.
def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_pattern: Optional[str], replace_string: Optional[str]) -> None:
def _update_yaml_file(file_path: Path, db_configs: list[dict[str, Any]], regexp_pattern: str | None, replace_string: str | None) -> None:
with belief_scope(f"Update YAML file: {file_path}"):
# Читаем содержимое файла
try:
with open(file_path, 'r', encoding='utf-8') as f:
with open(file_path, encoding='utf-8') as f:
content = f.read()
except Exception as e:
app_logger.error("[_update_yaml_file][Failure] Failed to read %s: %s", file_path, e)
@@ -300,7 +304,7 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_
val_pattern = re.escape(old_val)
# Группы: 1=ключ+разделитель, 2=открывающая кавычка (опц), 3=значение, 4=закрывающая кавычка (опц)
pattern = rf'({key_pattern}\s*:\s*)(["\']?)({val_pattern})(["\']?)'
# #region replacer [TYPE Function]
# @PURPOSE: Функция замены, сохраняющая кавычки если они были.
# @PRE: match должен быть объектом совпадения регулярного выражения.
@@ -323,7 +327,7 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_
# @BRIEF Создает ZIP-архив из указанных исходных путей.
# @PRE: source_paths должен содержать существующие пути.
# @POST: ZIP-архив создан по пути zip_path.
def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union[str, Path]], exclude_extensions: Optional[List[str]] = None) -> bool:
def create_dashboard_export(zip_path: str | Path, source_paths: list[str | Path], exclude_extensions: list[str] | None = None) -> bool:
with belief_scope(f"Create dashboard export: {zip_path}"):
app_logger.info("[create_dashboard_export][Enter] Packing dashboard: %s -> %s", source_paths, zip_path)
try:
@@ -338,7 +342,7 @@ def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union
zipf.write(item, arcname)
app_logger.reflect(f"Archive created: {zip_path}", extra={"src": "create_dashboard_export"})
return True
except (IOError, zipfile.BadZipFile, AssertionError) as e:
except (OSError, zipfile.BadZipFile, AssertionError) as e:
app_logger.error("[create_dashboard_export][Failure] Error: %s", e, exc_info=True)
return False
# #endregion create_dashboard_export
@@ -354,7 +358,7 @@ def sanitize_filename(filename: str) -> str:
# @BRIEF Извлекает имя файла из HTTP заголовка 'Content-Disposition'.
# @PRE: headers должен быть словарем заголовков.
# @POST: Возвращает имя файла или None, если заголовок отсутствует.
def get_filename_from_headers(headers: dict) -> Optional[str]:
def get_filename_from_headers(headers: dict) -> str | None:
with belief_scope("Get filename from headers"):
content_disposition = headers.get("Content-Disposition", "")
if match := re.search(r'filename="?([^"]+)"?', content_disposition):
@@ -367,51 +371,51 @@ def get_filename_from_headers(headers: dict) -> Optional[str]:
# @POST: Директории с одинаковым префиксом объединены в одну.
def consolidate_archive_folders(root_directory: Path) -> None:
with belief_scope(f"Consolidate archives in {root_directory}"):
assert isinstance(root_directory, Path), "root_directory must be a Path object."
assert root_directory.is_dir(), "root_directory must be an existing directory."
app_logger.info("[consolidate_archive_folders][Enter] Consolidating archives in %s", root_directory)
# Собираем все директории с архивами
archive_dirs = []
for item in root_directory.iterdir():
if item.is_dir():
# Проверяем, есть ли в директории ZIP-архивы
if any(item.glob("*.zip")):
archive_dirs.append(item)
# Группируем по слагу (части имени до первого '_')
slug_groups = {}
for dir_path in archive_dirs:
dir_name = dir_path.name
slug = dir_name.split('_')[0] if '_' in dir_name else dir_name
if slug not in slug_groups:
slug_groups[slug] = []
slug_groups[slug].append(dir_path)
# Для каждой группы консолидируем
for slug, dirs in slug_groups.items():
if len(dirs) <= 1:
continue
# Создаем целевую директорию
target_dir = root_directory / slug
target_dir.mkdir(exist_ok=True)
app_logger.info("[consolidate_archive_folders][State] Consolidating %d directories under %s", len(dirs), target_dir)
# Перемещаем содержимое
for source_dir in dirs:
if source_dir == target_dir:
continue
for item in source_dir.iterdir():
dest_item = target_dir / item.name
try:
if item.is_dir():
shutil.move(str(item), str(dest_item))
else:
shutil.move(str(item), str(dest_item))
except Exception as e:
app_logger.error("[consolidate_archive_folders][Failure] Failed to move %s to %s: %s", item, dest_item, e)
# Удаляем исходную директорию
try:
source_dir.rmdir()
app_logger.info("[consolidate_archive_folders][State] Removed source directory: %s", source_dir)
except Exception as e:
app_logger.error("[consolidate_archive_folders][Failure] Failed to remove source directory %s: %s", source_dir, e)
assert isinstance(root_directory, Path), "root_directory must be a Path object."
assert root_directory.is_dir(), "root_directory must be an existing directory."
app_logger.info("[consolidate_archive_folders][Enter] Consolidating archives in %s", root_directory)
# Собираем все директории с архивами
archive_dirs = []
for item in root_directory.iterdir():
if item.is_dir():
# Проверяем, есть ли в директории ZIP-архивы
if any(item.glob("*.zip")):
archive_dirs.append(item)
# Группируем по слагу (части имени до первого '_')
slug_groups = {}
for dir_path in archive_dirs:
dir_name = dir_path.name
slug = dir_name.split('_')[0] if '_' in dir_name else dir_name
if slug not in slug_groups:
slug_groups[slug] = []
slug_groups[slug].append(dir_path)
# Для каждой группы консолидируем
for slug, dirs in slug_groups.items():
if len(dirs) <= 1:
continue
# Создаем целевую директорию
target_dir = root_directory / slug
target_dir.mkdir(exist_ok=True)
app_logger.info("[consolidate_archive_folders][State] Consolidating %d directories under %s", len(dirs), target_dir)
# Перемещаем содержимое
for source_dir in dirs:
if source_dir == target_dir:
continue
for item in source_dir.iterdir():
dest_item = target_dir / item.name
try:
if item.is_dir():
shutil.move(str(item), str(dest_item))
else:
shutil.move(str(item), str(dest_item))
except Exception as e:
app_logger.error("[consolidate_archive_folders][Failure] Failed to move %s to %s: %s", item, dest_item, e)
# Удаляем исходную директорию
try:
source_dir.rmdir()
app_logger.info("[consolidate_archive_folders][State] Removed source directory: %s", source_dir)
except Exception as e:
app_logger.error("[consolidate_archive_folders][Failure] Failed to remove source directory %s: %s", source_dir, e)
# #endregion consolidate_archive_folders
# #endregion FileIO
# #endregion FileIO

View File

@@ -6,14 +6,15 @@
#
# @INVARIANT: Confidence scores are returned as floats between 0.0 and 1.0.
from rapidfuzz import fuzz, process
from typing import List, Dict
# #region suggest_mappings [TYPE Function]
# @BRIEF Suggests mappings between source and target databases using fuzzy matching.
# @PRE: source_databases and target_databases are lists of dictionaries with 'uuid' and 'database_name'.
# @POST: Returns a list of suggested mappings with confidence scores.
def suggest_mappings(source_databases: List[Dict], target_databases: List[Dict], threshold: int = 60) -> List[Dict]:
def suggest_mappings(source_databases: list[dict], target_databases: list[dict], threshold: int = 60) -> list[dict]:
"""
Suggest mappings between source and target databases using fuzzy matching.
"""
@@ -22,15 +23,15 @@ def suggest_mappings(source_databases: List[Dict], target_databases: List[Dict],
return suggestions
target_names = [db['database_name'] for db in target_databases]
for s_db in source_databases:
# Use token_sort_ratio as decided in research.md
match = process.extractOne(
s_db['database_name'],
target_names,
s_db['database_name'],
target_names,
scorer=fuzz.token_sort_ratio
)
if match:
name, score, index = match
if score >= threshold:
@@ -41,7 +42,7 @@ def suggest_mappings(source_databases: List[Dict], target_databases: List[Dict],
"target_db_uuid": target_databases[index]['uuid'],
"confidence": score / 100.0
})
return suggestions
# #endregion suggest_mappings

View File

@@ -4,17 +4,22 @@
# @LAYER: Infra
# @RELATION DEPENDS_ON -> [LoggerModule]
# @PUBLIC_API: APIClient
from typing import Optional, Dict, Any, List, Union, cast, Tuple
import json
import io
from pathlib import Path
import json
import threading
import time
from pathlib import Path
from typing import Any, cast
import requests
from requests.adapters import HTTPAdapter
import urllib3
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from ..logger import logger as app_logger, belief_scope
from ..logger import belief_scope
from ..logger import logger as app_logger
# #region SupersetAPIError [C:1] [TYPE Class]
# @BRIEF Base exception for all Superset API related errors.
class SupersetAPIError(Exception):
@@ -59,7 +64,7 @@ class DashboardNotFoundError(SupersetAPIError):
# @PURPOSE: Initializes the not found error with resource ID.
# @PRE: resource_id is provided.
# @POST: DashboardNotFoundError is initialized.
def __init__(self, resource_id: Union[int, str], message: str = "Dashboard not found", **context: Any):
def __init__(self, resource_id: int | str, message: str = "Dashboard not found", **context: Any):
with belief_scope("DashboardNotFoundError.__init__"):
super().__init__(f"Dashboard '{resource_id}' {message}", subtype="not_found", resource_id=resource_id, **context)
# #endregion __init__
@@ -84,16 +89,16 @@ class NetworkError(Exception):
class SupersetAuthCache:
TTL_SECONDS = 300
_lock = threading.Lock()
_entries: Dict[Tuple[str, str, bool], Dict[str, Any]] = {}
_entries: dict[tuple[str, str, bool], dict[str, Any]] = {}
@classmethod
def build_key(cls, base_url: str, auth: Optional[Dict[str, Any]], verify_ssl: bool) -> Tuple[str, str, bool]:
def build_key(cls, base_url: str, auth: dict[str, Any] | None, verify_ssl: bool) -> tuple[str, str, bool]:
username = ""
if isinstance(auth, dict):
username = str(auth.get("username") or "").strip()
return (str(base_url or "").strip(), username, bool(verify_ssl))
@classmethod
# #region SupersetAuthCache.get [TYPE Function]
def get(cls, key: Tuple[str, str, bool]) -> Optional[Dict[str, str]]:
def get(cls, key: tuple[str, str, bool]) -> dict[str, str] | None:
now = time.time()
with cls._lock:
payload = cls._entries.get(key)
@@ -114,7 +119,7 @@ class SupersetAuthCache:
# #endregion SupersetAuthCache.get
@classmethod
# #region SupersetAuthCache.set [TYPE Function]
def set(cls, key: Tuple[str, str, bool], tokens: Dict[str, str], ttl_seconds: Optional[int] = None) -> None:
def set(cls, key: tuple[str, str, bool], tokens: dict[str, str], ttl_seconds: int | None = None) -> None:
normalized_ttl = max(int(ttl_seconds or cls.TTL_SECONDS), 1)
with cls._lock:
cls._entries[key] = {
@@ -126,7 +131,7 @@ class SupersetAuthCache:
}
# #endregion SupersetAuthCache.set
@classmethod
def invalidate(cls, key: Tuple[str, str, bool]) -> None:
def invalidate(cls, key: tuple[str, str, bool]) -> None:
with cls._lock:
cls._entries.pop(key, None)
# #endregion SupersetAuthCache
@@ -143,7 +148,7 @@ class APIClient:
# @PARAM: timeout (int) - Таймаут запросов.
# @PRE: config must contain 'base_url' and 'auth'.
# @POST: APIClient instance is initialized with a session.
def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
def __init__(self, config: dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
with belief_scope("__init__"):
app_logger.reason("Initializing APIClient.", extra={"src": "APIClient.__init__"})
self.base_url: str = self._normalize_base_url(config.get("base_url", ""))
@@ -151,7 +156,7 @@ class APIClient:
self.auth = config.get("auth")
self.request_settings = {"verify_ssl": verify_ssl, "timeout": timeout}
self.session = self._init_session()
self._tokens: Dict[str, str] = {}
self._tokens: dict[str, str] = {}
self._auth_cache_key = SupersetAuthCache.build_key(
self.base_url,
self.auth,
@@ -168,33 +173,34 @@ class APIClient:
def _init_session(self) -> requests.Session:
with belief_scope("_init_session"):
session = requests.Session()
# Create a custom adapter that handles TLS issues
class TLSAdapter(HTTPAdapter):
def init_poolmanager(self, connections, maxsize, block=False):
from urllib3.poolmanager import PoolManager
import ssl
from urllib3.poolmanager import PoolManager
# Create an SSL context that ignores TLSv1 unrecognized name errors
ctx = ssl.create_default_context()
ctx.set_ciphers('HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!SRP:!CAMELLIA')
# Ignore TLSV1_UNRECOGNIZED_NAME errors by disabling hostname verification
# This is safe when verify_ssl is false (we're already not verifying the certificate)
ctx.check_hostname = False
self.poolmanager = PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
ssl_context=ctx
)
retries = Retry(total=3, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504])
adapter = TLSAdapter(max_retries=retries)
session.mount('http://', adapter)
session.mount('https://', adapter)
if not self.request_settings["verify_ssl"]:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
app_logger.warning("[_init_session][State] SSL verification disabled.")
@@ -202,7 +208,7 @@ class APIClient:
session.verify = False
else:
session.verify = True
return session
# #endregion _init_session
# #region _normalize_base_url [TYPE Function]
@@ -239,7 +245,7 @@ class APIClient:
# @THROW: AuthenticationError, NetworkError - при ошибках.
# @RELATION: [CALLS] ->[SupersetAuthCache.get]
# @RELATION: [CALLS] ->[SupersetAuthCache.set]
def authenticate(self) -> Dict[str, str]:
def authenticate(self) -> dict[str, str]:
with belief_scope("authenticate"):
app_logger.info("[authenticate][Enter] Authenticating to %s", self.base_url)
cached_tokens = SupersetAuthCache.get(self._auth_cache_key)
@@ -288,19 +294,19 @@ class APIClient:
masked_auth = {k: ("******" if k == "password" else v) for k, v in self.auth.items()}
app_logger.info(f"[authenticate][Debug] Login URL: {login_url}")
app_logger.info(f"[authenticate][Debug] Auth payload: {masked_auth}")
response = self.session.post(login_url, json=self.auth, timeout=self.request_settings["timeout"])
if response.status_code != 200:
app_logger.error(f"[authenticate][Error] Status: {response.status_code}, Response: {response.text}")
response.raise_for_status()
access_token = response.json()["access_token"]
csrf_url = f"{self.api_base_url}/security/csrf_token/"
csrf_response = self.session.get(csrf_url, headers={"Authorization": f"Bearer {access_token}"}, timeout=self.request_settings["timeout"])
csrf_response.raise_for_status()
self._tokens = {"access_token": access_token, "csrf_token": csrf_response.json()["result"]}
self._authenticated = True
SupersetAuthCache.set(self._auth_cache_key, self._tokens)
@@ -321,7 +327,7 @@ class APIClient:
# @PURPOSE: Возвращает HTTP-заголовки для аутентифицированных запросов.
# @PRE: APIClient is initialized and authenticated or can be authenticated.
# @POST: Returns headers including auth tokens.
def headers(self) -> Dict[str, str]:
def headers(self) -> dict[str, str]:
if not self._authenticated:
self.authenticate()
return {
@@ -341,12 +347,12 @@ class APIClient:
# @POST: Returns response content or raw Response object.
# @RETURN: `requests.Response` если `raw_response=True`, иначе `dict`.
# @THROW: SupersetAPIError, NetworkError и их подклассы.
def request(self, method: str, endpoint: str, headers: Optional[Dict] = None, raw_response: bool = False, **kwargs) -> Union[requests.Response, Dict[str, Any]]:
def request(self, method: str, endpoint: str, headers: dict | None = None, raw_response: bool = False, **kwargs) -> requests.Response | dict[str, Any]:
full_url = self._build_api_url(endpoint)
_headers = self.headers.copy()
if headers:
_headers.update(headers)
try:
response = self.session.request(method, full_url, headers=_headers, **kwargs)
response.raise_for_status()
@@ -429,15 +435,15 @@ class APIClient:
# @POST: File is uploaded and response returned.
# @RETURN: Ответ API в виде словаря.
# @THROW: SupersetAPIError, NetworkError, TypeError.
def upload_file(self, endpoint: str, file_info: Dict[str, Any], extra_data: Optional[Dict] = None, timeout: Optional[int] = None) -> Dict:
def upload_file(self, endpoint: str, file_info: dict[str, Any], extra_data: dict | None = None, timeout: int | None = None) -> dict:
with belief_scope("upload_file"):
full_url = self._build_api_url(endpoint)
_headers = self.headers.copy()
_headers.pop('Content-Type', None)
file_obj, file_name, form_field = file_info.get("file_obj"), file_info.get("file_name"), file_info.get("form_field", "file")
files_payload = {}
if isinstance(file_obj, (str, Path)):
with open(file_obj, 'rb') as f:
@@ -446,7 +452,7 @@ class APIClient:
files_payload = {form_field: (file_name, file_obj.getvalue(), 'application/x-zip-compressed')}
else:
raise TypeError(f"Unsupported file_obj type: {type(file_obj)}")
return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout)
# #endregion upload_file
# #region _perform_upload [TYPE Function]
@@ -459,7 +465,7 @@ class APIClient:
# @PRE: url, files, and headers must be provided.
# @POST: POST request is performed and JSON response returned.
# @RETURN: Dict - Ответ.
def _perform_upload(self, url: str, files: Dict, data: Optional[Dict], headers: Dict, timeout: Optional[int]) -> Dict:
def _perform_upload(self, url: str, files: dict, data: dict | None, headers: dict, timeout: int | None) -> dict:
with belief_scope("_perform_upload"):
try:
response = self.session.post(url, files=files, data=data or {}, headers=headers, timeout=timeout or self.request_settings["timeout"])
@@ -484,9 +490,9 @@ class APIClient:
# @PRE: query_params must be a dictionary.
# @POST: Returns total count of items.
# @RETURN: int - Количество.
def fetch_paginated_count(self, endpoint: str, query_params: Dict, count_field: str = "count") -> int:
def fetch_paginated_count(self, endpoint: str, query_params: dict, count_field: str = "count") -> int:
with belief_scope("fetch_paginated_count"):
response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query_params)}))
response_json = cast(dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query_params)}))
return response_json.get(count_field, 0)
# #endregion fetch_paginated_count
# #region fetch_paginated_data [TYPE Function]
@@ -496,26 +502,26 @@ class APIClient:
# @PRE: pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional.
# @POST: Returns all items across all pages.
# @RETURN: List[Any] - Список данных.
def fetch_paginated_data(self, endpoint: str, pagination_options: Dict[str, Any]) -> List[Any]:
def fetch_paginated_data(self, endpoint: str, pagination_options: dict[str, Any]) -> list[Any]:
with belief_scope("fetch_paginated_data"):
base_query = pagination_options["base_query"]
total_count = pagination_options.get("total_count")
results_field = pagination_options["results_field"]
count_field = pagination_options.get("count_field", "count")
page_size = base_query.get('page_size', 1000)
assert page_size > 0, "'page_size' must be a positive number."
results = []
page = 0
# Fetch first page to get data and total count if not provided
query = {**base_query, 'page': page}
response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)}))
response_json = cast(dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)}))
first_page_results = response_json.get(results_field, [])
results.extend(first_page_results)
if total_count is None:
total_count = response_json.get(count_field, len(first_page_results))
app_logger.debug(f"[fetch_paginated_data][State] Total count resolved from first page: {total_count}")
@@ -523,10 +529,10 @@ class APIClient:
total_pages = (total_count + page_size - 1) // page_size
for page in range(1, total_pages):
query = {**base_query, 'page': page}
response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)}))
response_json = cast(dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)}))
results.extend(response_json.get(results_field, []))
return results
# #endregion fetch_paginated_data
# #endregion APIClient
# #endregion NetworkModule
# #endregion NetworkModule

View File

@@ -8,14 +8,18 @@
# @SIDE_EFFECT: performs upstream Superset preview and SQL Lab calls.
# @INVARIANT: The adapter never fabricates compiled SQL locally; preview truth is delegated to Superset only.
from __future__ import annotations
# #region SupersetCompilationAdapter.imports [TYPE Block]
from dataclasses import dataclass
from datetime import datetime
from typing import Any, Dict, List, Optional
from typing import Any
from src.core.config_models import Environment
from src.core.logger import belief_scope, logger
from src.core.superset_client import SupersetClient
from src.models.dataset_review import CompiledPreview, PreviewStatus
# #endregion SupersetCompilationAdapter.imports
# #region PreviewCompilationPayload [C:2] [TYPE Class]
# @BRIEF Typed preview payload for Superset-side compilation.
@@ -24,8 +28,8 @@ class PreviewCompilationPayload:
session_id: str
dataset_id: int
preview_fingerprint: str
template_params: Dict[str, Any]
effective_filters: List[Dict[str, Any]]
template_params: dict[str, Any]
effective_filters: list[dict[str, Any]]
# #endregion PreviewCompilationPayload
# #region SqlLabLaunchPayload [C:2] [TYPE Class]
# @BRIEF Typed SQL Lab payload for audited launch handoff.
@@ -35,7 +39,7 @@ class SqlLabLaunchPayload:
dataset_id: int
preview_id: str
compiled_sql: str
template_params: Dict[str, Any]
template_params: dict[str, Any]
# #endregion SqlLabLaunchPayload
# #region SupersetCompilationAdapter [C:4] [TYPE Class]
# @BRIEF Delegate preview compilation and SQL Lab launch to Superset without local SQL fabrication.
@@ -47,7 +51,7 @@ class SupersetCompilationAdapter:
# #region SupersetCompilationAdapter.__init__ [TYPE Function] [C:2]
# @PURPOSE: Bind adapter to one Superset environment and client instance.
def __init__(
self, environment: Environment, client: Optional[SupersetClient] = None
self, environment: Environment, client: SupersetClient | None = None
) -> None:
self.environment = environment
self.client = client or SupersetClient(environment)
@@ -221,7 +225,7 @@ class SupersetCompilationAdapter:
# @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[Dict[str,Any]]
def _request_superset_preview(
self, payload: PreviewCompilationPayload
) -> Dict[str, Any]:
) -> dict[str, Any]:
direct_compile_preview = getattr(self.client, "compile_preview", None)
if self._supports_client_method("compile_preview") and callable(
direct_compile_preview
@@ -313,7 +317,7 @@ class SupersetCompilationAdapter:
"Superset preview compilation response could not be normalized"
)
return normalized
errors: List[str] = []
errors: list[str] = []
for endpoint in (
f"/dataset/{payload.dataset_id}/preview",
f"/dataset/{payload.dataset_id}/sql",
@@ -357,7 +361,7 @@ class SupersetCompilationAdapter:
# @POST: returns the first successful SQL Lab execution response from Superset.
# @SIDE_EFFECT: issues Superset dataset lookup and SQL Lab execution requests.
# @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[Dict[str,Any]]
def _request_sql_lab_session(self, payload: SqlLabLaunchPayload) -> Dict[str, Any]:
def _request_sql_lab_session(self, payload: SqlLabLaunchPayload) -> dict[str, Any]:
dataset_raw = self.client.get_dataset(payload.dataset_id)
dataset_record = (
dataset_raw.get("result", dataset_raw)
@@ -384,7 +388,7 @@ class SupersetCompilationAdapter:
{"kind": "network", "target": "/sqllab/execute/", "http_method": "POST"},
{"kind": "network", "target": "/sql_lab/execute/", "http_method": "POST"},
]
errors: List[str] = []
errors: list[str] = []
for candidate in candidate_calls:
try:
response = self.client.network.request(
@@ -408,7 +412,7 @@ class SupersetCompilationAdapter:
# #region SupersetCompilationAdapter._normalize_preview_response [TYPE Function] [C:3]
# @PURPOSE: Normalize candidate Superset preview responses into one compiled-sql structure.
# @RELATION: [DEPENDS_ON] ->[CompiledPreview]
def _normalize_preview_response(self, response: Any) -> Optional[Dict[str, Any]]:
def _normalize_preview_response(self, response: Any) -> dict[str, Any] | None:
if not isinstance(response, dict):
return None
compiled_sql_candidates = [
@@ -436,9 +440,9 @@ class SupersetCompilationAdapter:
# #endregion SupersetCompilationAdapter._normalize_preview_response
# #region SupersetCompilationAdapter._dump_json [TYPE Function] [C:1]
# @PURPOSE: Serialize Superset request payload deterministically for network transport.
def _dump_json(self, payload: Dict[str, Any]) -> str:
def _dump_json(self, payload: dict[str, Any]) -> str:
import json
return json.dumps(payload, sort_keys=True, default=str)
# #endregion SupersetCompilationAdapter._dump_json
# #endregion SupersetCompilationAdapter
# #endregion SupersetCompilationAdapter
# #endregion SupersetCompilationAdapter

View File

@@ -17,11 +17,11 @@
# #endregion SupersetContextExtractorPackage
from ._base import SupersetContextExtractorBase, SupersetParsedContext
from ._filters import SupersetContextFiltersExtractMixin
from ._parsing import SupersetContextParsingMixin
from ._pii import _mask_sensitive_text, mask_pii_value, sanitize_imported_filter_for_assistant
from ._recovery import SupersetContextRecoveryMixin
from ._templates import SupersetContextTemplatesMixin
from ._filters import SupersetContextFiltersExtractMixin
from ._pii import mask_pii_value, sanitize_imported_filter_for_assistant, _mask_sensitive_text
# #region SupersetContextExtractor [C:4] [TYPE Class]
@@ -57,7 +57,7 @@ class SupersetContextExtractor(
__all__ = [
"SupersetContextExtractor",
"SupersetParsedContext",
"_mask_sensitive_text",
"mask_pii_value",
"sanitize_imported_filter_for_assistant",
"_mask_sensitive_text",
]

View File

@@ -11,15 +11,16 @@
# #endregion SupersetContextExtractorBase
# #region _base_imports [TYPE Block]
from __future__ import annotations
import json
import re
from copy import deepcopy
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Set, cast
from urllib.parse import parse_qs, unquote, urlparse
from typing import Any, cast
from urllib.parse import unquote
from ...config_models import Environment
from ...logger import belief_scope, logger
from ...logger import logger
from ...superset_client import SupersetClient
# #endregion _base_imports
logger = cast(Any, logger)
# #region SupersetParsedContext [C:2] [TYPE Class]
@@ -28,15 +29,15 @@ logger = cast(Any, logger)
class SupersetParsedContext:
source_url: str
dataset_ref: str
dataset_id: Optional[int] = None
dashboard_id: Optional[int] = None
chart_id: Optional[int] = None
dataset_id: int | None = None
dashboard_id: int | None = None
chart_id: int | None = None
resource_type: str = "unknown"
query_state: Dict[str, Any] = field(default_factory=dict)
imported_filters: List[Dict[str, Any]] = field(default_factory=list)
unresolved_references: List[str] = field(default_factory=list)
query_state: dict[str, Any] = field(default_factory=dict)
imported_filters: list[dict[str, Any]] = field(default_factory=list)
unresolved_references: list[str] = field(default_factory=list)
partial_recovery: bool = False
dataset_payload: Optional[Dict[str, Any]] = None
dataset_payload: dict[str, Any] | None = None
# #endregion SupersetParsedContext
# #region SupersetContextExtractorBase [C:4] [TYPE Class]
# @BRIEF Bind extractor to one Superset environment and client instance; provide shared URL-parsing helpers.
@@ -48,7 +49,7 @@ class SupersetContextExtractorBase:
# #region SupersetContextExtractorBase.__init__ [TYPE Function] [C:2]
# @PURPOSE: Bind extractor to one Superset environment and client instance.
def __init__(
self, environment: Environment, client: Optional[SupersetClient] = None
self, environment: Environment, client: SupersetClient | None = None
) -> None:
self.environment = environment
self.client = client or SupersetClient(environment)
@@ -57,7 +58,7 @@ class SupersetContextExtractorBase:
# @PURPOSE: Summarize recovered, partial, and unresolved context for session state and UX.
def build_recovery_summary(
self, parsed_context: SupersetParsedContext
) -> Dict[str, Any]:
) -> dict[str, Any]:
return {
"dataset_ref": parsed_context.dataset_ref,
"dataset_id": parsed_context.dataset_id,
@@ -71,8 +72,8 @@ class SupersetContextExtractorBase:
# #region SupersetContextExtractorBase._extract_numeric_identifier [TYPE Function] [C:2]
# @PURPOSE: Extract a numeric identifier from a REST-like Superset URL path.
def _extract_numeric_identifier(
self, path_parts: List[str], resource_name: str
) -> Optional[int]:
self, path_parts: list[str], resource_name: str
) -> int | None:
if resource_name not in path_parts:
return None
try:
@@ -88,7 +89,7 @@ class SupersetContextExtractorBase:
# #endregion SupersetContextExtractorBase._extract_numeric_identifier
# #region SupersetContextExtractorBase._extract_dashboard_reference [TYPE Function] [C:2]
# @PURPOSE: Extract a dashboard id-or-slug reference from a Superset URL path.
def _extract_dashboard_reference(self, path_parts: List[str]) -> Optional[str]:
def _extract_dashboard_reference(self, path_parts: list[str]) -> str | None:
if "dashboard" not in path_parts:
return None
try:
@@ -104,7 +105,7 @@ class SupersetContextExtractorBase:
# #endregion SupersetContextExtractorBase._extract_dashboard_reference
# #region SupersetContextExtractorBase._extract_dashboard_permalink_key [TYPE Function] [C:2]
# @PURPOSE: Extract a dashboard permalink key from a Superset URL path.
def _extract_dashboard_permalink_key(self, path_parts: List[str]) -> Optional[str]:
def _extract_dashboard_permalink_key(self, path_parts: list[str]) -> str | None:
if "dashboard" not in path_parts:
return None
try:
@@ -121,7 +122,7 @@ class SupersetContextExtractorBase:
# #endregion SupersetContextExtractorBase._extract_dashboard_permalink_key
# #region SupersetContextExtractorBase._extract_dashboard_id_from_state [TYPE Function] [C:2]
# @PURPOSE: Extract a dashboard identifier from returned permalink state when present.
def _extract_dashboard_id_from_state(self, state: Dict[str, Any]) -> Optional[int]:
def _extract_dashboard_id_from_state(self, state: dict[str, Any]) -> int | None:
return self._search_nested_numeric_key(
payload=state,
candidate_keys={"dashboardId", "dashboard_id", "dashboard_id_value"},
@@ -129,7 +130,7 @@ class SupersetContextExtractorBase:
# #endregion SupersetContextExtractorBase._extract_dashboard_id_from_state
# #region SupersetContextExtractorBase._extract_chart_id_from_state [TYPE Function] [C:2]
# @PURPOSE: Extract a chart identifier from returned permalink state when dashboard id is absent.
def _extract_chart_id_from_state(self, state: Dict[str, Any]) -> Optional[int]:
def _extract_chart_id_from_state(self, state: dict[str, Any]) -> int | None:
return self._search_nested_numeric_key(
payload=state,
candidate_keys={"slice_id", "sliceId", "chartId", "chart_id"},
@@ -139,8 +140,8 @@ class SupersetContextExtractorBase:
# @PURPOSE: Recursively search nested dict/list payloads for the first numeric value under a candidate key set.
# @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase]
def _search_nested_numeric_key(
self, payload: Any, candidate_keys: Set[str]
) -> Optional[int]:
self, payload: Any, candidate_keys: set[str]
) -> int | None:
if isinstance(payload, dict):
for key, value in payload.items():
if key in candidate_keys:
@@ -161,8 +162,8 @@ class SupersetContextExtractorBase:
# #endregion SupersetContextExtractorBase._search_nested_numeric_key
# #region SupersetContextExtractorBase._decode_query_state [TYPE Function] [C:2]
# @PURPOSE: Decode query-string structures used by Superset URL state transport.
def _decode_query_state(self, query_params: Dict[str, List[str]]) -> Dict[str, Any]:
query_state: Dict[str, Any] = {}
def _decode_query_state(self, query_params: dict[str, list[str]]) -> dict[str, Any]:
query_state: dict[str, Any] = {}
for key, values in query_params.items():
if not values:
continue
@@ -180,4 +181,4 @@ class SupersetContextExtractorBase:
query_state[key] = decoded_value
return query_state
# #endregion SupersetContextExtractorBase._decode_query_state
# #endregion SupersetContextExtractorBase
# #endregion SupersetContextExtractorBase

View File

@@ -5,9 +5,12 @@
# #endregion SupersetContextFiltersExtractMixin
# #region _filters_imports [TYPE Block]
from __future__ import annotations
from copy import deepcopy
from typing import Any, Dict, List, cast
from typing import Any, cast
from ...logger import logger as app_logger
app_logger = cast(Any, app_logger)
# #endregion _filters_imports
# #region SupersetContextFiltersExtractMixin [C:3] [TYPE Class]
@@ -16,9 +19,9 @@ class SupersetContextFiltersExtractMixin:
# #region SupersetContextFiltersExtractMixin._extract_imported_filters [TYPE Function] [C:2]
# @PURPOSE: Normalize imported filters from decoded query state without fabricating missing values.
def _extract_imported_filters(
self, query_state: Dict[str, Any]
) -> List[Dict[str, Any]]:
imported_filters: List[Dict[str, Any]] = []
self, query_state: dict[str, Any]
) -> list[dict[str, Any]]:
imported_filters: list[dict[str, Any]] = []
native_filters_payload = query_state.get("native_filters")
if isinstance(native_filters_payload, list):
for index, item in enumerate(native_filters_payload):
@@ -230,4 +233,4 @@ class SupersetContextFiltersExtractMixin:
)
return imported_filters
# #endregion SupersetContextFiltersExtractMixin._extract_imported_filters
# #endregion SupersetContextFiltersExtractMixin
# #endregion SupersetContextFiltersExtractMixin

View File

@@ -6,10 +6,13 @@
# #endregion SupersetContextParsingMixin
# #region _parsing_imports [TYPE Block]
from __future__ import annotations
from typing import Any, Dict, List, Optional, cast
from typing import Any, cast
from urllib.parse import parse_qs, urlparse
from ...logger import belief_scope, logger
from ._base import SupersetParsedContext
logger = cast(Any, logger)
# #endregion _parsing_imports
# #region SupersetContextParsingMixin [C:4] [TYPE Class]
@@ -48,9 +51,9 @@ class SupersetContextParsingMixin:
dashboard_permalink_key = self._extract_dashboard_permalink_key(path_parts)
chart_id = self._extract_numeric_identifier(path_parts, "chart")
resource_type = "unknown"
dataset_ref: Optional[str] = None
dataset_ref: str | None = None
partial_recovery = False
unresolved_references: List[str] = []
unresolved_references: list[str] = []
if dataset_id is not None:
resource_type = "dataset"
dataset_ref = f"dataset:{dataset_id}"
@@ -253,7 +256,7 @@ class SupersetContextParsingMixin:
extra={"path": parsed_url.path},
)
raise ValueError("Unsupported Superset link shape")
dataset_payload: Optional[Dict[str, Any]] = None
dataset_payload: dict[str, Any] | None = None
if dataset_id is not None:
try:
dataset_payload = self.client.get_dataset_detail(dataset_id)
@@ -317,9 +320,9 @@ class SupersetContextParsingMixin:
def _recover_dataset_binding_from_dashboard(
self,
dashboard_id: int,
dataset_ref: Optional[str],
unresolved_references: List[str],
) -> tuple[Optional[int], List[str]]:
dataset_ref: str | None,
unresolved_references: list[str],
) -> tuple[int | None, list[str]]:
dashboard_detail = self.client.get_dashboard_detail(dashboard_id)
datasets = dashboard_detail.get("datasets") or []
if datasets:
@@ -349,4 +352,4 @@ class SupersetContextParsingMixin:
unresolved_references.append("dashboard_dataset_binding_missing")
return None, unresolved_references
# #endregion SupersetContextParsingMixin._recover_dataset_binding_from_dashboard
# #endregion SupersetContextParsingMixin
# #endregion SupersetContextParsingMixin

View File

@@ -6,7 +6,8 @@
# #region _pii_imports [TYPE Block]
import re
from copy import deepcopy
from typing import Any, Dict, List, Tuple
from typing import Any
# #endregion _pii_imports
_EMAIL_PATTERN = re.compile(
@@ -23,12 +24,12 @@ _MIXED_IDENTIFIER_PATTERN = re.compile(
# #region mask_pii_value [C:2] [TYPE Function]
# @BRIEF Redact likely sensitive identifiers while preserving structural hints for assistant-facing dataset-review context.
def mask_pii_value(value: Any) -> Tuple[Any, bool]:
def mask_pii_value(value: Any) -> tuple[Any, bool]:
if isinstance(value, str):
return _mask_sensitive_text(value)
if isinstance(value, list):
masked_any = False
masked_list: List[Any] = []
masked_list: list[Any] = []
for item in value:
masked_item, item_masked = mask_pii_value(item)
masked_list.append(masked_item)
@@ -36,7 +37,7 @@ def mask_pii_value(value: Any) -> Tuple[Any, bool]:
return masked_list, masked_any
if isinstance(value, dict):
masked_any = False
masked_dict: Dict[str, Any] = {}
masked_dict: dict[str, Any] = {}
for key, item in value.items():
masked_item, item_masked = mask_pii_value(item)
masked_dict[key] = masked_item
@@ -51,8 +52,8 @@ def mask_pii_value(value: Any) -> Tuple[Any, bool]:
# #region sanitize_imported_filter_for_assistant [C:2] [TYPE Function]
# @BRIEF Build assistant-safe imported-filter payload with raw-value redaction metadata when sensitive tokens are detected.
def sanitize_imported_filter_for_assistant(
filter_payload: Dict[str, Any],
) -> Dict[str, Any]:
filter_payload: dict[str, Any],
) -> dict[str, Any]:
sanitized = deepcopy(filter_payload)
masked_raw_value, was_masked = mask_pii_value(filter_payload.get("raw_value"))
sanitized["raw_value"] = masked_raw_value
@@ -67,7 +68,7 @@ def sanitize_imported_filter_for_assistant(
# #region _mask_sensitive_text [C:2] [TYPE Function]
# @BRIEF Redact likely PII substrings from one string while preserving non-sensitive delimiters and token shape.
def _mask_sensitive_text(value: str) -> Tuple[str, bool]:
def _mask_sensitive_text(value: str) -> tuple[str, bool]:
masked = value
masked_any = False

View File

@@ -6,11 +6,14 @@
# #endregion SupersetContextRecoveryMixin
# #region _recovery_imports [TYPE Block]
from __future__ import annotations
import json
from copy import deepcopy
from typing import Any, Dict, List, Set, cast
from typing import Any, cast
from ...logger import belief_scope, logger
from ._base import SupersetParsedContext
logger = cast(Any, logger)
# #endregion _recovery_imports
# #region SupersetContextRecoveryMixin [C:4] [TYPE Class]
@@ -25,13 +28,13 @@ class SupersetContextRecoveryMixin:
# @DATA_CONTRACT: Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]]
def recover_imported_filters(
self, parsed_context: SupersetParsedContext
) -> List[Dict[str, Any]]:
) -> list[dict[str, Any]]:
with belief_scope("SupersetContextExtractor.recover_imported_filters"):
recovered_filters: List[Dict[str, Any]] = []
seen_filter_keys: Set[str] = set()
metadata_filters: List[Dict[str, Any]] = []
metadata_filters_by_id: Dict[str, Dict[str, Any]] = {}
def merge_recovered_filter(candidate: Dict[str, Any]) -> None:
recovered_filters: list[dict[str, Any]] = []
seen_filter_keys: set[str] = set()
metadata_filters: list[dict[str, Any]] = []
metadata_filters_by_id: dict[str, dict[str, Any]] = {}
def merge_recovered_filter(candidate: dict[str, Any]) -> None:
filter_key = candidate["filter_name"].strip().lower()
existing_index = next(
(
@@ -242,10 +245,10 @@ class SupersetContextRecoveryMixin:
# @PURPOSE: Normalize one imported-filter payload with explicit provenance and confirmation state.
def _normalize_imported_filter_payload(
self,
payload: Dict[str, Any],
payload: dict[str, Any],
default_source: str,
default_note: str,
) -> Dict[str, Any]:
) -> dict[str, Any]:
raw_value = payload.get("raw_value")
if "raw_value" not in payload and "value" in payload:
raw_value = payload.get("value")
@@ -279,4 +282,4 @@ class SupersetContextRecoveryMixin:
"notes": str(payload.get("notes") or default_note),
}
# #endregion SupersetContextRecoveryMixin._normalize_imported_filter_payload
# #endregion SupersetContextRecoveryMixin
# #endregion SupersetContextRecoveryMixin

View File

@@ -6,9 +6,12 @@
# #endregion SupersetContextTemplatesMixin
# #region _templates_imports [TYPE Block]
from __future__ import annotations
import re
from typing import Any, Dict, List, Optional, Set, cast
from typing import Any, cast
from ...logger import belief_scope, logger
logger = cast(Any, logger)
# #endregion _templates_imports
# #region SupersetContextTemplatesMixin [C:3] [TYPE Class]
@@ -21,11 +24,11 @@ class SupersetContextTemplatesMixin:
# @SIDE_EFFECT: none.
# @DATA_CONTRACT: Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]]
def discover_template_variables(
self, dataset_payload: Dict[str, Any]
) -> List[Dict[str, Any]]:
self, dataset_payload: dict[str, Any]
) -> list[dict[str, Any]]:
with belief_scope("SupersetContextExtractor.discover_template_variables"):
discovered: List[Dict[str, Any]] = []
seen_variable_names: Set[str] = set()
discovered: list[dict[str, Any]] = []
seen_variable_names: set[str] = set()
for expression_source in self._collect_query_bearing_expressions(
dataset_payload
):
@@ -111,9 +114,9 @@ class SupersetContextTemplatesMixin:
# @PURPOSE: Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery.
# @RELATION: DEPENDS_ON -> [SupersetContextTemplatesMixin.discover_template_variables]
def _collect_query_bearing_expressions(
self, dataset_payload: Dict[str, Any]
) -> List[str]:
expressions: List[str] = []
self, dataset_payload: dict[str, Any]
) -> list[str]:
expressions: list[str] = []
def append_expression(candidate: Any) -> None:
if not isinstance(candidate, str):
return
@@ -147,8 +150,8 @@ class SupersetContextTemplatesMixin:
# @PURPOSE: Append one deduplicated template-variable descriptor.
def _append_template_variable(
self,
discovered: List[Dict[str, Any]],
seen_variable_names: Set[str],
discovered: list[dict[str, Any]],
seen_variable_names: set[str],
variable_name: str,
expression_source: str,
variable_kind: str,
@@ -175,7 +178,7 @@ class SupersetContextTemplatesMixin:
# #endregion SupersetContextTemplatesMixin._append_template_variable
# #region SupersetContextTemplatesMixin._extract_primary_jinja_identifier [TYPE Function] [C:2]
# @PURPOSE: Extract a deterministic primary identifier from a Jinja expression without executing it.
def _extract_primary_jinja_identifier(self, expression: str) -> Optional[str]:
def _extract_primary_jinja_identifier(self, expression: str) -> str | None:
matched = re.match(r"([A-Za-z_][A-Za-z0-9_]*)", expression.strip())
if matched is None:
return None
@@ -195,7 +198,7 @@ class SupersetContextTemplatesMixin:
# #endregion SupersetContextTemplatesMixin._extract_primary_jinja_identifier
# #region SupersetContextTemplatesMixin._normalize_default_literal [TYPE Function] [C:2]
# @PURPOSE: Normalize literal default fragments from template helper calls into JSON-safe values.
def _normalize_default_literal(self, literal: Optional[str]) -> Any:
def _normalize_default_literal(self, literal: str | None) -> Any:
normalized_literal = str(literal or "").strip()
if not normalized_literal:
return None
@@ -218,4 +221,4 @@ class SupersetContextTemplatesMixin:
except ValueError:
return normalized_literal
# #endregion SupersetContextTemplatesMixin._normalize_default_literal
# #endregion SupersetContextTemplatesMixin
# #endregion SupersetContextTemplatesMixin