project map script | semantic parcer

2026-01-01 16:58:21 +03:00
parent a747a163c8
commit 4c6fc8256d
84 changed files with 10178 additions and 537 deletions
--- a/superset_tool/utils/init.py
+++ b/superset_tool/utils/init.py
@@ -2,4 +2,4 @@
 # @SEMANTICS: package, utils
 # @PURPOSE: Utility package for superset_tool.
 # @LAYER: Infra
-# [/DEF:superset_tool.utils]
+# [/DEF:superset_tool.utils:Module]
--- a/superset_tool/utils/dataset_mapper.py
+++ b/superset_tool/utils/dataset_mapper.py
@@ -85,7 +85,7 @@ class DatasetMapper:
            self.logger.error("[get_postgres_comments][Failure] %s", e, exc_info=True)
            raise
        return comments
-    # [/DEF:DatasetMapper.get_postgres_comments]
+    # [/DEF:DatasetMapper.get_postgres_comments:Function]
 
    # [DEF:DatasetMapper.load_excel_mappings:Function]
    # @PURPOSE: Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла.
@@ -104,7 +104,7 @@ class DatasetMapper:
        except Exception as e:
            self.logger.error("[load_excel_mappings][Failure] %s", e, exc_info=True)
            raise
-    # [/DEF:DatasetMapper.load_excel_mappings]
+    # [/DEF:DatasetMapper.load_excel_mappings:Function]
 
    # [DEF:DatasetMapper.run_mapping:Function]
    # @PURPOSE: Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset.
@@ -223,7 +223,7 @@ class DatasetMapper:
        except (AssertionError, FileNotFoundError, Exception) as e:
            self.logger.error("[run_mapping][Failure] %s", e, exc_info=True)
            return
-    # [/DEF:DatasetMapper.run_mapping]
-# [/DEF:DatasetMapper]
+    # [/DEF:DatasetMapper.run_mapping:Function]
+# [/DEF:DatasetMapper:Class]
 
-# [/DEF:superset_tool.utils.dataset_mapper]
+# [/DEF:superset_tool.utils.dataset_mapper:Module]
--- a/superset_tool/utils/fileio.py
+++ b/superset_tool/utils/fileio.py
@@ -64,7 +64,7 @@ def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode
                    logger.debug("[create_temp_file][Cleanup] Removed temporary file: %s", resource_path)
            except OSError as e:
                logger.error("[create_temp_file][Failure] Error during cleanup of %s: %s", resource_path, e)
-# [/DEF:create_temp_file]
+# [/DEF:create_temp_file:Function]

 # [DEF:remove_empty_directories:Function]
 # @PURPOSE: Рекурсивно удаляет все пустые поддиректории, начиная с указанного пути.
@@ -88,7 +88,7 @@ def remove_empty_directories(root_dir: str, logger: Optional[SupersetLogger] = N
                logger.error("[remove_empty_directories][Failure] Failed to remove %s: %s", current_dir, e)
    logger.info("[remove_empty_directories][Exit] Removed %d empty directories.", removed_count)
    return removed_count
-# [/DEF:remove_empty_directories]
+# [/DEF:remove_empty_directories:Function]

 # [DEF:read_dashboard_from_disk:Function]
 # @PURPOSE: Читает бинарное содержимое файла с диска.
@@ -105,7 +105,7 @@ def read_dashboard_from_disk(file_path: str, logger: Optional[SupersetLogger] =
    if not content:
        logger.warning("[read_dashboard_from_disk][Warning] File is empty: %s", file_path)
    return content, path.name
-# [/DEF:read_dashboard_from_disk]
+# [/DEF:read_dashboard_from_disk:Function]

 # [DEF:calculate_crc32:Function]
 # @PURPOSE: Вычисляет контрольную сумму CRC32 для файла.
@@ -116,7 +116,7 @@ def calculate_crc32(file_path: Path) -> str:
    with open(file_path, 'rb') as f:
        crc32_value = zlib.crc32(f.read())
    return f"{crc32_value:08x}"
-# [/DEF:calculate_crc32]
+# [/DEF:calculate_crc32:Function]

 # [DEF:RetentionPolicy:DataClass]
 # @PURPOSE: Определяет политику хранения для архивов (ежедневные, еженедельные, ежемесячные).
@@ -125,7 +125,7 @@ class RetentionPolicy:
    daily: int = 7
    weekly: int = 4
    monthly: int = 12
-# [/DEF:RetentionPolicy]
+# [/DEF:RetentionPolicy:DataClass]

 # [DEF:archive_exports:Function]
 # @PURPOSE: Управляет архивом экспортированных файлов, применяя политику хранения и дедупликацию.
@@ -207,7 +207,7 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
                logger.info("[archive_exports][State] Removed by retention policy: %s", file_path.name)
            except OSError as e:
                logger.error("[archive_exports][Failure] Failed to remove %s: %s", file_path, e)
-# [/DEF:archive_exports]
+# [/DEF:archive_exports:Function]

 # [DEF:apply_retention_policy:Function]
 # @PURPOSE: (Helper) Применяет политику хранения к списку файлов, возвращая те, что нужно сохранить.
@@ -240,7 +240,7 @@ def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: Re
    files_to_keep.update(monthly_files[:policy.monthly])  
    logger.debug("[apply_retention_policy][State] Keeping %d files according to retention policy", len(files_to_keep))  
    return files_to_keep  
-# [/DEF:apply_retention_policy]  
+# [/DEF:apply_retention_policy:Function]

 # [DEF:save_and_unpack_dashboard:Function]
 # @PURPOSE: Сохраняет бинарное содержимое ZIP-архива на диск и опционально распаковывает его.
@@ -270,7 +270,7 @@ def save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path],
    except zipfile.BadZipFile as e:
        logger.error("[save_and_unpack_dashboard][Failure] Invalid ZIP archive: %s", e)
        raise InvalidZipFormatError(f"Invalid ZIP file: {e}") from e
-# [/DEF:save_and_unpack_dashboard]
+# [/DEF:save_and_unpack_dashboard:Function]

 # [DEF:update_yamls:Function]
 # @PURPOSE: Обновляет конфигурации в YAML-файлах, заменяя значения или применяя regex.
@@ -291,7 +291,7 @@ def update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str =
    
    for file_path in dir_path.rglob("*.yaml"):
        _update_yaml_file(file_path, configs, regexp_pattern, replace_string, logger)
-# [/DEF:update_yamls]
+# [/DEF:update_yamls:Function]

 # [DEF:_update_yaml_file:Function]
 # @PURPOSE: (Helper) Обновляет один YAML файл.
@@ -352,7 +352,7 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_
                f.write(modified_content)
        except Exception as e:
            logger.error("[_update_yaml_file][Failure] Error performing raw replacement in %s: %s", file_path, e)
-# [/DEF:_update_yaml_file]  
+# [/DEF:_update_yaml_file:Function]

 # [DEF:create_dashboard_export:Function]
 # @PURPOSE: Создает ZIP-архив из указанных исходных путей.
@@ -379,7 +379,7 @@ def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union
    except (IOError, zipfile.BadZipFile, AssertionError) as e:
        logger.error("[create_dashboard_export][Failure] Error: %s", e, exc_info=True)
        return False
-# [/DEF:create_dashboard_export]
+# [/DEF:create_dashboard_export:Function]

 # [DEF:sanitize_filename:Function]
 # @PURPOSE: Очищает строку от символов, недопустимых в именах файлов.
@@ -387,7 +387,7 @@ def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union
 # @RETURN: str - Очищенная строка.
 def sanitize_filename(filename: str) -> str:
    return re.sub(r'[\\/*?:"<>|]', "_", filename).strip()
-# [/DEF:sanitize_filename]
+# [/DEF:sanitize_filename:Function]

 # [DEF:get_filename_from_headers:Function]
 # @PURPOSE: Извлекает имя файла из HTTP заголовка 'Content-Disposition'.
@@ -398,7 +398,7 @@ def get_filename_from_headers(headers: dict) -> Optional[str]:
    if match := re.search(r'filename="?([^"]+)"?', content_disposition):
        return match.group(1).strip()
    return None
-# [/DEF:get_filename_from_headers]
+# [/DEF:get_filename_from_headers:Function]

 # [DEF:consolidate_archive_folders:Function]
 # @PURPOSE: Консолидирует директории архивов на основе общего слага в имени.
@@ -453,6 +453,6 @@ def consolidate_archive_folders(root_directory: Path, logger: Optional[SupersetL
                logger.info("[consolidate_archive_folders][State] Removed source directory: %s", source_dir)  
            except Exception as e:  
                logger.error("[consolidate_archive_folders][Failure] Failed to remove source directory %s: %s", source_dir, e)  
-# [/DEF:consolidate_archive_folders]  
+# [/DEF:consolidate_archive_folders:Function]

-# [/DEF:superset_tool.utils.fileio]
+# [/DEF:superset_tool.utils.fileio:Module]
--- a/superset_tool/utils/init_clients.py
+++ b/superset_tool/utils/init_clients.py
@@ -105,6 +105,6 @@ def setup_clients(logger: SupersetLogger, custom_envs: Optional[List[Any]] = Non
    except Exception as e:
        logger.critical("[setup_clients][Failure] Critical error during client initialization: %s", e, exc_info=True)
        raise
-# [/DEF:setup_clients]
+# [/DEF:setup_clients:Function]

-# [/DEF:superset_tool.utils.init_clients]
+# [/DEF:superset_tool.utils.init_clients:Module]
--- a/superset_tool/utils/logger.py
+++ b/superset_tool/utils/logger.py
@@ -53,7 +53,7 @@ class SupersetLogger:
            console_handler = logging.StreamHandler(sys.stdout)
            console_handler.setFormatter(formatter)
            self.logger.addHandler(console_handler)
-    # [/DEF:SupersetLogger.__init__]
+    # [/DEF:SupersetLogger.__init__:Function]

    # [DEF:SupersetLogger._log:Function]
    # @PURPOSE: (Helper) Универсальный метод для вызова соответствующего уровня логирования.
@@ -64,44 +64,44 @@ class SupersetLogger:
    # @PARAM: exc_info (bool) - Добавлять ли информацию об исключении.
    def _log(self, level_method: Any, msg: str, *args: Any, extra: Optional[Mapping[str, Any]] = None, exc_info: bool = False) -> None:
        level_method(msg, *args, extra=extra, exc_info=exc_info)
-    # [/DEF:SupersetLogger._log]
+    # [/DEF:SupersetLogger._log:Function]

    # [DEF:SupersetLogger.info:Function]
    # @PURPOSE: Записывает сообщение уровня INFO.
    def info(self, msg: str, *args: Any, extra: Optional[Mapping[str, Any]] = None, exc_info: bool = False) -> None:
        self._log(self.logger.info, msg, *args, extra=extra, exc_info=exc_info)
-    # [/DEF:SupersetLogger.info]
+    # [/DEF:SupersetLogger.info:Function]

    # [DEF:SupersetLogger.debug:Function]
    # @PURPOSE: Записывает сообщение уровня DEBUG.
    def debug(self, msg: str, *args: Any, extra: Optional[Mapping[str, Any]] = None, exc_info: bool = False) -> None:
        self._log(self.logger.debug, msg, *args, extra=extra, exc_info=exc_info)
-    # [/DEF:SupersetLogger.debug]
+    # [/DEF:SupersetLogger.debug:Function]

    # [DEF:SupersetLogger.warning:Function]
    # @PURPOSE: Записывает сообщение уровня WARNING.
    def warning(self, msg: str, *args: Any, extra: Optional[Mapping[str, Any]] = None, exc_info: bool = False) -> None:
        self._log(self.logger.warning, msg, *args, extra=extra, exc_info=exc_info)
-    # [/DEF:SupersetLogger.warning]
+    # [/DEF:SupersetLogger.warning:Function]

    # [DEF:SupersetLogger.error:Function]
    # @PURPOSE: Записывает сообщение уровня ERROR.
    def error(self, msg: str, *args: Any, extra: Optional[Mapping[str, Any]] = None, exc_info: bool = False) -> None:
        self._log(self.logger.error, msg, *args, extra=extra, exc_info=exc_info)
-    # [/DEF:SupersetLogger.error]
+    # [/DEF:SupersetLogger.error:Function]

    # [DEF:SupersetLogger.critical:Function]
    # @PURPOSE: Записывает сообщение уровня CRITICAL.
    def critical(self, msg: str, *args: Any, extra: Optional[Mapping[str, Any]] = None, exc_info: bool = False) -> None:
        self._log(self.logger.critical, msg, *args, extra=extra, exc_info=exc_info)
-    # [/DEF:SupersetLogger.critical]
+    # [/DEF:SupersetLogger.critical:Function]

    # [DEF:SupersetLogger.exception:Function]
    # @PURPOSE: Записывает сообщение уровня ERROR вместе с трассировкой стека текущего исключения.
    def exception(self, msg: str, *args: Any, **kwargs: Any) -> None:
        self.logger.exception(msg, *args, **kwargs)
-    # [/DEF:SupersetLogger.exception]
+    # [/DEF:SupersetLogger.exception:Function]

-# [/DEF:SupersetLogger]
+# [/DEF:SupersetLogger:Class]

-# [/DEF:superset_tool.utils.logger]
+# [/DEF:superset_tool.utils.logger:Module]
--- a/superset_tool/utils/network.py
+++ b/superset_tool/utils/network.py
@@ -42,7 +42,7 @@ class APIClient:
        self._tokens: Dict[str, str] = {}
        self._authenticated = False
        self.logger.info("[APIClient.__init__][Exit] APIClient initialized.")
-    # [/DEF:APIClient.__init__]
+    # [/DEF:APIClient.__init__:Function]

    # [DEF:APIClient._init_session:Function]
    # @PURPOSE: Создает и настраивает `requests.Session` с retry-логикой.
@@ -58,7 +58,7 @@ class APIClient:
            self.logger.warning("[_init_session][State] SSL verification disabled.")
        session.verify = self.request_settings["verify_ssl"]
        return session
-    # [/DEF:APIClient._init_session]
+    # [/DEF:APIClient._init_session:Function]

    # [DEF:APIClient.authenticate:Function]
    # @PURPOSE: Выполняет аутентификацию в Superset API и получает access и CSRF токены.
@@ -85,7 +85,7 @@ class APIClient:
            raise AuthenticationError(f"Authentication failed: {e}") from e
        except (requests.exceptions.RequestException, KeyError) as e:
            raise NetworkError(f"Network or parsing error during authentication: {e}") from e
-    # [/DEF:APIClient.authenticate]
+    # [/DEF:APIClient.authenticate:Function]

    @property
    def headers(self) -> Dict[str, str]:
@@ -98,7 +98,7 @@ class APIClient:
            "Referer": self.base_url,
            "Content-Type": "application/json"
        }
-        # [/DEF:APIClient.headers]
+        # [/DEF:APIClient.headers:Function]

    # [DEF:APIClient.request:Function]
    # @PURPOSE: Выполняет универсальный HTTP-запрос к API.
@@ -121,7 +121,7 @@ class APIClient:
            self._handle_http_error(e, endpoint)
        except requests.exceptions.RequestException as e:
            self._handle_network_error(e, full_url)
-    # [/DEF:APIClient.request]
+    # [/DEF:APIClient.request:Function]

    # [DEF:APIClient._handle_http_error:Function]
    # @PURPOSE: (Helper) Преобразует HTTP ошибки в кастомные исключения.
@@ -133,7 +133,7 @@ class APIClient:
        if status_code == 403: raise PermissionDeniedError() from e
        if status_code == 401: raise AuthenticationError() from e
        raise SupersetAPIError(f"API Error {status_code}: {e.response.text}") from e
-    # [/DEF:APIClient._handle_http_error]
+    # [/DEF:APIClient._handle_http_error:Function]

    # [DEF:APIClient._handle_network_error:Function]
    # @PURPOSE: (Helper) Преобразует сетевые ошибки в `NetworkError`.
@@ -144,7 +144,7 @@ class APIClient:
        elif isinstance(e, requests.exceptions.ConnectionError): msg = "Connection error"
        else: msg = f"Unknown network error: {e}"
        raise NetworkError(msg, url=url) from e
-    # [/DEF:APIClient._handle_network_error]
+    # [/DEF:APIClient._handle_network_error:Function]

    # [DEF:APIClient.upload_file:Function]
    # @PURPOSE: Загружает файл на сервер через multipart/form-data.
@@ -170,7 +170,7 @@ class APIClient:
            raise TypeError(f"Unsupported file_obj type: {type(file_obj)}")
            
        return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout)
-    # [/DEF:APIClient.upload_file]
+    # [/DEF:APIClient.upload_file:Function]

    # [DEF:APIClient._perform_upload:Function]
    # @PURPOSE: (Helper) Выполняет POST запрос с файлом.
@@ -196,7 +196,7 @@ class APIClient:
            raise SupersetAPIError(f"API error during upload: {e.response.text}") from e
        except requests.exceptions.RequestException as e:
            raise NetworkError(f"Network error during upload: {e}", url=url) from e
-    # [/DEF:APIClient._perform_upload]
+    # [/DEF:APIClient._perform_upload:Function]

    # [DEF:APIClient.fetch_paginated_count:Function]
    # @PURPOSE: Получает общее количество элементов для пагинации.
@@ -207,7 +207,7 @@ class APIClient:
    def fetch_paginated_count(self, endpoint: str, query_params: Dict, count_field: str = "count") -> int:
        response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query_params)}))
        return response_json.get(count_field, 0)
-    # [/DEF:APIClient.fetch_paginated_count]
+    # [/DEF:APIClient.fetch_paginated_count:Function]

    # [DEF:APIClient.fetch_paginated_data:Function]
    # @PURPOSE: Автоматически собирает данные со всех страниц пагинированного эндпоинта.
@@ -225,8 +225,8 @@ class APIClient:
            response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)}))
            results.extend(response_json.get(results_field, []))
        return results
-    # [/DEF:APIClient.fetch_paginated_data]
+    # [/DEF:APIClient.fetch_paginated_data:Function]

-# [/DEF:APIClient]
+# [/DEF:APIClient:Class]

-# [/DEF:superset_tool.utils.network]
+# [/DEF:superset_tool.utils.network:Module]
--- a/superset_tool/utils/whiptail_fallback.py
+++ b/superset_tool/utils/whiptail_fallback.py
@@ -26,7 +26,7 @@ def menu(title: str, prompt: str, choices: List[str], **kwargs) -> Tuple[int, Op
        return (0, choices[sel - 1]) if 0 < sel <= len(choices) else (1, None)
    except (ValueError, IndexError):
        return 1, None
-# [/DEF:menu]
+# [/DEF:menu:Function]

 # [DEF:checklist:Function]
 # @PURPOSE: Отображает список с возможностью множественного выбора.
@@ -46,7 +46,7 @@ def checklist(title: str, prompt: str, options: List[Tuple[str, str]], **kwargs)
        return 0, selected_values
    except (ValueError, IndexError):
        return 1, []
-# [/DEF:checklist]
+# [/DEF:checklist:Function]

 # [DEF:yesno:Function]
 # @PURPOSE: Задает вопрос с ответом да/нет.
@@ -56,7 +56,7 @@ def checklist(title: str, prompt: str, options: List[Tuple[str, str]], **kwargs)
 def yesno(title: str, question: str, **kwargs) -> bool:
    ans = input(f"\n=== {title} ===\n{question} (y/n): ").strip().lower()
    return ans in ("y", "yes", "да", "д")
-# [/DEF:yesno]
+# [/DEF:yesno:Function]

 # [DEF:msgbox:Function]
 # @PURPOSE: Отображает информационное сообщение.
@@ -64,7 +64,7 @@ def yesno(title: str, question: str, **kwargs) -> bool:
 # @PARAM: msg (str) - Текст сообщения.
 def msgbox(title: str, msg: str, **kwargs) -> None:
    print(f"\n=== {title} ===\n{msg}\n")
-# [/DEF:msgbox]
+# [/DEF:msgbox:Function]

 # [DEF:inputbox:Function]
 # @PURPOSE: Запрашивает у пользователя текстовый ввод.
@@ -75,7 +75,7 @@ def inputbox(title: str, prompt: str, **kwargs) -> Tuple[int, Optional[str]]:
    print(f"\n=== {title} ===")
    val = input(f"{prompt}\n")
    return (0, val) if val else (1, None)
-# [/DEF:inputbox]
+# [/DEF:inputbox:Function]

 # [DEF:_ConsoleGauge:Class]
 # @PURPOSE: Контекстный менеджер для имитации `whiptail gauge` в консоли.
@@ -91,7 +91,7 @@ class _ConsoleGauge:
        sys.stdout.write(f"\r{txt} "); sys.stdout.flush()
    def set_percent(self, percent: int) -> None:
        sys.stdout.write(f"{percent}%"); sys.stdout.flush()
-# [/DEF:_ConsoleGauge]
+# [/DEF:_ConsoleGauge:Class]

 # [DEF:gauge:Function]
 # @PURPOSE: Создает и возвращает экземпляр `_ConsoleGauge`.
@@ -99,6 +99,6 @@ class _ConsoleGauge:
 # @RETURN: _ConsoleGauge - Экземпляр контекстного менеджера.
 def gauge(title: str, **kwargs) -> _ConsoleGauge:
    return _ConsoleGauge(title, **kwargs)
-# [/DEF:gauge]
+# [/DEF:gauge:Function]

-# [/DEF:superset_tool.utils.whiptail_fallback]
+# [/DEF:superset_tool.utils.whiptail_fallback:Module]