mapper + lint
This commit is contained in:
@@ -1,265 +1,198 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# pylint: disable=too-many-arguments,too-many-locals,too-many-statements,too-many-branches,unused-argument
|
||||
"""
|
||||
[MODULE] Сетевой клиент для API
|
||||
# <GRACE_MODULE id="superset_tool.utils.network" name="network.py">
|
||||
# @SEMANTICS: network, http, client, api, requests, session, authentication
|
||||
# @PURPOSE: Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API, включая аутентификацию, управление сессией, retry-логику и обработку ошибок.
|
||||
# @DEPENDS_ON: superset_tool.exceptions -> Для генерации специфичных сетевых и API ошибок.
|
||||
# @DEPENDS_ON: superset_tool.utils.logger -> Для детального логирования сетевых операций.
|
||||
# @DEPENDS_ON: requests -> Основа для выполнения HTTP-запросов.
|
||||
|
||||
[DESCRIPTION]
|
||||
Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API.
|
||||
"""
|
||||
|
||||
# [IMPORTS] Стандартная библиотека
|
||||
from typing import Optional, Dict, Any, BinaryIO, List, Union
|
||||
# <IMPORTS>
|
||||
from typing import Optional, Dict, Any, List, Union
|
||||
import json
|
||||
import io
|
||||
from pathlib import Path
|
||||
|
||||
# [IMPORTS] Сторонние библиотеки
|
||||
import requests
|
||||
import urllib3 # Для отключения SSL-предупреждений
|
||||
import urllib3
|
||||
from superset_tool.exceptions import AuthenticationError, NetworkError, DashboardNotFoundError, SupersetAPIError, PermissionDeniedError
|
||||
from superset_tool.utils.logger import SupersetLogger
|
||||
# </IMPORTS>
|
||||
|
||||
# [IMPORTS] Локальные модули
|
||||
from superset_tool.exceptions import (
|
||||
AuthenticationError,
|
||||
NetworkError,
|
||||
DashboardNotFoundError,
|
||||
SupersetAPIError,
|
||||
PermissionDeniedError
|
||||
)
|
||||
from superset_tool.utils.logger import SupersetLogger # Импорт логгера
|
||||
|
||||
# [CONSTANTS]
|
||||
DEFAULT_RETRIES = 3
|
||||
DEFAULT_BACKOFF_FACTOR = 0.5
|
||||
DEFAULT_TIMEOUT = 30
|
||||
# --- Начало кода модуля ---
|
||||
|
||||
# <ANCHOR id="APIClient" type="Class">
|
||||
# @PURPOSE: Инкапсулирует HTTP-логику для работы с API, включая сессии, аутентификацию, и обработку запросов.
|
||||
class APIClient:
|
||||
"""[NETWORK-CORE] Инкапсулирует HTTP-логику для работы с API."""
|
||||
DEFAULT_TIMEOUT = 30
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Dict[str, Any],
|
||||
verify_ssl: bool = True,
|
||||
timeout: int = DEFAULT_TIMEOUT,
|
||||
logger: Optional[SupersetLogger] = None
|
||||
):
|
||||
def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT, logger: Optional[SupersetLogger] = None):
|
||||
# <ANCHOR id="APIClient.__init__" type="Function">
|
||||
# @PURPOSE: Инициализирует API клиент с конфигурацией, сессией и логгером.
|
||||
self.logger = logger or SupersetLogger(name="APIClient")
|
||||
self.logger.info("[INFO][APIClient.__init__][ENTER] Initializing APIClient.")
|
||||
self.logger.info("[APIClient.__init__][Enter] Initializing APIClient.")
|
||||
self.base_url = config.get("base_url")
|
||||
self.auth = config.get("auth")
|
||||
self.request_settings = {
|
||||
"verify_ssl": verify_ssl,
|
||||
"timeout": timeout
|
||||
}
|
||||
self.request_settings = {"verify_ssl": verify_ssl, "timeout": timeout}
|
||||
self.session = self._init_session()
|
||||
self._tokens: Dict[str, str] = {}
|
||||
self._authenticated = False
|
||||
self.logger.info("[INFO][APIClient.__init__][SUCCESS] APIClient initialized.")
|
||||
self.logger.info("[APIClient.__init__][Exit] APIClient initialized.")
|
||||
# </ANCHOR>
|
||||
|
||||
def _init_session(self) -> requests.Session:
|
||||
self.logger.debug("[DEBUG][APIClient._init_session][ENTER] Initializing session.")
|
||||
# <ANCHOR id="APIClient._init_session" type="Function">
|
||||
# @PURPOSE: Создает и настраивает `requests.Session` с retry-логикой.
|
||||
# @INTERNAL
|
||||
session = requests.Session()
|
||||
retries = requests.adapters.Retry(
|
||||
total=DEFAULT_RETRIES,
|
||||
backoff_factor=DEFAULT_BACKOFF_FACTOR,
|
||||
status_forcelist=[500, 502, 503, 504],
|
||||
allowed_methods={"HEAD", "GET", "POST", "PUT", "DELETE"}
|
||||
)
|
||||
retries = requests.adapters.Retry(total=3, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504])
|
||||
adapter = requests.adapters.HTTPAdapter(max_retries=retries)
|
||||
session.mount('http://', adapter)
|
||||
session.mount('https://', adapter)
|
||||
verify_ssl = self.request_settings.get("verify_ssl", True)
|
||||
session.verify = verify_ssl
|
||||
if not verify_ssl:
|
||||
if not self.request_settings["verify_ssl"]:
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
self.logger.warning("[WARNING][APIClient._init_session][STATE_CHANGE] SSL verification disabled.")
|
||||
self.logger.debug("[DEBUG][APIClient._init_session][SUCCESS] Session initialized.")
|
||||
self.logger.warning("[_init_session][State] SSL verification disabled.")
|
||||
session.verify = self.request_settings["verify_ssl"]
|
||||
return session
|
||||
# </ANCHOR>
|
||||
|
||||
def authenticate(self) -> Dict[str, str]:
|
||||
self.logger.info(f"[INFO][APIClient.authenticate][ENTER] Authenticating to {self.base_url}")
|
||||
# <ANCHOR id="APIClient.authenticate" type="Function">
|
||||
# @PURPOSE: Выполняет аутентификацию в Superset API и получает access и CSRF токены.
|
||||
# @POST: `self._tokens` заполнен, `self._authenticated` установлен в `True`.
|
||||
# @RETURN: Словарь с токенами.
|
||||
# @THROW: AuthenticationError, NetworkError - при ошибках.
|
||||
self.logger.info("[authenticate][Enter] Authenticating to %s", self.base_url)
|
||||
try:
|
||||
login_url = f"{self.base_url}/security/login"
|
||||
response = self.session.post(
|
||||
login_url,
|
||||
json=self.auth,
|
||||
timeout=self.request_settings.get("timeout", DEFAULT_TIMEOUT)
|
||||
)
|
||||
response = self.session.post(login_url, json=self.auth, timeout=self.request_settings["timeout"])
|
||||
response.raise_for_status()
|
||||
access_token = response.json()["access_token"]
|
||||
|
||||
csrf_url = f"{self.base_url}/security/csrf_token/"
|
||||
csrf_response = self.session.get(
|
||||
csrf_url,
|
||||
headers={"Authorization": f"Bearer {access_token}"},
|
||||
timeout=self.request_settings.get("timeout", DEFAULT_TIMEOUT)
|
||||
)
|
||||
csrf_response = self.session.get(csrf_url, headers={"Authorization": f"Bearer {access_token}"}, timeout=self.request_settings["timeout"])
|
||||
csrf_response.raise_for_status()
|
||||
csrf_token = csrf_response.json()["result"]
|
||||
self._tokens = {
|
||||
"access_token": access_token,
|
||||
"csrf_token": csrf_token
|
||||
}
|
||||
|
||||
self._tokens = {"access_token": access_token, "csrf_token": csrf_response.json()["result"]}
|
||||
self._authenticated = True
|
||||
self.logger.info(f"[INFO][APIClient.authenticate][SUCCESS] Authenticated successfully. Tokens {self._tokens}")
|
||||
self.logger.info("[authenticate][Exit] Authenticated successfully.")
|
||||
return self._tokens
|
||||
except requests.exceptions.HTTPError as e:
|
||||
self.logger.error(f"[ERROR][APIClient.authenticate][FAILURE] Authentication failed: {e}")
|
||||
raise AuthenticationError(f"Authentication failed: {e}") from e
|
||||
except (requests.exceptions.RequestException, KeyError) as e:
|
||||
self.logger.error(f"[ERROR][APIClient.authenticate][FAILURE] Network or parsing error: {e}")
|
||||
raise NetworkError(f"Network or parsing error during authentication: {e}") from e
|
||||
# </ANCHOR>
|
||||
|
||||
@property
|
||||
def headers(self) -> Dict[str, str]:
|
||||
if not self._authenticated:
|
||||
self.authenticate()
|
||||
# <ANCHOR id="APIClient.headers" type="Property">
|
||||
# @PURPOSE: Возвращает HTTP-заголовки для аутентифицированных запросов.
|
||||
if not self._authenticated: self.authenticate()
|
||||
return {
|
||||
"Authorization": f"Bearer {self._tokens['access_token']}",
|
||||
"X-CSRFToken": self._tokens.get("csrf_token", ""),
|
||||
"Referer": self.base_url,
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
# </ANCHOR>
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
endpoint: str,
|
||||
headers: Optional[Dict] = None,
|
||||
raw_response: bool = False,
|
||||
**kwargs
|
||||
) -> Union[requests.Response, Dict[str, Any]]:
|
||||
self.logger.debug(f"[DEBUG][APIClient.request][ENTER] Requesting {method} {endpoint}")
|
||||
def request(self, method: str, endpoint: str, headers: Optional[Dict] = None, raw_response: bool = False, **kwargs) -> Union[requests.Response, Dict[str, Any]]:
|
||||
# <ANCHOR id="APIClient.request" type="Function">
|
||||
# @PURPOSE: Выполняет универсальный HTTP-запрос к API.
|
||||
# @RETURN: `requests.Response` если `raw_response=True`, иначе `dict`.
|
||||
# @THROW: SupersetAPIError, NetworkError и их подклассы.
|
||||
full_url = f"{self.base_url}{endpoint}"
|
||||
_headers = self.headers.copy()
|
||||
if headers:
|
||||
_headers.update(headers)
|
||||
timeout = kwargs.pop('timeout', self.request_settings.get("timeout", DEFAULT_TIMEOUT))
|
||||
if headers: _headers.update(headers)
|
||||
|
||||
try:
|
||||
response = self.session.request(
|
||||
method,
|
||||
full_url,
|
||||
headers=_headers,
|
||||
timeout=timeout,
|
||||
**kwargs
|
||||
)
|
||||
response = self.session.request(method, full_url, headers=_headers, **kwargs)
|
||||
response.raise_for_status()
|
||||
self.logger.debug(f"[DEBUG][APIClient.request][SUCCESS] Request successful for {method} {endpoint}")
|
||||
return response if raw_response else response.json()
|
||||
except requests.exceptions.HTTPError as e:
|
||||
self.logger.error(f"[ERROR][APIClient.request][FAILURE] HTTP error for {method} {endpoint}: {e}")
|
||||
self._handle_http_error(e, endpoint, context={})
|
||||
self._handle_http_error(e, endpoint)
|
||||
except requests.exceptions.RequestException as e:
|
||||
self.logger.error(f"[ERROR][APIClient.request][FAILURE] Network error for {method} {endpoint}: {e}")
|
||||
self._handle_network_error(e, full_url)
|
||||
# </ANCHOR>
|
||||
|
||||
def _handle_http_error(self, e, endpoint, context):
|
||||
def _handle_http_error(self, e: requests.exceptions.HTTPError, endpoint: str):
|
||||
# <ANCHOR id="APIClient._handle_http_error" type="Function">
|
||||
# @PURPOSE: (Helper) Преобразует HTTP ошибки в кастомные исключения.
|
||||
# @INTERNAL
|
||||
status_code = e.response.status_code
|
||||
if status_code == 404:
|
||||
raise DashboardNotFoundError(endpoint, context=context) from e
|
||||
if status_code == 403:
|
||||
raise PermissionDeniedError("Доступ запрещен.", **context) from e
|
||||
if status_code == 401:
|
||||
raise AuthenticationError("Аутентификация не удалась.", **context) from e
|
||||
raise SupersetAPIError(f"Ошибка API: {status_code} - {e.response.text}", **context) from e
|
||||
if status_code == 404: raise DashboardNotFoundError(endpoint) from e
|
||||
if status_code == 403: raise PermissionDeniedError() from e
|
||||
if status_code == 401: raise AuthenticationError() from e
|
||||
raise SupersetAPIError(f"API Error {status_code}: {e.response.text}") from e
|
||||
# </ANCHOR>
|
||||
|
||||
def _handle_network_error(self, e, url):
|
||||
if isinstance(e, requests.exceptions.Timeout):
|
||||
msg = "Таймаут запроса"
|
||||
elif isinstance(e, requests.exceptions.ConnectionError):
|
||||
msg = "Ошибка соединения"
|
||||
else:
|
||||
msg = f"Неизвестная сетевая ошибка: {e}"
|
||||
def _handle_network_error(self, e: requests.exceptions.RequestException, url: str):
|
||||
# <ANCHOR id="APIClient._handle_network_error" type="Function">
|
||||
# @PURPOSE: (Helper) Преобразует сетевые ошибки в `NetworkError`.
|
||||
# @INTERNAL
|
||||
if isinstance(e, requests.exceptions.Timeout): msg = "Request timeout"
|
||||
elif isinstance(e, requests.exceptions.ConnectionError): msg = "Connection error"
|
||||
else: msg = f"Unknown network error: {e}"
|
||||
raise NetworkError(msg, url=url) from e
|
||||
# </ANCHOR>
|
||||
|
||||
def upload_file(
|
||||
self,
|
||||
endpoint: str,
|
||||
file_info: Dict[str, Any],
|
||||
extra_data: Optional[Dict] = None,
|
||||
timeout: Optional[int] = None
|
||||
) -> Dict:
|
||||
self.logger.info(f"[INFO][APIClient.upload_file][ENTER] Uploading file to {endpoint}")
|
||||
def upload_file(self, endpoint: str, file_info: Dict[str, Any], extra_data: Optional[Dict] = None, timeout: Optional[int] = None) -> Dict:
|
||||
# <ANCHOR id="APIClient.upload_file" type="Function">
|
||||
# @PURPOSE: Загружает файл на сервер через multipart/form-data.
|
||||
# @RETURN: Ответ API в виде словаря.
|
||||
# @THROW: SupersetAPIError, NetworkError, TypeError.
|
||||
full_url = f"{self.base_url}{endpoint}"
|
||||
_headers = self.headers.copy()
|
||||
_headers.pop('Content-Type', None)
|
||||
file_obj = file_info.get("file_obj")
|
||||
file_name = file_info.get("file_name")
|
||||
form_field = file_info.get("form_field", "file")
|
||||
_headers = self.headers.copy(); _headers.pop('Content-Type', None)
|
||||
|
||||
file_obj, file_name, form_field = file_info.get("file_obj"), file_info.get("file_name"), file_info.get("form_field", "file")
|
||||
|
||||
files_payload = {}
|
||||
if isinstance(file_obj, (str, Path)):
|
||||
with open(file_obj, 'rb') as file_to_upload:
|
||||
files_payload = {form_field: (file_name, file_to_upload, 'application/x-zip-compressed')}
|
||||
return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout)
|
||||
with open(file_obj, 'rb') as f:
|
||||
files_payload = {form_field: (file_name, f.read(), 'application/x-zip-compressed')}
|
||||
elif isinstance(file_obj, io.BytesIO):
|
||||
files_payload = {form_field: (file_name, file_obj.getvalue(), 'application/x-zip-compressed')}
|
||||
return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout)
|
||||
elif hasattr(file_obj, 'read'):
|
||||
files_payload = {form_field: (file_name, file_obj, 'application/x-zip-compressed')}
|
||||
return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout)
|
||||
else:
|
||||
self.logger.error(f"[ERROR][APIClient.upload_file][FAILURE] Unsupported file_obj type: {type(file_obj)}")
|
||||
raise TypeError(f"Неподдерживаемый тип 'file_obj': {type(file_obj)}")
|
||||
raise TypeError(f"Unsupported file_obj type: {type(file_obj)}")
|
||||
|
||||
return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout)
|
||||
# </ANCHOR>
|
||||
|
||||
def _perform_upload(self, url, files, data, headers, timeout):
|
||||
self.logger.debug(f"[DEBUG][APIClient._perform_upload][ENTER] Performing upload to {url}")
|
||||
def _perform_upload(self, url: str, files: Dict, data: Optional[Dict], headers: Dict, timeout: Optional[int]) -> Dict:
|
||||
# <ANCHOR id="APIClient._perform_upload" type="Function">
|
||||
# @PURPOSE: (Helper) Выполняет POST запрос с файлом.
|
||||
# @INTERNAL
|
||||
try:
|
||||
response = self.session.post(
|
||||
url=url,
|
||||
files=files,
|
||||
data=data or {},
|
||||
headers=headers,
|
||||
timeout=timeout or self.request_settings.get("timeout")
|
||||
)
|
||||
response = self.session.post(url, files=files, data=data or {}, headers=headers, timeout=timeout or self.request_settings["timeout"])
|
||||
response.raise_for_status()
|
||||
self.logger.info(f"[INFO][APIClient._perform_upload][SUCCESS] Upload successful to {url}")
|
||||
return response.json()
|
||||
except requests.exceptions.HTTPError as e:
|
||||
self.logger.error(f"[ERROR][APIClient._perform_upload][FAILURE] HTTP error during upload: {e}")
|
||||
raise SupersetAPIError(f"Ошибка API при загрузке: {e.response.text}") from e
|
||||
raise SupersetAPIError(f"API error during upload: {e.response.text}") from e
|
||||
except requests.exceptions.RequestException as e:
|
||||
self.logger.error(f"[ERROR][APIClient._perform_upload][FAILURE] Network error during upload: {e}")
|
||||
raise NetworkError(f"Ошибка сети при загрузке: {e}", url=url) from e
|
||||
raise NetworkError(f"Network error during upload: {e}", url=url) from e
|
||||
# </ANCHOR>
|
||||
|
||||
def fetch_paginated_count(
|
||||
self,
|
||||
endpoint: str,
|
||||
query_params: Dict,
|
||||
count_field: str = "count",
|
||||
timeout: Optional[int] = None
|
||||
) -> int:
|
||||
self.logger.debug(f"[DEBUG][APIClient.fetch_paginated_count][ENTER] Fetching paginated count for {endpoint}")
|
||||
response_json = self.request(
|
||||
method="GET",
|
||||
endpoint=endpoint,
|
||||
params={"q": json.dumps(query_params)},
|
||||
timeout=timeout or self.request_settings.get("timeout")
|
||||
)
|
||||
count = response_json.get(count_field, 0)
|
||||
self.logger.debug(f"[DEBUG][APIClient.fetch_paginated_count][SUCCESS] Fetched paginated count: {count}")
|
||||
return count
|
||||
def fetch_paginated_count(self, endpoint: str, query_params: Dict, count_field: str = "count") -> int:
|
||||
# <ANCHOR id="APIClient.fetch_paginated_count" type="Function">
|
||||
# @PURPOSE: Получает общее количество элементов для пагинации.
|
||||
response_json = self.request("GET", endpoint, params={"q": json.dumps(query_params)})
|
||||
return response_json.get(count_field, 0)
|
||||
# </ANCHOR>
|
||||
|
||||
def fetch_paginated_data(
|
||||
self,
|
||||
endpoint: str,
|
||||
pagination_options: Dict[str, Any],
|
||||
timeout: Optional[int] = None
|
||||
) -> List[Any]:
|
||||
self.logger.debug(f"[DEBUG][APIClient.fetch_paginated_data][ENTER] Fetching paginated data for {endpoint}")
|
||||
base_query = pagination_options.get("base_query", {})
|
||||
total_count = pagination_options.get("total_count", 0)
|
||||
results_field = pagination_options.get("results_field", "result")
|
||||
page_size = base_query.get('page_size')
|
||||
if not page_size or page_size <= 0:
|
||||
raise ValueError("'page_size' должен быть положительным числом.")
|
||||
total_pages = (total_count + page_size - 1) // page_size
|
||||
def fetch_paginated_data(self, endpoint: str, pagination_options: Dict[str, Any]) -> List[Any]:
|
||||
# <ANCHOR id="APIClient.fetch_paginated_data" type="Function">
|
||||
# @PURPOSE: Автоматически собирает данные со всех страниц пагинированного эндпоинта.
|
||||
base_query, total_count = pagination_options["base_query"], pagination_options["total_count"]
|
||||
results_field, page_size = pagination_options["results_field"], base_query.get('page_size')
|
||||
assert page_size and page_size > 0, "'page_size' must be a positive number."
|
||||
|
||||
results = []
|
||||
for page in range(total_pages):
|
||||
for page in range((total_count + page_size - 1) // page_size):
|
||||
query = {**base_query, 'page': page}
|
||||
response_json = self.request(
|
||||
method="GET",
|
||||
endpoint=endpoint,
|
||||
params={"q": json.dumps(query)},
|
||||
timeout=timeout or self.request_settings.get("timeout")
|
||||
)
|
||||
page_results = response_json.get(results_field, [])
|
||||
results.extend(page_results)
|
||||
self.logger.debug(f"[DEBUG][APIClient.fetch_paginated_data][SUCCESS] Fetched paginated data. Total items: {len(results)}")
|
||||
return results
|
||||
response_json = self.request("GET", endpoint, params={"q": json.dumps(query)})
|
||||
results.extend(response_json.get(results_field, []))
|
||||
return results
|
||||
# </ANCHOR>
|
||||
|
||||
# </ANCHOR id="APIClient">
|
||||
|
||||
# --- Конец кода модуля ---
|
||||
|
||||
# </GRACE_MODULE id="superset_tool.utils.network">
|
||||
Reference in New Issue
Block a user