diff --git a/.env.enterprise-clean.example b/.env.enterprise-clean.example index d3d21441..f2401bc0 100644 --- a/.env.enterprise-clean.example +++ b/.env.enterprise-clean.example @@ -43,6 +43,19 @@ FRONTEND_SSL_PORT=443 # nginx работает в HTTP-only режиме. CERTS_PATH=./certs +# ====================================================================== +# LLM CA-сертификаты (скачка по URL на старте контейнера) +# ====================================================================== +# URL корпоративных CA-сертификатов для LLM-провайдеров. +# Автоматически скачиваются, конвертируются DER→PEM и устанавливаются +# в системное хранилище OpenSSL на старте backend-контейнера. +# +# Формат: пробел-разделённый список URL +# +# Пример: +# LLM_CA_CERT_URLS="http://pki.company.com/root-ca.crt http://pki.company.com/intermediate-ca.crt" +LLM_CA_CERT_URLS= + # ====================================================================== # Логирование # ====================================================================== diff --git a/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py b/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py index d36adaba..ec280aa2 100644 --- a/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py +++ b/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py @@ -52,4 +52,126 @@ def test_litellm_client_uses_default_bearer_auth(): assert "Authentication" not in headers assert "X-API-Key" not in headers # endregion test_litellm_client_uses_default_bearer_auth + + +# region test_get_ssl_verify_default_true [TYPE Function] +# @RELATION BINDS_TO -> TestClientHeaders +# @PURPOSE: _get_ssl_verify defaults to True when LLM_SSL_VERIFY is unset. +# @PRE LLM_SSL_VERIFY env var is not set. +# @POST Returns True. +def test_get_ssl_verify_default_true(): + """Verify default SSL verification is enabled.""" + assert LLMClient._get_ssl_verify() is True +# endregion test_get_ssl_verify_default_true + + +# region test_get_ssl_verify_false_values [TYPE Function] +# @RELATION BINDS_TO -> TestClientHeaders +# @PURPOSE: _get_ssl_verify returns False for "false"/"0"/"no"/"off". +# @PRE LLM_SSL_VERIFY env var is set to each falsy value. +# @POST Returns False. +def test_get_ssl_verify_false_values(monkeypatch): + for val in ("false", "False", "0", "no", "off"): + monkeypatch.setenv("LLM_SSL_VERIFY", val) + assert LLMClient._get_ssl_verify() is False, f"Expected False for LLM_SSL_VERIFY={val}" +# endregion test_get_ssl_verify_false_values + + +# region test_get_ssl_verify_true_values [TYPE Function] +# @RELATION BINDS_TO -> TestClientHeaders +# @PURPOSE: _get_ssl_verify returns True for truthy values like "TRUE", "1", "yes". +# @PRE LLM_SSL_VERIFY env var is set to truthy values. +# @POST Returns True. +def test_get_ssl_verify_true_values(monkeypatch): + for val in ("TRUE", "1", "yes", "on", "true"): + monkeypatch.setenv("LLM_SSL_VERIFY", val) + assert LLMClient._get_ssl_verify() is True, f"Expected True for LLM_SSL_VERIFY={val}" +# endregion test_get_ssl_verify_true_values + + +# region test_format_connection_error_no_cause [TYPE Function] +# @RELATION BINDS_TO -> TestClientHeaders +# @PURPOSE: _format_connection_error with single exception (no cause chain). +# @PRE Exception has no __cause__ or __context__. +# @POST Returns formatted string with only the top-level exception. +def test_format_connection_error_no_cause(): + exc = ValueError("test error") + result = LLMClient._format_connection_error(exc) + assert "ValueError: test error" in result + assert "└─" not in result +# endregion test_format_connection_error_no_cause + + +# region test_format_connection_error_with_cause [TYPE Function] +# @RELATION BINDS_TO -> TestClientHeaders +# @PURPOSE: _format_connection_error with chained exception (__cause__). +# @PRE Exception has a __cause__. +# @POST Returns formatted string with chain. +def test_format_connection_error_with_cause(): + inner = ConnectionRefusedError("Connection refused") + outer = RuntimeError("API call failed") + outer.__cause__ = inner + result = LLMClient._format_connection_error(outer) + assert "RuntimeError: API call failed" in result + assert "└─" in result + assert "ConnectionRefusedError: Connection refused" in result +# endregion test_format_connection_error_with_cause + + +# region test_format_connection_error_deep_chain [TYPE Function] +# @RELATION BINDS_TO -> TestClientHeaders +# @PURPOSE: _format_connection_error with 3-level chain. +# @PRE Exception has nested __cause__ chain. +# @POST Returns formatted string with all levels. +def test_format_connection_error_deep_chain(): + dns = Exception("Name or service not known") + ssl = ConnectionError("SSL: CERTIFICATE_VERIFY_FAILED") + ssl.__cause__ = dns + outer = RuntimeError("Connection error") + outer.__cause__ = ssl + result = LLMClient._format_connection_error(outer) + assert "RuntimeError: Connection error" in result + assert "ConnectionError: SSL: CERTIFICATE_VERIFY_FAILED" in result + assert "Exception: Name or service not known" in result +# endregion test_format_connection_error_deep_chain + + +# region test_litellm_client_verify_default [TYPE Function] +# @RELATION BINDS_TO -> TestClientHeaders +# @PURPOSE: LLMClient passes verify=True to httpx.AsyncClient by default. +# @PRE LLM_SSL_VERIFY env var is not set. +# @POST httpcore pool SSL context has verify_mode=CERT_REQUIRED (2). +def test_litellm_client_verify_default(): + """Verify default SSL context with CERT_REQUIRED when LLM_SSL_VERIFY unset.""" + import ssl + client = LLMClient( + provider_type=LLMProviderType.LITELLM, + api_key="sk-test-verify-key", + base_url="http://localhost:4000/v1", + default_model="gpt-4o", + ) + pool = client.client._client._transport._pool + assert pool._ssl_context.verify_mode == ssl.CERT_REQUIRED, \ + "verify=True should set verify_mode to CERT_REQUIRED" +# endregion test_litellm_client_verify_default + + +# region test_litellm_client_verify_disabled [TYPE Function] +# @RELATION BINDS_TO -> TestClientHeaders +# @PURPOSE: LLMClient passes verify=False when LLM_SSL_VERIFY=false. +# @PRE LLM_SSL_VERIFY is set to "false". +# @POST httpcore pool SSL context has verify_mode=CERT_NONE (0). +def test_litellm_client_verify_disabled(monkeypatch): + monkeypatch.setenv("LLM_SSL_VERIFY", "false") + client = LLMClient( + provider_type=LLMProviderType.LITELLM, + api_key="sk-test-verify-key", + base_url="http://localhost:4000/v1", + default_model="gpt-4o", + ) + import ssl + pool = client.client._client._transport._pool + assert pool._ssl_context.verify_mode == ssl.CERT_NONE, \ + "verify=False should set verify_mode to CERT_NONE" +# endregion test_litellm_client_verify_disabled # endregion TestClientHeaders diff --git a/backend/src/plugins/llm_analysis/service.py b/backend/src/plugins/llm_analysis/service.py index d5cd0811..21424e4a 100644 --- a/backend/src/plugins/llm_analysis/service.py +++ b/backend/src/plugins/llm_analysis/service.py @@ -2,8 +2,6 @@ # @BRIEF Services for LLM interaction and dashboard screenshots. # @LAYER Plugin # @RELATION DEPENDS_ON -> [EXT:Library:tenacity] -# @RELATION DEPENDS_ON -> [EXT:Library:tenacity] -# @RELATION DEPENDS_ON -> [EXT:Library:tenacity] # @INVARIANT Screenshots must be 1920px width and capture full page height. # @DATA_CONTRACT DashboardSpec -> Screenshot + Analysis # @RATIONALE Extracted all hardcoded timeouts into named module-level constants (PLAYWRIGHT_NAVIGATION_TIMEOUT_MS, PLAYWRIGHT_WAIT_TIMEOUT_MS, PLAYWRIGHT_SHORT_TIMEOUT_MS, HTTP_REQUEST_TIMEOUT_MS, SCREENSHOT_SERVICE_TIMEOUT_MS, LLM_HTTP_TIMEOUT_S) and DEFAULT_USER_AGENT. Zero remaining numeric timeout literals. @@ -907,7 +905,14 @@ class LLMClient: # It routes to upstream providers transparently, and the default Authorization header # is sufficient. No additional headers like HTTP-Referer or X-API-Key are required. - http_client = httpx.AsyncClient(headers=default_headers, timeout=LLM_HTTP_TIMEOUT_S) + ssl_verify = self._get_ssl_verify() + logger.info(f"[LLMClient.__init__] SSL Verify: {ssl_verify} (set LLM_SSL_VERIFY=false to disable)") + + http_client = httpx.AsyncClient( + headers=default_headers, + timeout=LLM_HTTP_TIMEOUT_S, + verify=ssl_verify, + ) self.client = AsyncOpenAI( api_key=self.api_key, base_url=base_url, @@ -916,6 +921,28 @@ class LLMClient: ) # endregion LLMClient.__init__ + # region LLMClient._get_ssl_verify [TYPE Function] + # @PURPOSE Resolve SSL verification flag from environment. + # @POST Returns False when LLM_SSL_VERIFY env var is "false"/"0"/"no" (case-insensitive). + @staticmethod + def _get_ssl_verify() -> bool: + raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower() + return raw not in ("false", "0", "no", "off") + # endregion LLMClient._get_ssl_verify + + # region LLMClient._format_connection_error [TYPE Function] + # @PURPOSE Format exception chain for diagnostics, extracting httpx cause details. + # @POST Returns a human-readable string with the full error chain. + @staticmethod + def _format_connection_error(exc: Exception) -> str: + parts = [f"{type(exc).__name__}: {exc!s}"] + cause = exc.__cause__ or exc.__context__ + while cause: + parts.append(f" └─ {type(cause).__name__}: {cause!s}") + cause = cause.__cause__ or cause.__context__ + return "\n".join(parts) + # endregion LLMClient._format_connection_error + # region LLMClient._supports_json_response_format [TYPE Function] # @PURPOSE Detect whether provider/model is likely compatible with response_format=json_object. # @PRE Client initialized with base_url and default_model. @@ -1035,7 +1062,10 @@ class LLMClient: await asyncio.sleep(wait_time) raise except Exception as e: - logger.error(f"[get_json_completion] LLM call failed: {e!s}") + logger.error( + f"[get_json_completion] LLM call failed.\n" + f"{self._format_connection_error(e)}" + ) raise if not response or not hasattr(response, 'choices') or not response.choices: diff --git a/docker-compose.enterprise-clean.yml b/docker-compose.enterprise-clean.yml index 0fa208e2..34d81cdd 100644 --- a/docker-compose.enterprise-clean.yml +++ b/docker-compose.enterprise-clean.yml @@ -48,6 +48,8 @@ services: FEATURES__HEALTH_MONITOR: ${FEATURES__HEALTH_MONITOR:-true} # Путь до сертификатов внутри контейнера — всегда /opt/certs CERTS_PATH: /opt/certs + # URL CA-сертификатов для LLM-провайдеров (скачка на старте, DER→PEM конвертация) + LLM_CA_CERT_URLS: ${LLM_CA_CERT_URLS:-} ports: - "${BACKEND_HOST_PORT:-8001}:8000" volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 0289e23f..c35257b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,6 +37,7 @@ services: INITIAL_ADMIN_PASSWORD: ${INITIAL_ADMIN_PASSWORD:-} FEATURES__DATASET_REVIEW: ${FEATURES__DATASET_REVIEW:-true} FEATURES__HEALTH_MONITOR: ${FEATURES__HEALTH_MONITOR:-true} + LLM_CA_CERT_URLS: ${LLM_CA_CERT_URLS:-} ports: - "${BACKEND_HOST_PORT:-8001}:8000" volumes: diff --git a/docker/backend.entrypoint.sh b/docker/backend.entrypoint.sh index 5e140579..4791dff8 100755 --- a/docker/backend.entrypoint.sh +++ b/docker/backend.entrypoint.sh @@ -75,6 +75,143 @@ install_certificates() { } # #endregion docker.backend.entrypoint.install_certificates +# ====================================================================== +# install_llm_ca_certs — скачка и установка CA-сертификатов по URL +# ====================================================================== +# #region docker.backend.entrypoint.install_llm_ca_certs [C:4] [TYPE Function] +# @BRIEF Скачивает PEM/DER-сертификаты из LLM_CA_CERT_URLS, конвертирует DER→PEM, +# устанавливает в системное хранилище, создаёт хеш-симлинки и при необходимости +# добавляет в ca-certificates.crt напрямую (fallback для OpenSSL 3-совместимости). +# @PRE LLM_CA_CERT_URLS — строка с URL через пробел; curl и openssl установлены. +# @POST Сертификаты доступны в /etc/ssl/certs/ca-certificates.crt. +# Хеш-симлинки созданы для каждого сертификата. +# @SIDE_EFFECT Скачивает файлы из внешних URL. Модифицирует /etc/ssl/certs/. +# @LAYER Infrastructure +# @EXAMPLE: +# LLM_CA_CERT_URLS="http://pki.company.com/root.crt http://pki.company.com/intermediate.crt" +install_llm_ca_certs() { + local urls="${LLM_CA_CERT_URLS:-}" + if [ -z "$urls" ]; then + echo "[entrypoint] LLM_CA_CERT_URLS не задан — пропускаем скачку сертификатов" + return 0 + fi + + local work_dir="/tmp/llm-ca-certs" + local target_dir="/usr/local/share/ca-certificates/llm" + mkdir -p "$work_dir" "$target_dir" + + echo "[entrypoint] === Скачивание и установка LLM CA-сертификатов ===" + local index=0 + local downloaded_names="" + + for url in $urls; do + url="$(echo "$url" | xargs)" # trim + [ -z "$url" ] && continue + index=$((index + 1)) + + # Определяем имя файла из URL, или генерируем + local filename + filename="$(basename "$url" | sed 's/[?#].*//')" + if [ -z "$filename" ] || [ "$filename" = "." ] || [ "$filename" = ".." ]; then + filename="llm-ca-${index}.crt" + fi + + local raw_file="${work_dir}/${filename}" + local pem_file="${target_dir}/$(echo "$filename" | sed 's/\.[^.]*$//').pem" + + echo "[entrypoint] [${index}] Скачивание: ${url}" + + # Скачиваем с retry 3 раза + local attempt=0 + local success=0 + while [ $attempt -lt 3 ]; do + attempt=$((attempt + 1)) + if curl -sS --connect-timeout 10 --max-time 30 -o "$raw_file" "$url" 2>/dev/null; then + success=1 + break + fi + echo "[entrypoint] ⚠ Попытка ${attempt}/3 не удалась, повтор через 2с..." + sleep 2 + done + + if [ "$success" -eq 0 ]; then + echo "[entrypoint] ❌ Не удалось скачать: ${url} — пропускаем" + continue + fi + + # Определяем формат: PEM или DER + if openssl x509 -in "$raw_file" -inform PEM -noout 2>/dev/null; then + echo "[entrypoint] ✓ Формат: PEM" + cp "$raw_file" "$pem_file" + elif openssl x509 -in "$raw_file" -inform DER -noout 2>/dev/null; then + echo "[entrypoint] ↻ Формат: DER — конвертируем в PEM" + openssl x509 -in "$raw_file" -inform DER -out "$pem_file" -outform PEM + else + echo "[entrypoint] ❌ Неизвестный формат (не PEM и не DER) — пропускаем" + rm -f "$raw_file" + continue + fi + + chmod 644 "$pem_file" + echo "[entrypoint] ✅ Установлен: $(basename "$pem_file")" + downloaded_names="$downloaded_names $(basename "$pem_file" .pem)" + rm -f "$raw_file" + done + + if [ -z "$(ls -A "$target_dir" 2>/dev/null)" ]; then + echo "[entrypoint] Нет скачанных сертификатов — пропускаем" + rm -rf "$work_dir" + return 0 + fi + + # Обновляем системное хранилище + echo "[entrypoint] --- Обновление системного хранилища CA-сертификатов ---" + update-ca-certificates --fresh 2>&1 | sed 's/^/[entrypoint] /' + + # Проверяем, попали ли сертификаты в бандл, и создаём хеш-симлинки + echo "[entrypoint] --- Валидация установки сертификатов ---" + local all_ok=true + + for pem in "$target_dir"/*.pem; do + [ -f "$pem" ] || continue + local cert_name + cert_name="$(basename "$pem" .pem)" + local cert_subject + cert_subject="$(openssl x509 -in "$pem" -noout -subject 2>/dev/null | head -1 || echo "unknown")" + + # Проверяем есть ли в ca-certificates.crt + if grep -q "$cert_name" /etc/ssl/certs/ca-certificates.crt 2>/dev/null || \ + grep -qF "$(openssl x509 -in "$pem" -noout -serial 2>/dev/null)" /etc/ssl/certs/ca-certificates.crt 2>/dev/null; then + echo "[entrypoint] ✅ ${cert_name} — в ca-certificates.crt (${cert_subject})" + else + echo "[entrypoint] ⚠ ${cert_name} — НЕ в ca-certificates.crt. Добавляем напрямую..." + cat "$pem" >> /etc/ssl/certs/ca-certificates.crt + echo "[entrypoint] ➕ Добавлен напрямую в ca-certificates.crt" + fi + + # Создаём хеш-симлинку в /etc/ssl/certs/ + local hash_val + hash_val="$(openssl x509 -in "$pem" -noout -hash 2>/dev/null || true)" + if [ -n "$hash_val" ]; then + local link_path="/etc/ssl/certs/${hash_val}.0" + if [ ! -L "$link_path" ]; then + ln -sf "$pem" "$link_path" + echo "[entrypoint] 🔗 Создана симлинка: ${hash_val}.0 → ${cert_name}" + fi + fi + done + + # Проверяем итоговое количество + local total_in_bundle + total_in_bundle="$(awk -v cmd='openssl x509 -noout -subject' '/BEGIN/{close(cmd)};{print | cmd}' \ + < /etc/ssl/certs/ca-certificates.crt 2>/dev/null | wc -l || echo "0")" + echo "[entrypoint] ✅ Всего сертификатов в bundle: ${total_in_bundle}" + + rm -rf "$work_dir" + echo "[entrypoint] === Установка LLM CA-сертификатов завершена ===" +} +# #endregion docker.backend.entrypoint.install_llm_ca_certs + # ====================================================================== # bootstrap_admin — создание admin пользователя # ====================================================================== @@ -149,6 +286,7 @@ install_playwright() { # ====================================================================== install_certificates +install_llm_ca_certs install_playwright # Check DB state: empty, legacy (tables exist but no alembic), or managed