feat: add LLM SSL verify control and auto CA cert download

- LLM_SSL_VERIFY env var to disable SSL verification for corporate proxies
- install_llm_ca_certs() entrypoint function — downloads PEM/DER certs
  from LLM_CA_CERT_URLS, converts DER→PEM, installs to system CA store
- _format_connection_error() — detailed exception chain logging
- 7 new unit tests for _get_ssl_verify, _format_connection_error, verify= param
- LLM_CA_CERT_URLS and LLM_SSL_VERIFY in .env.enterprise-clean
- Removed duplicate @RELATION DEPENDS_ON -> [EXT:Library:tenacity]
This commit is contained in:
2026-05-27 14:44:46 +03:00
parent 55c7e323c4
commit 7ffddff2a6
6 changed files with 310 additions and 4 deletions

View File

@@ -43,6 +43,19 @@ FRONTEND_SSL_PORT=443
# nginx работает в HTTP-only режиме. # nginx работает в HTTP-only режиме.
CERTS_PATH=./certs CERTS_PATH=./certs
# ======================================================================
# LLM CA-сертификаты (скачка по URL на старте контейнера)
# ======================================================================
# URL корпоративных CA-сертификатов для LLM-провайдеров.
# Автоматически скачиваются, конвертируются DER→PEM и устанавливаются
# в системное хранилище OpenSSL на старте backend-контейнера.
#
# Формат: пробел-разделённый список URL
#
# Пример:
# LLM_CA_CERT_URLS="http://pki.company.com/root-ca.crt http://pki.company.com/intermediate-ca.crt"
LLM_CA_CERT_URLS=
# ====================================================================== # ======================================================================
# Логирование # Логирование
# ====================================================================== # ======================================================================

View File

@@ -52,4 +52,126 @@ def test_litellm_client_uses_default_bearer_auth():
assert "Authentication" not in headers assert "Authentication" not in headers
assert "X-API-Key" not in headers assert "X-API-Key" not in headers
# endregion test_litellm_client_uses_default_bearer_auth # endregion test_litellm_client_uses_default_bearer_auth
# region test_get_ssl_verify_default_true [TYPE Function]
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: _get_ssl_verify defaults to True when LLM_SSL_VERIFY is unset.
# @PRE LLM_SSL_VERIFY env var is not set.
# @POST Returns True.
def test_get_ssl_verify_default_true():
"""Verify default SSL verification is enabled."""
assert LLMClient._get_ssl_verify() is True
# endregion test_get_ssl_verify_default_true
# region test_get_ssl_verify_false_values [TYPE Function]
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: _get_ssl_verify returns False for "false"/"0"/"no"/"off".
# @PRE LLM_SSL_VERIFY env var is set to each falsy value.
# @POST Returns False.
def test_get_ssl_verify_false_values(monkeypatch):
for val in ("false", "False", "0", "no", "off"):
monkeypatch.setenv("LLM_SSL_VERIFY", val)
assert LLMClient._get_ssl_verify() is False, f"Expected False for LLM_SSL_VERIFY={val}"
# endregion test_get_ssl_verify_false_values
# region test_get_ssl_verify_true_values [TYPE Function]
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: _get_ssl_verify returns True for truthy values like "TRUE", "1", "yes".
# @PRE LLM_SSL_VERIFY env var is set to truthy values.
# @POST Returns True.
def test_get_ssl_verify_true_values(monkeypatch):
for val in ("TRUE", "1", "yes", "on", "true"):
monkeypatch.setenv("LLM_SSL_VERIFY", val)
assert LLMClient._get_ssl_verify() is True, f"Expected True for LLM_SSL_VERIFY={val}"
# endregion test_get_ssl_verify_true_values
# region test_format_connection_error_no_cause [TYPE Function]
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: _format_connection_error with single exception (no cause chain).
# @PRE Exception has no __cause__ or __context__.
# @POST Returns formatted string with only the top-level exception.
def test_format_connection_error_no_cause():
exc = ValueError("test error")
result = LLMClient._format_connection_error(exc)
assert "ValueError: test error" in result
assert "└─" not in result
# endregion test_format_connection_error_no_cause
# region test_format_connection_error_with_cause [TYPE Function]
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: _format_connection_error with chained exception (__cause__).
# @PRE Exception has a __cause__.
# @POST Returns formatted string with chain.
def test_format_connection_error_with_cause():
inner = ConnectionRefusedError("Connection refused")
outer = RuntimeError("API call failed")
outer.__cause__ = inner
result = LLMClient._format_connection_error(outer)
assert "RuntimeError: API call failed" in result
assert "└─" in result
assert "ConnectionRefusedError: Connection refused" in result
# endregion test_format_connection_error_with_cause
# region test_format_connection_error_deep_chain [TYPE Function]
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: _format_connection_error with 3-level chain.
# @PRE Exception has nested __cause__ chain.
# @POST Returns formatted string with all levels.
def test_format_connection_error_deep_chain():
dns = Exception("Name or service not known")
ssl = ConnectionError("SSL: CERTIFICATE_VERIFY_FAILED")
ssl.__cause__ = dns
outer = RuntimeError("Connection error")
outer.__cause__ = ssl
result = LLMClient._format_connection_error(outer)
assert "RuntimeError: Connection error" in result
assert "ConnectionError: SSL: CERTIFICATE_VERIFY_FAILED" in result
assert "Exception: Name or service not known" in result
# endregion test_format_connection_error_deep_chain
# region test_litellm_client_verify_default [TYPE Function]
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: LLMClient passes verify=True to httpx.AsyncClient by default.
# @PRE LLM_SSL_VERIFY env var is not set.
# @POST httpcore pool SSL context has verify_mode=CERT_REQUIRED (2).
def test_litellm_client_verify_default():
"""Verify default SSL context with CERT_REQUIRED when LLM_SSL_VERIFY unset."""
import ssl
client = LLMClient(
provider_type=LLMProviderType.LITELLM,
api_key="sk-test-verify-key",
base_url="http://localhost:4000/v1",
default_model="gpt-4o",
)
pool = client.client._client._transport._pool
assert pool._ssl_context.verify_mode == ssl.CERT_REQUIRED, \
"verify=True should set verify_mode to CERT_REQUIRED"
# endregion test_litellm_client_verify_default
# region test_litellm_client_verify_disabled [TYPE Function]
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: LLMClient passes verify=False when LLM_SSL_VERIFY=false.
# @PRE LLM_SSL_VERIFY is set to "false".
# @POST httpcore pool SSL context has verify_mode=CERT_NONE (0).
def test_litellm_client_verify_disabled(monkeypatch):
monkeypatch.setenv("LLM_SSL_VERIFY", "false")
client = LLMClient(
provider_type=LLMProviderType.LITELLM,
api_key="sk-test-verify-key",
base_url="http://localhost:4000/v1",
default_model="gpt-4o",
)
import ssl
pool = client.client._client._transport._pool
assert pool._ssl_context.verify_mode == ssl.CERT_NONE, \
"verify=False should set verify_mode to CERT_NONE"
# endregion test_litellm_client_verify_disabled
# endregion TestClientHeaders # endregion TestClientHeaders

View File

@@ -2,8 +2,6 @@
# @BRIEF Services for LLM interaction and dashboard screenshots. # @BRIEF Services for LLM interaction and dashboard screenshots.
# @LAYER Plugin # @LAYER Plugin
# @RELATION DEPENDS_ON -> [EXT:Library:tenacity] # @RELATION DEPENDS_ON -> [EXT:Library:tenacity]
# @RELATION DEPENDS_ON -> [EXT:Library:tenacity]
# @RELATION DEPENDS_ON -> [EXT:Library:tenacity]
# @INVARIANT Screenshots must be 1920px width and capture full page height. # @INVARIANT Screenshots must be 1920px width and capture full page height.
# @DATA_CONTRACT DashboardSpec -> Screenshot + Analysis # @DATA_CONTRACT DashboardSpec -> Screenshot + Analysis
# @RATIONALE Extracted all hardcoded timeouts into named module-level constants (PLAYWRIGHT_NAVIGATION_TIMEOUT_MS, PLAYWRIGHT_WAIT_TIMEOUT_MS, PLAYWRIGHT_SHORT_TIMEOUT_MS, HTTP_REQUEST_TIMEOUT_MS, SCREENSHOT_SERVICE_TIMEOUT_MS, LLM_HTTP_TIMEOUT_S) and DEFAULT_USER_AGENT. Zero remaining numeric timeout literals. # @RATIONALE Extracted all hardcoded timeouts into named module-level constants (PLAYWRIGHT_NAVIGATION_TIMEOUT_MS, PLAYWRIGHT_WAIT_TIMEOUT_MS, PLAYWRIGHT_SHORT_TIMEOUT_MS, HTTP_REQUEST_TIMEOUT_MS, SCREENSHOT_SERVICE_TIMEOUT_MS, LLM_HTTP_TIMEOUT_S) and DEFAULT_USER_AGENT. Zero remaining numeric timeout literals.
@@ -907,7 +905,14 @@ class LLMClient:
# It routes to upstream providers transparently, and the default Authorization header # It routes to upstream providers transparently, and the default Authorization header
# is sufficient. No additional headers like HTTP-Referer or X-API-Key are required. # is sufficient. No additional headers like HTTP-Referer or X-API-Key are required.
http_client = httpx.AsyncClient(headers=default_headers, timeout=LLM_HTTP_TIMEOUT_S) ssl_verify = self._get_ssl_verify()
logger.info(f"[LLMClient.__init__] SSL Verify: {ssl_verify} (set LLM_SSL_VERIFY=false to disable)")
http_client = httpx.AsyncClient(
headers=default_headers,
timeout=LLM_HTTP_TIMEOUT_S,
verify=ssl_verify,
)
self.client = AsyncOpenAI( self.client = AsyncOpenAI(
api_key=self.api_key, api_key=self.api_key,
base_url=base_url, base_url=base_url,
@@ -916,6 +921,28 @@ class LLMClient:
) )
# endregion LLMClient.__init__ # endregion LLMClient.__init__
# region LLMClient._get_ssl_verify [TYPE Function]
# @PURPOSE Resolve SSL verification flag from environment.
# @POST Returns False when LLM_SSL_VERIFY env var is "false"/"0"/"no" (case-insensitive).
@staticmethod
def _get_ssl_verify() -> bool:
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
return raw not in ("false", "0", "no", "off")
# endregion LLMClient._get_ssl_verify
# region LLMClient._format_connection_error [TYPE Function]
# @PURPOSE Format exception chain for diagnostics, extracting httpx cause details.
# @POST Returns a human-readable string with the full error chain.
@staticmethod
def _format_connection_error(exc: Exception) -> str:
parts = [f"{type(exc).__name__}: {exc!s}"]
cause = exc.__cause__ or exc.__context__
while cause:
parts.append(f" └─ {type(cause).__name__}: {cause!s}")
cause = cause.__cause__ or cause.__context__
return "\n".join(parts)
# endregion LLMClient._format_connection_error
# region LLMClient._supports_json_response_format [TYPE Function] # region LLMClient._supports_json_response_format [TYPE Function]
# @PURPOSE Detect whether provider/model is likely compatible with response_format=json_object. # @PURPOSE Detect whether provider/model is likely compatible with response_format=json_object.
# @PRE Client initialized with base_url and default_model. # @PRE Client initialized with base_url and default_model.
@@ -1035,7 +1062,10 @@ class LLMClient:
await asyncio.sleep(wait_time) await asyncio.sleep(wait_time)
raise raise
except Exception as e: except Exception as e:
logger.error(f"[get_json_completion] LLM call failed: {e!s}") logger.error(
f"[get_json_completion] LLM call failed.\n"
f"{self._format_connection_error(e)}"
)
raise raise
if not response or not hasattr(response, 'choices') or not response.choices: if not response or not hasattr(response, 'choices') or not response.choices:

View File

@@ -48,6 +48,8 @@ services:
FEATURES__HEALTH_MONITOR: ${FEATURES__HEALTH_MONITOR:-true} FEATURES__HEALTH_MONITOR: ${FEATURES__HEALTH_MONITOR:-true}
# Путь до сертификатов внутри контейнера — всегда /opt/certs # Путь до сертификатов внутри контейнера — всегда /opt/certs
CERTS_PATH: /opt/certs CERTS_PATH: /opt/certs
# URL CA-сертификатов для LLM-провайдеров (скачка на старте, DER→PEM конвертация)
LLM_CA_CERT_URLS: ${LLM_CA_CERT_URLS:-}
ports: ports:
- "${BACKEND_HOST_PORT:-8001}:8000" - "${BACKEND_HOST_PORT:-8001}:8000"
volumes: volumes:

View File

@@ -37,6 +37,7 @@ services:
INITIAL_ADMIN_PASSWORD: ${INITIAL_ADMIN_PASSWORD:-} INITIAL_ADMIN_PASSWORD: ${INITIAL_ADMIN_PASSWORD:-}
FEATURES__DATASET_REVIEW: ${FEATURES__DATASET_REVIEW:-true} FEATURES__DATASET_REVIEW: ${FEATURES__DATASET_REVIEW:-true}
FEATURES__HEALTH_MONITOR: ${FEATURES__HEALTH_MONITOR:-true} FEATURES__HEALTH_MONITOR: ${FEATURES__HEALTH_MONITOR:-true}
LLM_CA_CERT_URLS: ${LLM_CA_CERT_URLS:-}
ports: ports:
- "${BACKEND_HOST_PORT:-8001}:8000" - "${BACKEND_HOST_PORT:-8001}:8000"
volumes: volumes:

View File

@@ -75,6 +75,143 @@ install_certificates() {
} }
# #endregion docker.backend.entrypoint.install_certificates # #endregion docker.backend.entrypoint.install_certificates
# ======================================================================
# install_llm_ca_certs — скачка и установка CA-сертификатов по URL
# ======================================================================
# #region docker.backend.entrypoint.install_llm_ca_certs [C:4] [TYPE Function]
# @BRIEF Скачивает PEM/DER-сертификаты из LLM_CA_CERT_URLS, конвертирует DER→PEM,
# устанавливает в системное хранилище, создаёт хеш-симлинки и при необходимости
# добавляет в ca-certificates.crt напрямую (fallback для OpenSSL 3-совместимости).
# @PRE LLM_CA_CERT_URLS — строка с URL через пробел; curl и openssl установлены.
# @POST Сертификаты доступны в /etc/ssl/certs/ca-certificates.crt.
# Хеш-симлинки созданы для каждого сертификата.
# @SIDE_EFFECT Скачивает файлы из внешних URL. Модифицирует /etc/ssl/certs/.
# @LAYER Infrastructure
# @EXAMPLE:
# LLM_CA_CERT_URLS="http://pki.company.com/root.crt http://pki.company.com/intermediate.crt"
install_llm_ca_certs() {
local urls="${LLM_CA_CERT_URLS:-}"
if [ -z "$urls" ]; then
echo "[entrypoint] LLM_CA_CERT_URLS не задан — пропускаем скачку сертификатов"
return 0
fi
local work_dir="/tmp/llm-ca-certs"
local target_dir="/usr/local/share/ca-certificates/llm"
mkdir -p "$work_dir" "$target_dir"
echo "[entrypoint] === Скачивание и установка LLM CA-сертификатов ==="
local index=0
local downloaded_names=""
for url in $urls; do
url="$(echo "$url" | xargs)" # trim
[ -z "$url" ] && continue
index=$((index + 1))
# Определяем имя файла из URL, или генерируем
local filename
filename="$(basename "$url" | sed 's/[?#].*//')"
if [ -z "$filename" ] || [ "$filename" = "." ] || [ "$filename" = ".." ]; then
filename="llm-ca-${index}.crt"
fi
local raw_file="${work_dir}/${filename}"
local pem_file="${target_dir}/$(echo "$filename" | sed 's/\.[^.]*$//').pem"
echo "[entrypoint] [${index}] Скачивание: ${url}"
# Скачиваем с retry 3 раза
local attempt=0
local success=0
while [ $attempt -lt 3 ]; do
attempt=$((attempt + 1))
if curl -sS --connect-timeout 10 --max-time 30 -o "$raw_file" "$url" 2>/dev/null; then
success=1
break
fi
echo "[entrypoint] ⚠ Попытка ${attempt}/3 не удалась, повтор через 2с..."
sleep 2
done
if [ "$success" -eq 0 ]; then
echo "[entrypoint] ❌ Не удалось скачать: ${url} — пропускаем"
continue
fi
# Определяем формат: PEM или DER
if openssl x509 -in "$raw_file" -inform PEM -noout 2>/dev/null; then
echo "[entrypoint] ✓ Формат: PEM"
cp "$raw_file" "$pem_file"
elif openssl x509 -in "$raw_file" -inform DER -noout 2>/dev/null; then
echo "[entrypoint] ↻ Формат: DER — конвертируем в PEM"
openssl x509 -in "$raw_file" -inform DER -out "$pem_file" -outform PEM
else
echo "[entrypoint] ❌ Неизвестный формат (не PEM и не DER) — пропускаем"
rm -f "$raw_file"
continue
fi
chmod 644 "$pem_file"
echo "[entrypoint] ✅ Установлен: $(basename "$pem_file")"
downloaded_names="$downloaded_names $(basename "$pem_file" .pem)"
rm -f "$raw_file"
done
if [ -z "$(ls -A "$target_dir" 2>/dev/null)" ]; then
echo "[entrypoint] Нет скачанных сертификатов — пропускаем"
rm -rf "$work_dir"
return 0
fi
# Обновляем системное хранилище
echo "[entrypoint] --- Обновление системного хранилища CA-сертификатов ---"
update-ca-certificates --fresh 2>&1 | sed 's/^/[entrypoint] /'
# Проверяем, попали ли сертификаты в бандл, и создаём хеш-симлинки
echo "[entrypoint] --- Валидация установки сертификатов ---"
local all_ok=true
for pem in "$target_dir"/*.pem; do
[ -f "$pem" ] || continue
local cert_name
cert_name="$(basename "$pem" .pem)"
local cert_subject
cert_subject="$(openssl x509 -in "$pem" -noout -subject 2>/dev/null | head -1 || echo "unknown")"
# Проверяем есть ли в ca-certificates.crt
if grep -q "$cert_name" /etc/ssl/certs/ca-certificates.crt 2>/dev/null || \
grep -qF "$(openssl x509 -in "$pem" -noout -serial 2>/dev/null)" /etc/ssl/certs/ca-certificates.crt 2>/dev/null; then
echo "[entrypoint] ✅ ${cert_name} — в ca-certificates.crt (${cert_subject})"
else
echo "[entrypoint] ⚠ ${cert_name}НЕ в ca-certificates.crt. Добавляем напрямую..."
cat "$pem" >> /etc/ssl/certs/ca-certificates.crt
echo "[entrypoint] Добавлен напрямую в ca-certificates.crt"
fi
# Создаём хеш-симлинку в /etc/ssl/certs/
local hash_val
hash_val="$(openssl x509 -in "$pem" -noout -hash 2>/dev/null || true)"
if [ -n "$hash_val" ]; then
local link_path="/etc/ssl/certs/${hash_val}.0"
if [ ! -L "$link_path" ]; then
ln -sf "$pem" "$link_path"
echo "[entrypoint] 🔗 Создана симлинка: ${hash_val}.0 → ${cert_name}"
fi
fi
done
# Проверяем итоговое количество
local total_in_bundle
total_in_bundle="$(awk -v cmd='openssl x509 -noout -subject' '/BEGIN/{close(cmd)};{print | cmd}' \
< /etc/ssl/certs/ca-certificates.crt 2>/dev/null | wc -l || echo "0")"
echo "[entrypoint] ✅ Всего сертификатов в bundle: ${total_in_bundle}"
rm -rf "$work_dir"
echo "[entrypoint] === Установка LLM CA-сертификатов завершена ==="
}
# #endregion docker.backend.entrypoint.install_llm_ca_certs
# ====================================================================== # ======================================================================
# bootstrap_admin — создание admin пользователя # bootstrap_admin — создание admin пользователя
# ====================================================================== # ======================================================================
@@ -149,6 +286,7 @@ install_playwright() {
# ====================================================================== # ======================================================================
install_certificates install_certificates
install_llm_ca_certs
install_playwright install_playwright
# Check DB state: empty, legacy (tables exist but no alembic), or managed # Check DB state: empty, legacy (tables exist but no alembic), or managed