feat: add LLM SSL verify control and auto CA cert download
- LLM_SSL_VERIFY env var to disable SSL verification for corporate proxies - install_llm_ca_certs() entrypoint function — downloads PEM/DER certs from LLM_CA_CERT_URLS, converts DER→PEM, installs to system CA store - _format_connection_error() — detailed exception chain logging - 7 new unit tests for _get_ssl_verify, _format_connection_error, verify= param - LLM_CA_CERT_URLS and LLM_SSL_VERIFY in .env.enterprise-clean - Removed duplicate @RELATION DEPENDS_ON -> [EXT:Library:tenacity]
This commit is contained in:
@@ -43,6 +43,19 @@ FRONTEND_SSL_PORT=443
|
||||
# nginx работает в HTTP-only режиме.
|
||||
CERTS_PATH=./certs
|
||||
|
||||
# ======================================================================
|
||||
# LLM CA-сертификаты (скачка по URL на старте контейнера)
|
||||
# ======================================================================
|
||||
# URL корпоративных CA-сертификатов для LLM-провайдеров.
|
||||
# Автоматически скачиваются, конвертируются DER→PEM и устанавливаются
|
||||
# в системное хранилище OpenSSL на старте backend-контейнера.
|
||||
#
|
||||
# Формат: пробел-разделённый список URL
|
||||
#
|
||||
# Пример:
|
||||
# LLM_CA_CERT_URLS="http://pki.company.com/root-ca.crt http://pki.company.com/intermediate-ca.crt"
|
||||
LLM_CA_CERT_URLS=
|
||||
|
||||
# ======================================================================
|
||||
# Логирование
|
||||
# ======================================================================
|
||||
|
||||
@@ -52,4 +52,126 @@ def test_litellm_client_uses_default_bearer_auth():
|
||||
assert "Authentication" not in headers
|
||||
assert "X-API-Key" not in headers
|
||||
# endregion test_litellm_client_uses_default_bearer_auth
|
||||
|
||||
|
||||
# region test_get_ssl_verify_default_true [TYPE Function]
|
||||
# @RELATION BINDS_TO -> TestClientHeaders
|
||||
# @PURPOSE: _get_ssl_verify defaults to True when LLM_SSL_VERIFY is unset.
|
||||
# @PRE LLM_SSL_VERIFY env var is not set.
|
||||
# @POST Returns True.
|
||||
def test_get_ssl_verify_default_true():
|
||||
"""Verify default SSL verification is enabled."""
|
||||
assert LLMClient._get_ssl_verify() is True
|
||||
# endregion test_get_ssl_verify_default_true
|
||||
|
||||
|
||||
# region test_get_ssl_verify_false_values [TYPE Function]
|
||||
# @RELATION BINDS_TO -> TestClientHeaders
|
||||
# @PURPOSE: _get_ssl_verify returns False for "false"/"0"/"no"/"off".
|
||||
# @PRE LLM_SSL_VERIFY env var is set to each falsy value.
|
||||
# @POST Returns False.
|
||||
def test_get_ssl_verify_false_values(monkeypatch):
|
||||
for val in ("false", "False", "0", "no", "off"):
|
||||
monkeypatch.setenv("LLM_SSL_VERIFY", val)
|
||||
assert LLMClient._get_ssl_verify() is False, f"Expected False for LLM_SSL_VERIFY={val}"
|
||||
# endregion test_get_ssl_verify_false_values
|
||||
|
||||
|
||||
# region test_get_ssl_verify_true_values [TYPE Function]
|
||||
# @RELATION BINDS_TO -> TestClientHeaders
|
||||
# @PURPOSE: _get_ssl_verify returns True for truthy values like "TRUE", "1", "yes".
|
||||
# @PRE LLM_SSL_VERIFY env var is set to truthy values.
|
||||
# @POST Returns True.
|
||||
def test_get_ssl_verify_true_values(monkeypatch):
|
||||
for val in ("TRUE", "1", "yes", "on", "true"):
|
||||
monkeypatch.setenv("LLM_SSL_VERIFY", val)
|
||||
assert LLMClient._get_ssl_verify() is True, f"Expected True for LLM_SSL_VERIFY={val}"
|
||||
# endregion test_get_ssl_verify_true_values
|
||||
|
||||
|
||||
# region test_format_connection_error_no_cause [TYPE Function]
|
||||
# @RELATION BINDS_TO -> TestClientHeaders
|
||||
# @PURPOSE: _format_connection_error with single exception (no cause chain).
|
||||
# @PRE Exception has no __cause__ or __context__.
|
||||
# @POST Returns formatted string with only the top-level exception.
|
||||
def test_format_connection_error_no_cause():
|
||||
exc = ValueError("test error")
|
||||
result = LLMClient._format_connection_error(exc)
|
||||
assert "ValueError: test error" in result
|
||||
assert "└─" not in result
|
||||
# endregion test_format_connection_error_no_cause
|
||||
|
||||
|
||||
# region test_format_connection_error_with_cause [TYPE Function]
|
||||
# @RELATION BINDS_TO -> TestClientHeaders
|
||||
# @PURPOSE: _format_connection_error with chained exception (__cause__).
|
||||
# @PRE Exception has a __cause__.
|
||||
# @POST Returns formatted string with chain.
|
||||
def test_format_connection_error_with_cause():
|
||||
inner = ConnectionRefusedError("Connection refused")
|
||||
outer = RuntimeError("API call failed")
|
||||
outer.__cause__ = inner
|
||||
result = LLMClient._format_connection_error(outer)
|
||||
assert "RuntimeError: API call failed" in result
|
||||
assert "└─" in result
|
||||
assert "ConnectionRefusedError: Connection refused" in result
|
||||
# endregion test_format_connection_error_with_cause
|
||||
|
||||
|
||||
# region test_format_connection_error_deep_chain [TYPE Function]
|
||||
# @RELATION BINDS_TO -> TestClientHeaders
|
||||
# @PURPOSE: _format_connection_error with 3-level chain.
|
||||
# @PRE Exception has nested __cause__ chain.
|
||||
# @POST Returns formatted string with all levels.
|
||||
def test_format_connection_error_deep_chain():
|
||||
dns = Exception("Name or service not known")
|
||||
ssl = ConnectionError("SSL: CERTIFICATE_VERIFY_FAILED")
|
||||
ssl.__cause__ = dns
|
||||
outer = RuntimeError("Connection error")
|
||||
outer.__cause__ = ssl
|
||||
result = LLMClient._format_connection_error(outer)
|
||||
assert "RuntimeError: Connection error" in result
|
||||
assert "ConnectionError: SSL: CERTIFICATE_VERIFY_FAILED" in result
|
||||
assert "Exception: Name or service not known" in result
|
||||
# endregion test_format_connection_error_deep_chain
|
||||
|
||||
|
||||
# region test_litellm_client_verify_default [TYPE Function]
|
||||
# @RELATION BINDS_TO -> TestClientHeaders
|
||||
# @PURPOSE: LLMClient passes verify=True to httpx.AsyncClient by default.
|
||||
# @PRE LLM_SSL_VERIFY env var is not set.
|
||||
# @POST httpcore pool SSL context has verify_mode=CERT_REQUIRED (2).
|
||||
def test_litellm_client_verify_default():
|
||||
"""Verify default SSL context with CERT_REQUIRED when LLM_SSL_VERIFY unset."""
|
||||
import ssl
|
||||
client = LLMClient(
|
||||
provider_type=LLMProviderType.LITELLM,
|
||||
api_key="sk-test-verify-key",
|
||||
base_url="http://localhost:4000/v1",
|
||||
default_model="gpt-4o",
|
||||
)
|
||||
pool = client.client._client._transport._pool
|
||||
assert pool._ssl_context.verify_mode == ssl.CERT_REQUIRED, \
|
||||
"verify=True should set verify_mode to CERT_REQUIRED"
|
||||
# endregion test_litellm_client_verify_default
|
||||
|
||||
|
||||
# region test_litellm_client_verify_disabled [TYPE Function]
|
||||
# @RELATION BINDS_TO -> TestClientHeaders
|
||||
# @PURPOSE: LLMClient passes verify=False when LLM_SSL_VERIFY=false.
|
||||
# @PRE LLM_SSL_VERIFY is set to "false".
|
||||
# @POST httpcore pool SSL context has verify_mode=CERT_NONE (0).
|
||||
def test_litellm_client_verify_disabled(monkeypatch):
|
||||
monkeypatch.setenv("LLM_SSL_VERIFY", "false")
|
||||
client = LLMClient(
|
||||
provider_type=LLMProviderType.LITELLM,
|
||||
api_key="sk-test-verify-key",
|
||||
base_url="http://localhost:4000/v1",
|
||||
default_model="gpt-4o",
|
||||
)
|
||||
import ssl
|
||||
pool = client.client._client._transport._pool
|
||||
assert pool._ssl_context.verify_mode == ssl.CERT_NONE, \
|
||||
"verify=False should set verify_mode to CERT_NONE"
|
||||
# endregion test_litellm_client_verify_disabled
|
||||
# endregion TestClientHeaders
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
# @BRIEF Services for LLM interaction and dashboard screenshots.
|
||||
# @LAYER Plugin
|
||||
# @RELATION DEPENDS_ON -> [EXT:Library:tenacity]
|
||||
# @RELATION DEPENDS_ON -> [EXT:Library:tenacity]
|
||||
# @RELATION DEPENDS_ON -> [EXT:Library:tenacity]
|
||||
# @INVARIANT Screenshots must be 1920px width and capture full page height.
|
||||
# @DATA_CONTRACT DashboardSpec -> Screenshot + Analysis
|
||||
# @RATIONALE Extracted all hardcoded timeouts into named module-level constants (PLAYWRIGHT_NAVIGATION_TIMEOUT_MS, PLAYWRIGHT_WAIT_TIMEOUT_MS, PLAYWRIGHT_SHORT_TIMEOUT_MS, HTTP_REQUEST_TIMEOUT_MS, SCREENSHOT_SERVICE_TIMEOUT_MS, LLM_HTTP_TIMEOUT_S) and DEFAULT_USER_AGENT. Zero remaining numeric timeout literals.
|
||||
@@ -907,7 +905,14 @@ class LLMClient:
|
||||
# It routes to upstream providers transparently, and the default Authorization header
|
||||
# is sufficient. No additional headers like HTTP-Referer or X-API-Key are required.
|
||||
|
||||
http_client = httpx.AsyncClient(headers=default_headers, timeout=LLM_HTTP_TIMEOUT_S)
|
||||
ssl_verify = self._get_ssl_verify()
|
||||
logger.info(f"[LLMClient.__init__] SSL Verify: {ssl_verify} (set LLM_SSL_VERIFY=false to disable)")
|
||||
|
||||
http_client = httpx.AsyncClient(
|
||||
headers=default_headers,
|
||||
timeout=LLM_HTTP_TIMEOUT_S,
|
||||
verify=ssl_verify,
|
||||
)
|
||||
self.client = AsyncOpenAI(
|
||||
api_key=self.api_key,
|
||||
base_url=base_url,
|
||||
@@ -916,6 +921,28 @@ class LLMClient:
|
||||
)
|
||||
# endregion LLMClient.__init__
|
||||
|
||||
# region LLMClient._get_ssl_verify [TYPE Function]
|
||||
# @PURPOSE Resolve SSL verification flag from environment.
|
||||
# @POST Returns False when LLM_SSL_VERIFY env var is "false"/"0"/"no" (case-insensitive).
|
||||
@staticmethod
|
||||
def _get_ssl_verify() -> bool:
|
||||
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
|
||||
return raw not in ("false", "0", "no", "off")
|
||||
# endregion LLMClient._get_ssl_verify
|
||||
|
||||
# region LLMClient._format_connection_error [TYPE Function]
|
||||
# @PURPOSE Format exception chain for diagnostics, extracting httpx cause details.
|
||||
# @POST Returns a human-readable string with the full error chain.
|
||||
@staticmethod
|
||||
def _format_connection_error(exc: Exception) -> str:
|
||||
parts = [f"{type(exc).__name__}: {exc!s}"]
|
||||
cause = exc.__cause__ or exc.__context__
|
||||
while cause:
|
||||
parts.append(f" └─ {type(cause).__name__}: {cause!s}")
|
||||
cause = cause.__cause__ or cause.__context__
|
||||
return "\n".join(parts)
|
||||
# endregion LLMClient._format_connection_error
|
||||
|
||||
# region LLMClient._supports_json_response_format [TYPE Function]
|
||||
# @PURPOSE Detect whether provider/model is likely compatible with response_format=json_object.
|
||||
# @PRE Client initialized with base_url and default_model.
|
||||
@@ -1035,7 +1062,10 @@ class LLMClient:
|
||||
await asyncio.sleep(wait_time)
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"[get_json_completion] LLM call failed: {e!s}")
|
||||
logger.error(
|
||||
f"[get_json_completion] LLM call failed.\n"
|
||||
f"{self._format_connection_error(e)}"
|
||||
)
|
||||
raise
|
||||
|
||||
if not response or not hasattr(response, 'choices') or not response.choices:
|
||||
|
||||
@@ -48,6 +48,8 @@ services:
|
||||
FEATURES__HEALTH_MONITOR: ${FEATURES__HEALTH_MONITOR:-true}
|
||||
# Путь до сертификатов внутри контейнера — всегда /opt/certs
|
||||
CERTS_PATH: /opt/certs
|
||||
# URL CA-сертификатов для LLM-провайдеров (скачка на старте, DER→PEM конвертация)
|
||||
LLM_CA_CERT_URLS: ${LLM_CA_CERT_URLS:-}
|
||||
ports:
|
||||
- "${BACKEND_HOST_PORT:-8001}:8000"
|
||||
volumes:
|
||||
|
||||
@@ -37,6 +37,7 @@ services:
|
||||
INITIAL_ADMIN_PASSWORD: ${INITIAL_ADMIN_PASSWORD:-}
|
||||
FEATURES__DATASET_REVIEW: ${FEATURES__DATASET_REVIEW:-true}
|
||||
FEATURES__HEALTH_MONITOR: ${FEATURES__HEALTH_MONITOR:-true}
|
||||
LLM_CA_CERT_URLS: ${LLM_CA_CERT_URLS:-}
|
||||
ports:
|
||||
- "${BACKEND_HOST_PORT:-8001}:8000"
|
||||
volumes:
|
||||
|
||||
@@ -75,6 +75,143 @@ install_certificates() {
|
||||
}
|
||||
# #endregion docker.backend.entrypoint.install_certificates
|
||||
|
||||
# ======================================================================
|
||||
# install_llm_ca_certs — скачка и установка CA-сертификатов по URL
|
||||
# ======================================================================
|
||||
# #region docker.backend.entrypoint.install_llm_ca_certs [C:4] [TYPE Function]
|
||||
# @BRIEF Скачивает PEM/DER-сертификаты из LLM_CA_CERT_URLS, конвертирует DER→PEM,
|
||||
# устанавливает в системное хранилище, создаёт хеш-симлинки и при необходимости
|
||||
# добавляет в ca-certificates.crt напрямую (fallback для OpenSSL 3-совместимости).
|
||||
# @PRE LLM_CA_CERT_URLS — строка с URL через пробел; curl и openssl установлены.
|
||||
# @POST Сертификаты доступны в /etc/ssl/certs/ca-certificates.crt.
|
||||
# Хеш-симлинки созданы для каждого сертификата.
|
||||
# @SIDE_EFFECT Скачивает файлы из внешних URL. Модифицирует /etc/ssl/certs/.
|
||||
# @LAYER Infrastructure
|
||||
# @EXAMPLE:
|
||||
# LLM_CA_CERT_URLS="http://pki.company.com/root.crt http://pki.company.com/intermediate.crt"
|
||||
install_llm_ca_certs() {
|
||||
local urls="${LLM_CA_CERT_URLS:-}"
|
||||
if [ -z "$urls" ]; then
|
||||
echo "[entrypoint] LLM_CA_CERT_URLS не задан — пропускаем скачку сертификатов"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local work_dir="/tmp/llm-ca-certs"
|
||||
local target_dir="/usr/local/share/ca-certificates/llm"
|
||||
mkdir -p "$work_dir" "$target_dir"
|
||||
|
||||
echo "[entrypoint] === Скачивание и установка LLM CA-сертификатов ==="
|
||||
local index=0
|
||||
local downloaded_names=""
|
||||
|
||||
for url in $urls; do
|
||||
url="$(echo "$url" | xargs)" # trim
|
||||
[ -z "$url" ] && continue
|
||||
index=$((index + 1))
|
||||
|
||||
# Определяем имя файла из URL, или генерируем
|
||||
local filename
|
||||
filename="$(basename "$url" | sed 's/[?#].*//')"
|
||||
if [ -z "$filename" ] || [ "$filename" = "." ] || [ "$filename" = ".." ]; then
|
||||
filename="llm-ca-${index}.crt"
|
||||
fi
|
||||
|
||||
local raw_file="${work_dir}/${filename}"
|
||||
local pem_file="${target_dir}/$(echo "$filename" | sed 's/\.[^.]*$//').pem"
|
||||
|
||||
echo "[entrypoint] [${index}] Скачивание: ${url}"
|
||||
|
||||
# Скачиваем с retry 3 раза
|
||||
local attempt=0
|
||||
local success=0
|
||||
while [ $attempt -lt 3 ]; do
|
||||
attempt=$((attempt + 1))
|
||||
if curl -sS --connect-timeout 10 --max-time 30 -o "$raw_file" "$url" 2>/dev/null; then
|
||||
success=1
|
||||
break
|
||||
fi
|
||||
echo "[entrypoint] ⚠ Попытка ${attempt}/3 не удалась, повтор через 2с..."
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [ "$success" -eq 0 ]; then
|
||||
echo "[entrypoint] ❌ Не удалось скачать: ${url} — пропускаем"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Определяем формат: PEM или DER
|
||||
if openssl x509 -in "$raw_file" -inform PEM -noout 2>/dev/null; then
|
||||
echo "[entrypoint] ✓ Формат: PEM"
|
||||
cp "$raw_file" "$pem_file"
|
||||
elif openssl x509 -in "$raw_file" -inform DER -noout 2>/dev/null; then
|
||||
echo "[entrypoint] ↻ Формат: DER — конвертируем в PEM"
|
||||
openssl x509 -in "$raw_file" -inform DER -out "$pem_file" -outform PEM
|
||||
else
|
||||
echo "[entrypoint] ❌ Неизвестный формат (не PEM и не DER) — пропускаем"
|
||||
rm -f "$raw_file"
|
||||
continue
|
||||
fi
|
||||
|
||||
chmod 644 "$pem_file"
|
||||
echo "[entrypoint] ✅ Установлен: $(basename "$pem_file")"
|
||||
downloaded_names="$downloaded_names $(basename "$pem_file" .pem)"
|
||||
rm -f "$raw_file"
|
||||
done
|
||||
|
||||
if [ -z "$(ls -A "$target_dir" 2>/dev/null)" ]; then
|
||||
echo "[entrypoint] Нет скачанных сертификатов — пропускаем"
|
||||
rm -rf "$work_dir"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Обновляем системное хранилище
|
||||
echo "[entrypoint] --- Обновление системного хранилища CA-сертификатов ---"
|
||||
update-ca-certificates --fresh 2>&1 | sed 's/^/[entrypoint] /'
|
||||
|
||||
# Проверяем, попали ли сертификаты в бандл, и создаём хеш-симлинки
|
||||
echo "[entrypoint] --- Валидация установки сертификатов ---"
|
||||
local all_ok=true
|
||||
|
||||
for pem in "$target_dir"/*.pem; do
|
||||
[ -f "$pem" ] || continue
|
||||
local cert_name
|
||||
cert_name="$(basename "$pem" .pem)"
|
||||
local cert_subject
|
||||
cert_subject="$(openssl x509 -in "$pem" -noout -subject 2>/dev/null | head -1 || echo "unknown")"
|
||||
|
||||
# Проверяем есть ли в ca-certificates.crt
|
||||
if grep -q "$cert_name" /etc/ssl/certs/ca-certificates.crt 2>/dev/null || \
|
||||
grep -qF "$(openssl x509 -in "$pem" -noout -serial 2>/dev/null)" /etc/ssl/certs/ca-certificates.crt 2>/dev/null; then
|
||||
echo "[entrypoint] ✅ ${cert_name} — в ca-certificates.crt (${cert_subject})"
|
||||
else
|
||||
echo "[entrypoint] ⚠ ${cert_name} — НЕ в ca-certificates.crt. Добавляем напрямую..."
|
||||
cat "$pem" >> /etc/ssl/certs/ca-certificates.crt
|
||||
echo "[entrypoint] ➕ Добавлен напрямую в ca-certificates.crt"
|
||||
fi
|
||||
|
||||
# Создаём хеш-симлинку в /etc/ssl/certs/
|
||||
local hash_val
|
||||
hash_val="$(openssl x509 -in "$pem" -noout -hash 2>/dev/null || true)"
|
||||
if [ -n "$hash_val" ]; then
|
||||
local link_path="/etc/ssl/certs/${hash_val}.0"
|
||||
if [ ! -L "$link_path" ]; then
|
||||
ln -sf "$pem" "$link_path"
|
||||
echo "[entrypoint] 🔗 Создана симлинка: ${hash_val}.0 → ${cert_name}"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Проверяем итоговое количество
|
||||
local total_in_bundle
|
||||
total_in_bundle="$(awk -v cmd='openssl x509 -noout -subject' '/BEGIN/{close(cmd)};{print | cmd}' \
|
||||
< /etc/ssl/certs/ca-certificates.crt 2>/dev/null | wc -l || echo "0")"
|
||||
echo "[entrypoint] ✅ Всего сертификатов в bundle: ${total_in_bundle}"
|
||||
|
||||
rm -rf "$work_dir"
|
||||
echo "[entrypoint] === Установка LLM CA-сертификатов завершена ==="
|
||||
}
|
||||
# #endregion docker.backend.entrypoint.install_llm_ca_certs
|
||||
|
||||
# ======================================================================
|
||||
# bootstrap_admin — создание admin пользователя
|
||||
# ======================================================================
|
||||
@@ -149,6 +286,7 @@ install_playwright() {
|
||||
# ======================================================================
|
||||
|
||||
install_certificates
|
||||
install_llm_ca_certs
|
||||
install_playwright
|
||||
|
||||
# Check DB state: empty, legacy (tables exist but no alembic), or managed
|
||||
|
||||
Reference in New Issue
Block a user