Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3f6b06b4b8 | |||
| d7556a1486 | |||
| 1a1657317a |
@@ -54,15 +54,18 @@ def test_litellm_client_uses_default_bearer_auth():
|
||||
# endregion test_litellm_client_uses_default_bearer_auth
|
||||
|
||||
|
||||
# region test_get_ssl_verify_default_true [TYPE Function]
|
||||
# region test_get_ssl_verify_default_context [TYPE Function]
|
||||
# @RELATION BINDS_TO -> TestClientHeaders
|
||||
# @PURPOSE: _get_ssl_verify defaults to True when LLM_SSL_VERIFY is unset.
|
||||
# @PURPOSE: _get_ssl_verify returns SSLContext with CERT_REQUIRED by default.
|
||||
# @PRE LLM_SSL_VERIFY env var is not set.
|
||||
# @POST Returns True.
|
||||
def test_get_ssl_verify_default_true():
|
||||
"""Verify default SSL verification is enabled."""
|
||||
assert LLMClient._get_ssl_verify() is True
|
||||
# endregion test_get_ssl_verify_default_true
|
||||
# @POST Returns SSLContext with verify_mode=CERT_REQUIRED.
|
||||
def test_get_ssl_verify_default_context():
|
||||
"""Verify default returns SSLContext with CERT_REQUIRED."""
|
||||
import ssl
|
||||
result = LLMClient._get_ssl_verify()
|
||||
assert isinstance(result, ssl.SSLContext), "Expected SSLContext, not bool"
|
||||
assert result.verify_mode == ssl.CERT_REQUIRED
|
||||
# endregion test_get_ssl_verify_default_context
|
||||
|
||||
|
||||
# region test_get_ssl_verify_false_values [TYPE Function]
|
||||
@@ -77,16 +80,19 @@ def test_get_ssl_verify_false_values(monkeypatch):
|
||||
# endregion test_get_ssl_verify_false_values
|
||||
|
||||
|
||||
# region test_get_ssl_verify_true_values [TYPE Function]
|
||||
# region test_get_ssl_verify_true_returns_context [TYPE Function]
|
||||
# @RELATION BINDS_TO -> TestClientHeaders
|
||||
# @PURPOSE: _get_ssl_verify returns True for truthy values like "TRUE", "1", "yes".
|
||||
# @PURPOSE: _get_ssl_verify returns SSLContext for truthy env values.
|
||||
# @PRE LLM_SSL_VERIFY env var is set to truthy values.
|
||||
# @POST Returns True.
|
||||
def test_get_ssl_verify_true_values(monkeypatch):
|
||||
# @POST Returns SSLContext with CERT_REQUIRED.
|
||||
def test_get_ssl_verify_true_returns_context(monkeypatch):
|
||||
import ssl
|
||||
for val in ("TRUE", "1", "yes", "on", "true"):
|
||||
monkeypatch.setenv("LLM_SSL_VERIFY", val)
|
||||
assert LLMClient._get_ssl_verify() is True, f"Expected True for LLM_SSL_VERIFY={val}"
|
||||
# endregion test_get_ssl_verify_true_values
|
||||
result = LLMClient._get_ssl_verify()
|
||||
assert isinstance(result, ssl.SSLContext), f"Expected SSLContext for LLM_SSL_VERIFY={val}"
|
||||
assert result.verify_mode == ssl.CERT_REQUIRED
|
||||
# endregion test_get_ssl_verify_true_returns_path
|
||||
|
||||
|
||||
# region test_format_connection_error_no_cause [TYPE Function]
|
||||
|
||||
@@ -13,6 +13,7 @@ import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import ssl
|
||||
import tempfile
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
@@ -923,11 +924,25 @@ class LLMClient:
|
||||
|
||||
# region LLMClient._get_ssl_verify [TYPE Function]
|
||||
# @PURPOSE Resolve SSL verification flag from environment.
|
||||
# @POST Returns False when LLM_SSL_VERIFY env var is "false"/"0"/"no" (case-insensitive).
|
||||
# @POST Returns SSLContext with system CA bundle when enabled,
|
||||
# False when LLM_SSL_VERIFY env var is "false"/"0"/"no"/"off".
|
||||
# @RATIONALE Возвращаем ssl.SSLContext вместо True, потому что httpx по
|
||||
# умолчанию использует certifi, в котором нет корпоративных CA.
|
||||
# ssl.create_default_context(cafile=...) гарантирует, что используются
|
||||
# сертификаты из системного хранилища (/etc/ssl/certs/ca-certificates.crt),
|
||||
# куда update-ca-certificates и наш fallback добавляют корпоративные CA.
|
||||
# @REJECTED verify=<string> отвергнут — httpx 0.28.x депрекейтит строковый
|
||||
# путь в verify=, требует SSLContext.
|
||||
@staticmethod
|
||||
def _get_ssl_verify() -> bool:
|
||||
def _get_ssl_verify() -> ssl.SSLContext | bool:
|
||||
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
|
||||
return raw not in ("false", "0", "no", "off")
|
||||
if raw in ("false", "0", "no", "off"):
|
||||
return False
|
||||
ca_path = "/etc/ssl/certs/ca-certificates.crt"
|
||||
if os.path.exists(ca_path):
|
||||
return ssl.create_default_context(cafile=ca_path)
|
||||
# fallback: если системного CA нет (редко), используем дефолтный
|
||||
return ssl.create_default_context()
|
||||
# endregion LLMClient._get_ssl_verify
|
||||
|
||||
# region LLMClient._format_connection_error [TYPE Function]
|
||||
|
||||
@@ -11,12 +11,27 @@
|
||||
# @RATIONALE Extracted from LLMTranslationService (793 lines) to keep module under INV_7 limit.
|
||||
# @REJECTED Single HTTP client class — kept as module-level functions for stateless reusability.
|
||||
|
||||
import os
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from ...core.logger import logger
|
||||
|
||||
|
||||
def _get_verify() -> str | bool:
|
||||
"""Resolve SSL verify from LLM_SSL_VERIFY env var.
|
||||
|
||||
Returns:
|
||||
- Path to system CA bundle when verification enabled
|
||||
(certifi bundle doesn't include corporate CA certs)
|
||||
- False when LLM_SSL_VERIFY is set to false/0/no/off
|
||||
"""
|
||||
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
|
||||
if raw in ("false", "0", "no", "off"):
|
||||
return False
|
||||
return "/etc/ssl/certs/ca-certificates.crt"
|
||||
|
||||
|
||||
# #region call_openai_compatible [C:3] [TYPE Function] [SEMANTICS translate, llm, http, openai]
|
||||
# @BRIEF Call OpenAI-compatible API with rate-limit handling and structured output fallback.
|
||||
# @PRE Valid API endpoint, key, model, and prompt.
|
||||
@@ -141,7 +156,7 @@ def _do_http_request(url: str, headers: dict, payload: dict) -> tuple[Any, str]:
|
||||
_max_retry_429 = 3
|
||||
_retry_count_429 = 0
|
||||
while _retry_count_429 < _max_retry_429:
|
||||
response = http_requests.post(url, headers=headers, json=payload, timeout=180)
|
||||
response = http_requests.post(url, headers=headers, json=payload, timeout=180, verify=_get_verify())
|
||||
response_text = response.text
|
||||
if response.status_code == 429:
|
||||
_retry_count_429 += 1
|
||||
@@ -179,7 +194,7 @@ def _handle_response_format_fallback(
|
||||
logger.explore("Structured outputs not supported, retrying without response_format",
|
||||
extra={"src": "executor"})
|
||||
payload.pop("response_format", None)
|
||||
new_response = http_requests.post(url, headers=headers, json=payload, timeout=180)
|
||||
new_response = http_requests.post(url, headers=headers, json=payload, timeout=180, verify=_get_verify())
|
||||
response.status_code = new_response.status_code
|
||||
response._content = new_response.content
|
||||
response.encoding = new_response.encoding
|
||||
|
||||
@@ -3,12 +3,27 @@
|
||||
# @SIDE_EFFECT Makes HTTP POST calls to external LLM API.
|
||||
# @RELATION DEPENDS_ON -> [EXT:requests]
|
||||
|
||||
import os
|
||||
import time as _time
|
||||
from typing import Any
|
||||
|
||||
from ...core.logger import logger
|
||||
|
||||
|
||||
def _get_verify() -> str | bool:
|
||||
"""Resolve SSL verify from LLM_SSL_VERIFY env var.
|
||||
|
||||
Returns:
|
||||
- Path to system CA bundle when verification enabled
|
||||
(certifi bundle doesn't include corporate CA certs)
|
||||
- False when LLM_SSL_VERIFY is set to false/0/no/off
|
||||
"""
|
||||
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
|
||||
if raw in ("false", "0", "no", "off"):
|
||||
return False
|
||||
return "/etc/ssl/certs/ca-certificates.crt"
|
||||
|
||||
|
||||
class LLMClient:
|
||||
"""Call OpenAI-compatible LLM APIs with retry and structured output handling."""
|
||||
|
||||
@@ -63,7 +78,7 @@ class LLMClient:
|
||||
_max_retry_429 = 3
|
||||
_retry_count_429 = 0
|
||||
while _retry_count_429 < _max_retry_429:
|
||||
response = http_requests.post(url, headers=headers, json=payload, timeout=600)
|
||||
response = http_requests.post(url, headers=headers, json=payload, timeout=600, verify=_get_verify())
|
||||
if response.status_code == 429:
|
||||
_retry_count_429 += 1
|
||||
retry_after = response.headers.get("Retry-After")
|
||||
@@ -80,7 +95,7 @@ class LLMClient:
|
||||
and any(p in (response.text or "").lower() for p in _response_format_error_patterns)):
|
||||
logger.explore("Structured outputs not supported, retrying without response_format")
|
||||
payload.pop("response_format", None)
|
||||
response = http_requests.post(url, headers=headers, json=payload, timeout=600)
|
||||
response = http_requests.post(url, headers=headers, json=payload, timeout=600, verify=_get_verify())
|
||||
|
||||
if not response.ok:
|
||||
logger.explore(f"LLM API error status={response.status_code} model={payload.get('model')} body={response.text[:2000]}")
|
||||
|
||||
111
docs/adr/ADR-0009-ssl-certificate-management.md
Normal file
111
docs/adr/ADR-0009-ssl-certificate-management.md
Normal file
@@ -0,0 +1,111 @@
|
||||
# [DEF:ADR-0009:ADR]
|
||||
# @STATUS ACTIVE
|
||||
# @PURPOSE Define the strategy for corporate SSL certificate management across all LLM HTTP clients in ss-tools (httpx, requests, openai), covering system CA store, NSS database for Chromium/Playwright, and the LLM_SSL_VERIFY escape hatch.
|
||||
# @RELATION DEPENDS_ON -> [ADR-0001:ADR]
|
||||
# @RELATION DEPENDS_ON -> [ADR-0004:ADR]
|
||||
# @RELATION CALLS -> [docker/backend.entrypoint.sh]
|
||||
# @RELATION CALLS -> [backend/src/plugins/llm_analysis/service.py]
|
||||
# @RELATION CALLS -> [backend/src/plugins/translate/_llm_http.py]
|
||||
# @RELATION CALLS -> [backend/src/plugins/translate/preview_llm_client.py]
|
||||
# @RATIONALE ss-tools operates in corporate environments with internal Certificate Authorities.
|
||||
# Three separate HTTP stacks are used: httpx (AsyncOpenAI in llm_analysis plugin),
|
||||
# requests (sync translate plugin), and Playwright Chromium (dashboard screenshots).
|
||||
# Each has its own CA trust store, requiring different installation strategies.
|
||||
# @REJECTED Centralizing all LLM calls into a single HTTP client — rejected because
|
||||
# llm_analysis uses async httpx+AsyncOpenAI for streaming and multiple concurrent calls,
|
||||
# while translate uses sync requests for simpler request/response. Merging would
|
||||
# require rewriting one or both, creating regression risk with no business value.
|
||||
|
||||
## Problem
|
||||
|
||||
Corporate SSL certificates installed via `update-ca-certificates` into `/etc/ssl/certs/ca-certificates.crt` are NOT automatically trusted by:
|
||||
|
||||
1. **Python `requests` library** — uses bundled `certifi` CA bundle, not system store
|
||||
2. **Python `httpx` library** — uses `certifi` by default when `verify=True`
|
||||
3. **Playwright Chromium** — uses NSS Shared DB (`~/.pki/nssdb`), not OpenSSL store
|
||||
|
||||
This causes `SSLError: CERTIFICATE_VERIFY_FAILED` and `ERR_CERT_AUTHORITY_INVALID` even after correct system-wide CA installation.
|
||||
|
||||
## Solution
|
||||
|
||||
### Layer 1: System CA Store (OpenSSL)
|
||||
|
||||
- Entrypoint `install_certificates()` copies `.crt/.pem` files from `CERTS_PATH` volume mount
|
||||
- Entrypoint `install_llm_ca_certs()` downloads PEM/DER certificates from `LLM_CA_CERT_URLS`
|
||||
- DER is auto-converted to PEM (`openssl x509 -inform DER -outform PEM`)
|
||||
- `update-ca-certificates --fresh` adds them to `/etc/ssl/certs/ca-certificates.crt`
|
||||
- **Fallback**: if `update-ca-certificates` misses certs (Debian Bookworm bug with subdirectories), they are appended to `ca-certificates.crt` with SHA256 fingerprint dedup
|
||||
|
||||
### Layer 2: Python HTTP Clients
|
||||
|
||||
| Library | File | Mechanism |
|
||||
|---------|------|-----------|
|
||||
| `httpx` (AsyncOpenAI) | `service.py:LLMClient._get_ssl_verify()` | Returns `ssl.create_default_context(cafile="/etc/ssl/certs/ca-certificates.crt")` |
|
||||
| `requests` (`_llm_http.py`) | `_get_verify()` | Returns `"/etc/ssl/certs/ca-certificates.crt"` (string path) |
|
||||
| `requests` (`preview_llm_client.py`) | `_get_verify()` | Returns `"/etc/ssl/certs/ca-certificates.crt"` (string path) |
|
||||
|
||||
Key insight: `verify=True` uses certifi, NOT system CA. All three functions now return the system CA path instead of boolean `True`.
|
||||
|
||||
### Layer 3: NSS Database (Playwright Chromium)
|
||||
|
||||
- Entrypoint `install_ca_to_nss()` imports PEM certs into `~/.pki/nssdb/` using `certutil`
|
||||
- Nickname format: `{dir_prefix}-{filename}` (e.g., `llm-UC_RUSAL_Policy_CA`)
|
||||
- Dedup by SHA256 fingerprint (not nickname), preventing duplicate imports
|
||||
- Trust attributes: `"C,,"` (trusted CA for TLS server certs)
|
||||
- Fixes `ERR_CERT_AUTHORITY_INVALID` in Playwright dashboard screenshots
|
||||
|
||||
### Layer 4: LLM_SSL_VERIFY Escape Hatch
|
||||
|
||||
- Env var `LLM_SSL_VERIFY=false` disables SSL verification entirely
|
||||
- Accepted values for false: `false`, `0`, `no`, `off` (case-insensitive)
|
||||
- Default: enabled (returns system CA path / SSLContext)
|
||||
- Used by all three HTTP client implementations
|
||||
|
||||
## Key Files
|
||||
|
||||
| File | Role |
|
||||
|------|------|
|
||||
| `docker/backend.Dockerfile` | Installs `libnss3-tools` (certutil), Playwright Chromium |
|
||||
| `docker/backend.entrypoint.sh` | `install_certificates`, `install_llm_ca_certs`, `install_ca_to_nss` |
|
||||
| `plugins/llm_analysis/service.py` | `LLMClient._get_ssl_verify()` → `ssl.create_default_context(cafile=...)` |
|
||||
| `plugins/translate/_llm_http.py` | `_get_verify()` → `"/etc/ssl/certs/ca-certificates.crt"` |
|
||||
| `plugins/translate/preview_llm_client.py` | `_get_verify()` → `"/etc/ssl/certs/ca-certificates.crt"` |
|
||||
| `docker-compose.yml` | Passes `LLM_SSL_VERIFY`, `LLM_CA_CERT_URLS` |
|
||||
| `docker-compose.enterprise-clean.yml` | Same as above |
|
||||
| `.env.enterprise-clean` | Default values for both env vars |
|
||||
|
||||
## Discovered Findings
|
||||
|
||||
### Finding 1: certifi vs system CA
|
||||
`requests` and `httpx` use `certifi` CA bundle by default. Corporate CA certs installed into `/etc/ssl/certs/` are invisible to Python HTTP clients unless explicitly pointed to the system bundle.
|
||||
|
||||
### Finding 2: Three separate HTTP stacks
|
||||
ss-tools has three LLM HTTP clients (httpx for async, requests for sync, Playwright for screenshots), each with independent CA trust configuration.
|
||||
|
||||
### Finding 3: update-ca-certificates inconsistency
|
||||
On Debian Bookworm (python:3.11-slim), `update-ca-certificates` occasionally skips symlinks for PEM files in subdirectories (`/usr/local/share/ca-certificates/custom/`, `llm/`), requiring a manual `cat >> ca-certificates.crt` fallback and explicit hash symlink creation.
|
||||
|
||||
### Finding 4: DER format from corporate PKI
|
||||
Corporate PKI servers (`pki.rusal.com`) often serve certificates in DER (binary) format. `update-ca-certificates` requires PEM. Auto-detection and conversion (`openssl x509 -inform DER -outform PEM`) is essential.
|
||||
|
||||
### Finding 5: Chicken-and-egg TLS bootstrap
|
||||
Downloading a CA certificate from a PKI server that uses the same CA causes a TLS verification loop. Entrypoint uses `curl --insecure` as fallback when initial `curl` fails with TLS error.
|
||||
|
||||
### Finding 6: NSS DB format
|
||||
Chromium uses NSS Shared DB in `~/.pki/nssdb/`. The `sql:` prefix is required for `certutil` to use the SQLite format. Without it, certutil defaults to the legacy DBM format which Chromium may not read.
|
||||
|
||||
## Deploy
|
||||
|
||||
```bash
|
||||
# On target server:
|
||||
xz -dc ss-tools-backend.0.1.6.tar.xz | docker load
|
||||
xz -dc ss-tools-frontend.0.1.6.tar.xz | docker load
|
||||
|
||||
docker compose -f docker-compose.enterprise-clean.yml \
|
||||
--env-file .env.enterprise-clean down
|
||||
|
||||
docker compose -f docker-compose.enterprise-clean.yml \
|
||||
--env-file .env.enterprise-clean up -d
|
||||
```
|
||||
|
||||
# [/DEF:ADR-0009:ADR]
|
||||
251
scripts/check_llm_certs.sh
Executable file
251
scripts/check_llm_certs.sh
Executable file
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env bash
|
||||
# #region check_llm_certs [C:4] [TYPE Module] [SEMANTICS ssl, certs, diagnostics, openssl, nss]
|
||||
# @BRIEF Проверка установки корпоративных SSL-сертификатов для LLM-провайдеров.
|
||||
# Читает конфигурацию из .env.enterprise-clean, проверяет все 4 слоя:
|
||||
# OpenSSL store, httpx, requests, Chromium NSS.
|
||||
# @RATIONALE После ручной/автоматической установки сертификатов нужно убедиться
|
||||
# что все три HTTP-стека (httpx, requests, Chromium) доверяют корпоративным CA.
|
||||
# По отдельности каждый слой имеет свою логику — скрипт собирает всё воедино.
|
||||
# @USAGE ./scripts/check_llm_certs.sh [--env-file .env.enterprise-clean] [--target https://lite.ai.rusal.com]
|
||||
# @LAYER Infrastructure
|
||||
# @EXAMPLE:
|
||||
# ./scripts/check_llm_certs.sh
|
||||
# ./scripts/check_llm_certs.sh --target https://api.corp.com/v1 --env-file /opt/ss-tools/.env
|
||||
# #endregion check_llm_certs
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── Colors ──
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
PASS="${GREEN}✅${NC}"
|
||||
FAIL="${RED}❌${NC}"
|
||||
WARN="${YELLOW}⚠${NC}"
|
||||
INFO="${CYAN}ℹ${NC}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
# ── Defaults ──
|
||||
ENV_FILE="${PROJECT_ROOT}/.env.enterprise-clean"
|
||||
TARGET_URL="https://lite.ai.rusal.com"
|
||||
VERBOSE=false
|
||||
|
||||
# ── Parse args ──
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env-file) ENV_FILE="$2"; shift 2 ;;
|
||||
--target) TARGET_URL="$2"; shift 2 ;;
|
||||
-v|--verbose) VERBOSE=true; shift ;;
|
||||
--help|-h) echo "Usage: $0 [--env-file .env] [--target https://...] [-v]"; exit 0 ;;
|
||||
*) echo "Unknown: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ── Load env ──
|
||||
if [ -f "$ENV_FILE" ]; then
|
||||
echo -e "${INFO} Loading env: ${ENV_FILE}"
|
||||
set -a
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
else
|
||||
echo -e "${WARN} Env file not found: ${ENV_FILE} (using defaults)"
|
||||
fi
|
||||
|
||||
LLM_SSL_VERIFY="${LLM_SSL_VERIFY:-true}"
|
||||
CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt"
|
||||
NSS_DB_DIR="${HOME:-/root}/.pki/nssdb"
|
||||
|
||||
echo ""
|
||||
echo -e "${CYAN}========================================${NC}"
|
||||
echo -e "${CYAN} LLM SSL Certificate Diagnostics${NC}"
|
||||
echo -e "${CYAN}========================================${NC}"
|
||||
echo ""
|
||||
echo "Target URL: ${TARGET_URL}"
|
||||
echo "LLM_SSL_VERIFY: ${LLM_SSL_VERIFY}"
|
||||
echo "Env file: ${ENV_FILE}"
|
||||
echo ""
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 1: System CA Store
|
||||
# ══════════════════════════════════════════
|
||||
echo -e "${CYAN}── Layer 1: System CA Store (OpenSSL) ──${NC}"
|
||||
|
||||
# Check bundle exists
|
||||
total_certs=0
|
||||
if [ -f "$CA_BUNDLE" ]; then
|
||||
total_certs=$(awk '/BEGIN CERTIFICATE/{c++} END{print c}' "$CA_BUNDLE" 2>/dev/null || echo "0")
|
||||
echo -e " ${PASS} Bundle: ${CA_BUNDLE} (${total_certs} certificates)"
|
||||
else
|
||||
echo -e " ${FAIL} Bundle NOT FOUND: ${CA_BUNDLE}"
|
||||
fi
|
||||
|
||||
# List custom certs
|
||||
echo -n " Custom certs: "
|
||||
custom_dir="/usr/local/share/ca-certificates/custom"
|
||||
llm_dir="/usr/local/share/ca-certificates/llm"
|
||||
found_custom=0
|
||||
for dir in "$custom_dir" "$llm_dir"; do
|
||||
if [ -d "$dir" ]; then
|
||||
for f in "$dir"/*.pem "$dir"/*.crt; do
|
||||
[ -f "$f" ] || continue
|
||||
found_custom=$((found_custom+1))
|
||||
subject="$(openssl x509 -in "$f" -noout -subject 2>/dev/null | head -1 || echo "?")"
|
||||
echo -e "\n ${PASS} $(basename "$f") — ${subject}"
|
||||
done
|
||||
fi
|
||||
done
|
||||
if [ "$found_custom" -eq 0 ]; then
|
||||
echo -e "${WARN} none found (check CERTS_PATH mount and LLM_CA_CERT_URLS)"
|
||||
fi
|
||||
|
||||
# Check symlinks
|
||||
echo -n " Hash symlinks: "
|
||||
hash_links=$(find /etc/ssl/certs/ -maxdepth 1 -type l -name '[0-9a-f]*.[0-9]' 2>/dev/null | wc -l)
|
||||
echo -e "${hash_links} found"
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 2: NSS Database (Chromium)
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 2: NSS Database (Chromium) ──${NC}"
|
||||
|
||||
if command -v certutil &>/dev/null && [ -d "$NSS_DB_DIR" ]; then
|
||||
nss_db="sql:${NSS_DB_DIR}"
|
||||
nss_certs=$(certutil -L -d "$nss_db" 2>/dev/null | wc -l || echo "0")
|
||||
echo -e " ${PASS} NSS DB: ${NSS_DB_DIR} (${nss_certs} certificates)"
|
||||
|
||||
# Show custom certs
|
||||
certutil -L -d "$nss_db" 2>/dev/null | grep -iE "rusal|llm-|custom-" | while read -r line; do
|
||||
echo -e " ${PASS} ${line}"
|
||||
done
|
||||
elif ! command -v certutil &>/dev/null; then
|
||||
echo -e " ${WARN} certutil not installed (libnss3-tools missing)"
|
||||
elif [ ! -d "$NSS_DB_DIR" ]; then
|
||||
echo -e " ${WARN} NSS DB not found at ${NSS_DB_DIR}"
|
||||
fi
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 3: OpenSSL Connectivity Test
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 3: OpenSSL Connectivity ──${NC}"
|
||||
|
||||
target_host="$(echo "${TARGET_URL}" | sed 's|https://||;s|/.*$||')"
|
||||
echo -n " ${target_host}:443 — "
|
||||
|
||||
if openssl s_client -connect "${target_host}:443" -CAfile "$CA_BUNDLE" -servername "$target_host" </dev/null 2>/dev/null \
|
||||
| grep -q "Verify return code: 0"; then
|
||||
echo -e "${PASS} SSL handshake OK (verify via system CA)"
|
||||
else
|
||||
verify_code=$(openssl s_client -connect "${target_host}:443" -CAfile "$CA_BUNDLE" -servername "$target_host" </dev/null 2>&1 \
|
||||
| grep "Verify return code" | head -1 || echo "unknown")
|
||||
echo -e "${FAIL} ${verify_code}"
|
||||
echo -e " ${INFO} Check: missing intermediate CA or wrong CA bundle"
|
||||
fi
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 4: Python httpx Test
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 4: Python httpx (AsyncOpenAI) ──${NC}"
|
||||
|
||||
CA_BUNDLE_PY="${CA_BUNDLE}"
|
||||
TARGET_URL_PY="${TARGET_URL}"
|
||||
PASS_PY="${PASS}"
|
||||
FAIL_PY="${FAIL}"
|
||||
INFO_PY="${INFO}"
|
||||
WARN_PY="${WARN}"
|
||||
export CA_BUNDLE_PY TARGET_URL_PY PASS_PY FAIL_PY INFO_PY WARN_PY
|
||||
python3 -c '
|
||||
import ssl, sys, os
|
||||
|
||||
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
|
||||
target = os.environ.get("TARGET_URL_PY", "https://lite.ai.rusal.com")
|
||||
pass_emoji = os.environ.get("PASS_PY", "✅")
|
||||
fail_emoji = os.environ.get("FAIL_PY", "❌")
|
||||
info_emoji = os.environ.get("INFO_PY", "ℹ")
|
||||
|
||||
try:
|
||||
ctx = ssl.create_default_context(cafile=cafile)
|
||||
print(f" {pass_emoji} SSLContext created with cafile={cafile}")
|
||||
print(f" {info_emoji} verify_mode={ctx.verify_mode}")
|
||||
except Exception as e:
|
||||
print(f" {fail_emoji} SSLContext error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
import httpx
|
||||
r = httpx.get(target, verify=ctx, timeout=10)
|
||||
print(f" {pass_emoji} httpx GET {target} → {r.status_code}")
|
||||
except httpx.ConnectError as e:
|
||||
print(f" {fail_emoji} httpx connection failed: {e}")
|
||||
except Exception as e:
|
||||
print(f" {fail_emoji} httpx error: {e}")
|
||||
' 2>&1 || echo -e " ${FAIL} Python test failed"
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 5: Python requests Test
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 5: Python requests ──${NC}"
|
||||
|
||||
python3 -c '
|
||||
import os, sys
|
||||
target = os.environ.get("TARGET_URL_PY", "https://lite.ai.rusal.com")
|
||||
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
|
||||
pass_emoji = os.environ.get("PASS_PY", "✅")
|
||||
fail_emoji = os.environ.get("FAIL_PY", "❌")
|
||||
try:
|
||||
import requests
|
||||
r = requests.get(target, verify=cafile, timeout=10)
|
||||
print(f" {pass_emoji} requests GET {target} → {r.status_code}")
|
||||
except requests.exceptions.SSLError as e:
|
||||
print(f" {fail_emoji} requests SSL error: {e}")
|
||||
except Exception as e:
|
||||
print(f" {fail_emoji} requests error: {e}")
|
||||
' 2>&1 || echo -e " ${FAIL} Python requests test failed"
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 6: LLM_SSL_VERIFY env var check
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 6: LLM_SSL_VERIFY env var ──${NC}"
|
||||
|
||||
python3 -c '
|
||||
import os
|
||||
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
|
||||
disabled = raw in ("false", "0", "no", "off")
|
||||
warn_emoji = os.environ.get("WARN_PY", "⚠")
|
||||
pass_emoji = os.environ.get("PASS_PY", "✅")
|
||||
info_emoji = os.environ.get("INFO_PY", "ℹ")
|
||||
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
|
||||
if disabled:
|
||||
print(f" {warn_emoji} LLM_SSL_VERIFY={raw} — SSL verification DISABLED")
|
||||
else:
|
||||
print(f" {pass_emoji} LLM_SSL_VERIFY={raw} — SSL verification ENABLED")
|
||||
import ssl
|
||||
if os.path.exists(cafile):
|
||||
ctx = ssl.create_default_context(cafile=cafile)
|
||||
print(f" {info_emoji} ssl.create_default_context(cafile=...) — OK")
|
||||
' 2>&1
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SUMMARY
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}========================================${NC}"
|
||||
echo -e "${CYAN} Summary${NC}"
|
||||
echo -e "${CYAN}========================================${NC}"
|
||||
echo ""
|
||||
echo " CA Bundle: ${CA_BUNDLE} (${total_certs} certs)"
|
||||
echo " NSS DB: ${NSS_DB_DIR}"
|
||||
echo " Target: ${TARGET_URL}"
|
||||
echo " LLM_SSL_VERIFY: ${LLM_SSL_VERIFY}"
|
||||
echo ""
|
||||
echo " If openssl test passes but Python fails — check certifi vs system CA"
|
||||
echo " If all tests pass — remove LLM_SSL_VERIFY=false from .env and rerun"
|
||||
echo ""
|
||||
Reference in New Issue
Block a user