feat: add SSL certificate diag script (check_llm_certs.sh)
Runs from inside Docker container to verify all 4 SSL layers: 1. System CA store (openssl s_client) 2. NSS database (certutil) for Chromium 3. Python httpx (SSLContext with cafile) 4. Python requests (verify with system CA path) Reads LLM_SSL_VERIFY and CA paths from .env.enterprise-clean
This commit is contained in:
251
scripts/check_llm_certs.sh
Executable file
251
scripts/check_llm_certs.sh
Executable file
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env bash
|
||||
# #region check_llm_certs [C:4] [TYPE Module] [SEMANTICS ssl, certs, diagnostics, openssl, nss]
|
||||
# @BRIEF Проверка установки корпоративных SSL-сертификатов для LLM-провайдеров.
|
||||
# Читает конфигурацию из .env.enterprise-clean, проверяет все 4 слоя:
|
||||
# OpenSSL store, httpx, requests, Chromium NSS.
|
||||
# @RATIONALE После ручной/автоматической установки сертификатов нужно убедиться
|
||||
# что все три HTTP-стека (httpx, requests, Chromium) доверяют корпоративным CA.
|
||||
# По отдельности каждый слой имеет свою логику — скрипт собирает всё воедино.
|
||||
# @USAGE ./scripts/check_llm_certs.sh [--env-file .env.enterprise-clean] [--target https://lite.ai.rusal.com]
|
||||
# @LAYER Infrastructure
|
||||
# @EXAMPLE:
|
||||
# ./scripts/check_llm_certs.sh
|
||||
# ./scripts/check_llm_certs.sh --target https://api.corp.com/v1 --env-file /opt/ss-tools/.env
|
||||
# #endregion check_llm_certs
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── Colors ──
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
PASS="${GREEN}✅${NC}"
|
||||
FAIL="${RED}❌${NC}"
|
||||
WARN="${YELLOW}⚠${NC}"
|
||||
INFO="${CYAN}ℹ${NC}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
# ── Defaults ──
|
||||
ENV_FILE="${PROJECT_ROOT}/.env.enterprise-clean"
|
||||
TARGET_URL="https://lite.ai.rusal.com"
|
||||
VERBOSE=false
|
||||
|
||||
# ── Parse args ──
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env-file) ENV_FILE="$2"; shift 2 ;;
|
||||
--target) TARGET_URL="$2"; shift 2 ;;
|
||||
-v|--verbose) VERBOSE=true; shift ;;
|
||||
--help|-h) echo "Usage: $0 [--env-file .env] [--target https://...] [-v]"; exit 0 ;;
|
||||
*) echo "Unknown: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ── Load env ──
|
||||
if [ -f "$ENV_FILE" ]; then
|
||||
echo -e "${INFO} Loading env: ${ENV_FILE}"
|
||||
set -a
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
else
|
||||
echo -e "${WARN} Env file not found: ${ENV_FILE} (using defaults)"
|
||||
fi
|
||||
|
||||
LLM_SSL_VERIFY="${LLM_SSL_VERIFY:-true}"
|
||||
CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt"
|
||||
NSS_DB_DIR="${HOME:-/root}/.pki/nssdb"
|
||||
|
||||
echo ""
|
||||
echo -e "${CYAN}========================================${NC}"
|
||||
echo -e "${CYAN} LLM SSL Certificate Diagnostics${NC}"
|
||||
echo -e "${CYAN}========================================${NC}"
|
||||
echo ""
|
||||
echo "Target URL: ${TARGET_URL}"
|
||||
echo "LLM_SSL_VERIFY: ${LLM_SSL_VERIFY}"
|
||||
echo "Env file: ${ENV_FILE}"
|
||||
echo ""
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 1: System CA Store
|
||||
# ══════════════════════════════════════════
|
||||
echo -e "${CYAN}── Layer 1: System CA Store (OpenSSL) ──${NC}"
|
||||
|
||||
# Check bundle exists
|
||||
total_certs=0
|
||||
if [ -f "$CA_BUNDLE" ]; then
|
||||
total_certs=$(awk '/BEGIN CERTIFICATE/{c++} END{print c}' "$CA_BUNDLE" 2>/dev/null || echo "0")
|
||||
echo -e " ${PASS} Bundle: ${CA_BUNDLE} (${total_certs} certificates)"
|
||||
else
|
||||
echo -e " ${FAIL} Bundle NOT FOUND: ${CA_BUNDLE}"
|
||||
fi
|
||||
|
||||
# List custom certs
|
||||
echo -n " Custom certs: "
|
||||
custom_dir="/usr/local/share/ca-certificates/custom"
|
||||
llm_dir="/usr/local/share/ca-certificates/llm"
|
||||
found_custom=0
|
||||
for dir in "$custom_dir" "$llm_dir"; do
|
||||
if [ -d "$dir" ]; then
|
||||
for f in "$dir"/*.pem "$dir"/*.crt; do
|
||||
[ -f "$f" ] || continue
|
||||
found_custom=$((found_custom+1))
|
||||
subject="$(openssl x509 -in "$f" -noout -subject 2>/dev/null | head -1 || echo "?")"
|
||||
echo -e "\n ${PASS} $(basename "$f") — ${subject}"
|
||||
done
|
||||
fi
|
||||
done
|
||||
if [ "$found_custom" -eq 0 ]; then
|
||||
echo -e "${WARN} none found (check CERTS_PATH mount and LLM_CA_CERT_URLS)"
|
||||
fi
|
||||
|
||||
# Check symlinks
|
||||
echo -n " Hash symlinks: "
|
||||
hash_links=$(find /etc/ssl/certs/ -maxdepth 1 -type l -name '[0-9a-f]*.[0-9]' 2>/dev/null | wc -l)
|
||||
echo -e "${hash_links} found"
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 2: NSS Database (Chromium)
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 2: NSS Database (Chromium) ──${NC}"
|
||||
|
||||
if command -v certutil &>/dev/null && [ -d "$NSS_DB_DIR" ]; then
|
||||
nss_db="sql:${NSS_DB_DIR}"
|
||||
nss_certs=$(certutil -L -d "$nss_db" 2>/dev/null | wc -l || echo "0")
|
||||
echo -e " ${PASS} NSS DB: ${NSS_DB_DIR} (${nss_certs} certificates)"
|
||||
|
||||
# Show custom certs
|
||||
certutil -L -d "$nss_db" 2>/dev/null | grep -iE "rusal|llm-|custom-" | while read -r line; do
|
||||
echo -e " ${PASS} ${line}"
|
||||
done
|
||||
elif ! command -v certutil &>/dev/null; then
|
||||
echo -e " ${WARN} certutil not installed (libnss3-tools missing)"
|
||||
elif [ ! -d "$NSS_DB_DIR" ]; then
|
||||
echo -e " ${WARN} NSS DB not found at ${NSS_DB_DIR}"
|
||||
fi
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 3: OpenSSL Connectivity Test
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 3: OpenSSL Connectivity ──${NC}"
|
||||
|
||||
target_host="$(echo "${TARGET_URL}" | sed 's|https://||;s|/.*$||')"
|
||||
echo -n " ${target_host}:443 — "
|
||||
|
||||
if openssl s_client -connect "${target_host}:443" -CAfile "$CA_BUNDLE" -servername "$target_host" </dev/null 2>/dev/null \
|
||||
| grep -q "Verify return code: 0"; then
|
||||
echo -e "${PASS} SSL handshake OK (verify via system CA)"
|
||||
else
|
||||
verify_code=$(openssl s_client -connect "${target_host}:443" -CAfile "$CA_BUNDLE" -servername "$target_host" </dev/null 2>&1 \
|
||||
| grep "Verify return code" | head -1 || echo "unknown")
|
||||
echo -e "${FAIL} ${verify_code}"
|
||||
echo -e " ${INFO} Check: missing intermediate CA or wrong CA bundle"
|
||||
fi
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 4: Python httpx Test
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 4: Python httpx (AsyncOpenAI) ──${NC}"
|
||||
|
||||
CA_BUNDLE_PY="${CA_BUNDLE}"
|
||||
TARGET_URL_PY="${TARGET_URL}"
|
||||
PASS_PY="${PASS}"
|
||||
FAIL_PY="${FAIL}"
|
||||
INFO_PY="${INFO}"
|
||||
WARN_PY="${WARN}"
|
||||
export CA_BUNDLE_PY TARGET_URL_PY PASS_PY FAIL_PY INFO_PY WARN_PY
|
||||
python3 -c '
|
||||
import ssl, sys, os
|
||||
|
||||
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
|
||||
target = os.environ.get("TARGET_URL_PY", "https://lite.ai.rusal.com")
|
||||
pass_emoji = os.environ.get("PASS_PY", "✅")
|
||||
fail_emoji = os.environ.get("FAIL_PY", "❌")
|
||||
info_emoji = os.environ.get("INFO_PY", "ℹ")
|
||||
|
||||
try:
|
||||
ctx = ssl.create_default_context(cafile=cafile)
|
||||
print(f" {pass_emoji} SSLContext created with cafile={cafile}")
|
||||
print(f" {info_emoji} verify_mode={ctx.verify_mode}")
|
||||
except Exception as e:
|
||||
print(f" {fail_emoji} SSLContext error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
import httpx
|
||||
r = httpx.get(target, verify=ctx, timeout=10)
|
||||
print(f" {pass_emoji} httpx GET {target} → {r.status_code}")
|
||||
except httpx.ConnectError as e:
|
||||
print(f" {fail_emoji} httpx connection failed: {e}")
|
||||
except Exception as e:
|
||||
print(f" {fail_emoji} httpx error: {e}")
|
||||
' 2>&1 || echo -e " ${FAIL} Python test failed"
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 5: Python requests Test
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 5: Python requests ──${NC}"
|
||||
|
||||
python3 -c '
|
||||
import os, sys
|
||||
target = os.environ.get("TARGET_URL_PY", "https://lite.ai.rusal.com")
|
||||
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
|
||||
pass_emoji = os.environ.get("PASS_PY", "✅")
|
||||
fail_emoji = os.environ.get("FAIL_PY", "❌")
|
||||
try:
|
||||
import requests
|
||||
r = requests.get(target, verify=cafile, timeout=10)
|
||||
print(f" {pass_emoji} requests GET {target} → {r.status_code}")
|
||||
except requests.exceptions.SSLError as e:
|
||||
print(f" {fail_emoji} requests SSL error: {e}")
|
||||
except Exception as e:
|
||||
print(f" {fail_emoji} requests error: {e}")
|
||||
' 2>&1 || echo -e " ${FAIL} Python requests test failed"
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 6: LLM_SSL_VERIFY env var check
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 6: LLM_SSL_VERIFY env var ──${NC}"
|
||||
|
||||
python3 -c '
|
||||
import os
|
||||
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
|
||||
disabled = raw in ("false", "0", "no", "off")
|
||||
warn_emoji = os.environ.get("WARN_PY", "⚠")
|
||||
pass_emoji = os.environ.get("PASS_PY", "✅")
|
||||
info_emoji = os.environ.get("INFO_PY", "ℹ")
|
||||
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
|
||||
if disabled:
|
||||
print(f" {warn_emoji} LLM_SSL_VERIFY={raw} — SSL verification DISABLED")
|
||||
else:
|
||||
print(f" {pass_emoji} LLM_SSL_VERIFY={raw} — SSL verification ENABLED")
|
||||
import ssl
|
||||
if os.path.exists(cafile):
|
||||
ctx = ssl.create_default_context(cafile=cafile)
|
||||
print(f" {info_emoji} ssl.create_default_context(cafile=...) — OK")
|
||||
' 2>&1
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SUMMARY
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}========================================${NC}"
|
||||
echo -e "${CYAN} Summary${NC}"
|
||||
echo -e "${CYAN}========================================${NC}"
|
||||
echo ""
|
||||
echo " CA Bundle: ${CA_BUNDLE} (${total_certs} certs)"
|
||||
echo " NSS DB: ${NSS_DB_DIR}"
|
||||
echo " Target: ${TARGET_URL}"
|
||||
echo " LLM_SSL_VERIFY: ${LLM_SSL_VERIFY}"
|
||||
echo ""
|
||||
echo " If openssl test passes but Python fails — check certifi vs system CA"
|
||||
echo " If all tests pass — remove LLM_SSL_VERIFY=false from .env and rerun"
|
||||
echo ""
|
||||
Reference in New Issue
Block a user