#!/usr/bin/env bash # #region check_llm_certs [C:4] [TYPE Module] [SEMANTICS ssl, certs, diagnostics, openssl, nss] # @BRIEF Проверка установки корпоративных SSL-сертификатов для LLM-провайдеров. # Читает конфигурацию из .env.enterprise-clean, проверяет все 4 слоя: # OpenSSL store, httpx, requests, Chromium NSS. # @RATIONALE После ручной/автоматической установки сертификатов нужно убедиться # что все три HTTP-стека (httpx, requests, Chromium) доверяют корпоративным CA. # По отдельности каждый слой имеет свою логику — скрипт собирает всё воедино. # @USAGE ./scripts/check_llm_certs.sh [--env-file .env.enterprise-clean] [--target https://lite.ai.rusal.com] # @LAYER Infrastructure # @EXAMPLE: # ./scripts/check_llm_certs.sh # ./scripts/check_llm_certs.sh --target https://api.corp.com/v1 --env-file /opt/ss-tools/.env # #endregion check_llm_certs set -euo pipefail # ── Colors ── RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' CYAN='\033[0;36m' NC='\033[0m' # No Color PASS="${GREEN}✅${NC}" FAIL="${RED}❌${NC}" WARN="${YELLOW}⚠${NC}" INFO="${CYAN}ℹ${NC}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" # ── Defaults ── ENV_FILE="${PROJECT_ROOT}/.env.enterprise-clean" TARGET_URL="https://lite.ai.rusal.com" VERBOSE=false # ── Parse args ── while [[ $# -gt 0 ]]; do case "$1" in --env-file) ENV_FILE="$2"; shift 2 ;; --target) TARGET_URL="$2"; shift 2 ;; -v|--verbose) VERBOSE=true; shift ;; --help|-h) echo "Usage: $0 [--env-file .env] [--target https://...] [-v]"; exit 0 ;; *) echo "Unknown: $1"; exit 1 ;; esac done # ── Load env ── if [ -f "$ENV_FILE" ]; then echo -e "${INFO} Loading env: ${ENV_FILE}" set -a source "$ENV_FILE" set +a else echo -e "${WARN} Env file not found: ${ENV_FILE} (using defaults)" fi LLM_SSL_VERIFY="${LLM_SSL_VERIFY:-true}" CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt" NSS_DB_DIR="${HOME:-/root}/.pki/nssdb" echo "" echo -e "${CYAN}========================================${NC}" echo -e "${CYAN} LLM SSL Certificate Diagnostics${NC}" echo -e "${CYAN}========================================${NC}" echo "" echo "Target URL: ${TARGET_URL}" echo "LLM_SSL_VERIFY: ${LLM_SSL_VERIFY}" echo "Env file: ${ENV_FILE}" echo "" # ══════════════════════════════════════════ # SECTION 1: System CA Store # ══════════════════════════════════════════ echo -e "${CYAN}── Layer 1: System CA Store (OpenSSL) ──${NC}" # Check bundle exists total_certs=0 if [ -f "$CA_BUNDLE" ]; then total_certs=$(awk '/BEGIN CERTIFICATE/{c++} END{print c}' "$CA_BUNDLE" 2>/dev/null || echo "0") echo -e " ${PASS} Bundle: ${CA_BUNDLE} (${total_certs} certificates)" else echo -e " ${FAIL} Bundle NOT FOUND: ${CA_BUNDLE}" fi # List custom certs echo -n " Custom certs: " custom_dir="/usr/local/share/ca-certificates/custom" llm_dir="/usr/local/share/ca-certificates/llm" found_custom=0 for dir in "$custom_dir" "$llm_dir"; do if [ -d "$dir" ]; then for f in "$dir"/*.pem "$dir"/*.crt; do [ -f "$f" ] || continue found_custom=$((found_custom+1)) subject="$(openssl x509 -in "$f" -noout -subject 2>/dev/null | head -1 || echo "?")" echo -e "\n ${PASS} $(basename "$f") — ${subject}" done fi done if [ "$found_custom" -eq 0 ]; then echo -e "${WARN} none found (check CERTS_PATH mount and LLM_CA_CERT_URLS)" fi # Check symlinks echo -n " Hash symlinks: " hash_links=$(find /etc/ssl/certs/ -maxdepth 1 -type l -name '[0-9a-f]*.[0-9]' 2>/dev/null | wc -l) echo -e "${hash_links} found" # ══════════════════════════════════════════ # SECTION 2: NSS Database (Chromium) # ══════════════════════════════════════════ echo "" echo -e "${CYAN}── Layer 2: NSS Database (Chromium) ──${NC}" if command -v certutil &>/dev/null && [ -d "$NSS_DB_DIR" ]; then nss_db="sql:${NSS_DB_DIR}" nss_certs=$(certutil -L -d "$nss_db" 2>/dev/null | wc -l || echo "0") echo -e " ${PASS} NSS DB: ${NSS_DB_DIR} (${nss_certs} certificates)" # Show custom certs certutil -L -d "$nss_db" 2>/dev/null | grep -iE "rusal|llm-|custom-" | while read -r line; do echo -e " ${PASS} ${line}" done elif ! command -v certutil &>/dev/null; then echo -e " ${WARN} certutil not installed (libnss3-tools missing)" elif [ ! -d "$NSS_DB_DIR" ]; then echo -e " ${WARN} NSS DB not found at ${NSS_DB_DIR}" fi # ══════════════════════════════════════════ # SECTION 3: OpenSSL Connectivity Test # ══════════════════════════════════════════ echo "" echo -e "${CYAN}── Layer 3: OpenSSL Connectivity ──${NC}" target_host="$(echo "${TARGET_URL}" | sed 's|https://||;s|/.*$||')" echo -n " ${target_host}:443 — " if openssl s_client -connect "${target_host}:443" -CAfile "$CA_BUNDLE" -servername "$target_host" /dev/null \ | grep -q "Verify return code: 0"; then echo -e "${PASS} SSL handshake OK (verify via system CA)" else verify_code=$(openssl s_client -connect "${target_host}:443" -CAfile "$CA_BUNDLE" -servername "$target_host" &1 \ | grep "Verify return code" | head -1 || echo "unknown") echo -e "${FAIL} ${verify_code}" echo -e " ${INFO} Check: missing intermediate CA or wrong CA bundle" fi # ══════════════════════════════════════════ # SECTION 4: Python httpx Test # ══════════════════════════════════════════ echo "" echo -e "${CYAN}── Layer 4: Python httpx (AsyncOpenAI) ──${NC}" CA_BUNDLE_PY="${CA_BUNDLE}" TARGET_URL_PY="${TARGET_URL}" PASS_PY="${PASS}" FAIL_PY="${FAIL}" INFO_PY="${INFO}" WARN_PY="${WARN}" export CA_BUNDLE_PY TARGET_URL_PY PASS_PY FAIL_PY INFO_PY WARN_PY python3 -c ' import ssl, sys, os cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt") target = os.environ.get("TARGET_URL_PY", "https://lite.ai.rusal.com") pass_emoji = os.environ.get("PASS_PY", "✅") fail_emoji = os.environ.get("FAIL_PY", "❌") info_emoji = os.environ.get("INFO_PY", "ℹ") try: ctx = ssl.create_default_context(cafile=cafile) print(f" {pass_emoji} SSLContext created with cafile={cafile}") print(f" {info_emoji} verify_mode={ctx.verify_mode}") except Exception as e: print(f" {fail_emoji} SSLContext error: {e}") sys.exit(1) try: import httpx r = httpx.get(target, verify=ctx, timeout=10) print(f" {pass_emoji} httpx GET {target} → {r.status_code}") except httpx.ConnectError as e: print(f" {fail_emoji} httpx connection failed: {e}") except Exception as e: print(f" {fail_emoji} httpx error: {e}") ' 2>&1 || echo -e " ${FAIL} Python test failed" # ══════════════════════════════════════════ # SECTION 5: Python requests Test # ══════════════════════════════════════════ echo "" echo -e "${CYAN}── Layer 5: Python requests ──${NC}" python3 -c ' import os, sys target = os.environ.get("TARGET_URL_PY", "https://lite.ai.rusal.com") cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt") pass_emoji = os.environ.get("PASS_PY", "✅") fail_emoji = os.environ.get("FAIL_PY", "❌") try: import requests r = requests.get(target, verify=cafile, timeout=10) print(f" {pass_emoji} requests GET {target} → {r.status_code}") except requests.exceptions.SSLError as e: print(f" {fail_emoji} requests SSL error: {e}") except Exception as e: print(f" {fail_emoji} requests error: {e}") ' 2>&1 || echo -e " ${FAIL} Python requests test failed" # ══════════════════════════════════════════ # SECTION 6: LLM_SSL_VERIFY env var check # ══════════════════════════════════════════ echo "" echo -e "${CYAN}── Layer 6: LLM_SSL_VERIFY env var ──${NC}" python3 -c ' import os raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower() disabled = raw in ("false", "0", "no", "off") warn_emoji = os.environ.get("WARN_PY", "⚠") pass_emoji = os.environ.get("PASS_PY", "✅") info_emoji = os.environ.get("INFO_PY", "ℹ") cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt") if disabled: print(f" {warn_emoji} LLM_SSL_VERIFY={raw} — SSL verification DISABLED") else: print(f" {pass_emoji} LLM_SSL_VERIFY={raw} — SSL verification ENABLED") import ssl if os.path.exists(cafile): ctx = ssl.create_default_context(cafile=cafile) print(f" {info_emoji} ssl.create_default_context(cafile=...) — OK") ' 2>&1 # ══════════════════════════════════════════ # SUMMARY # ══════════════════════════════════════════ echo "" echo -e "${CYAN}========================================${NC}" echo -e "${CYAN} Summary${NC}" echo -e "${CYAN}========================================${NC}" echo "" echo " CA Bundle: ${CA_BUNDLE} (${total_certs} certs)" echo " NSS DB: ${NSS_DB_DIR}" echo " Target: ${TARGET_URL}" echo " LLM_SSL_VERIFY: ${LLM_SSL_VERIFY}" echo "" echo " If openssl test passes but Python fails — check certifi vs system CA" echo " If all tests pass — remove LLM_SSL_VERIFY=false from .env and rerun" echo ""