diff --git a/scripts/check_llm_certs.sh b/scripts/check_llm_certs.sh new file mode 100755 index 00000000..34d7cbc0 --- /dev/null +++ b/scripts/check_llm_certs.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash +# #region check_llm_certs [C:4] [TYPE Module] [SEMANTICS ssl, certs, diagnostics, openssl, nss] +# @BRIEF Проверка установки корпоративных SSL-сертификатов для LLM-провайдеров. +# Читает конфигурацию из .env.enterprise-clean, проверяет все 4 слоя: +# OpenSSL store, httpx, requests, Chromium NSS. +# @RATIONALE После ручной/автоматической установки сертификатов нужно убедиться +# что все три HTTP-стека (httpx, requests, Chromium) доверяют корпоративным CA. +# По отдельности каждый слой имеет свою логику — скрипт собирает всё воедино. +# @USAGE ./scripts/check_llm_certs.sh [--env-file .env.enterprise-clean] [--target https://lite.ai.rusal.com] +# @LAYER Infrastructure +# @EXAMPLE: +# ./scripts/check_llm_certs.sh +# ./scripts/check_llm_certs.sh --target https://api.corp.com/v1 --env-file /opt/ss-tools/.env +# #endregion check_llm_certs + +set -euo pipefail + +# ── Colors ── +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color +PASS="${GREEN}✅${NC}" +FAIL="${RED}❌${NC}" +WARN="${YELLOW}⚠${NC}" +INFO="${CYAN}ℹ${NC}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +# ── Defaults ── +ENV_FILE="${PROJECT_ROOT}/.env.enterprise-clean" +TARGET_URL="https://lite.ai.rusal.com" +VERBOSE=false + +# ── Parse args ── +while [[ $# -gt 0 ]]; do + case "$1" in + --env-file) ENV_FILE="$2"; shift 2 ;; + --target) TARGET_URL="$2"; shift 2 ;; + -v|--verbose) VERBOSE=true; shift ;; + --help|-h) echo "Usage: $0 [--env-file .env] [--target https://...] [-v]"; exit 0 ;; + *) echo "Unknown: $1"; exit 1 ;; + esac +done + +# ── Load env ── +if [ -f "$ENV_FILE" ]; then + echo -e "${INFO} Loading env: ${ENV_FILE}" + set -a + source "$ENV_FILE" + set +a +else + echo -e "${WARN} Env file not found: ${ENV_FILE} (using defaults)" +fi + +LLM_SSL_VERIFY="${LLM_SSL_VERIFY:-true}" +CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt" +NSS_DB_DIR="${HOME:-/root}/.pki/nssdb" + +echo "" +echo -e "${CYAN}========================================${NC}" +echo -e "${CYAN} LLM SSL Certificate Diagnostics${NC}" +echo -e "${CYAN}========================================${NC}" +echo "" +echo "Target URL: ${TARGET_URL}" +echo "LLM_SSL_VERIFY: ${LLM_SSL_VERIFY}" +echo "Env file: ${ENV_FILE}" +echo "" + +# ══════════════════════════════════════════ +# SECTION 1: System CA Store +# ══════════════════════════════════════════ +echo -e "${CYAN}── Layer 1: System CA Store (OpenSSL) ──${NC}" + +# Check bundle exists +total_certs=0 +if [ -f "$CA_BUNDLE" ]; then + total_certs=$(awk '/BEGIN CERTIFICATE/{c++} END{print c}' "$CA_BUNDLE" 2>/dev/null || echo "0") + echo -e " ${PASS} Bundle: ${CA_BUNDLE} (${total_certs} certificates)" +else + echo -e " ${FAIL} Bundle NOT FOUND: ${CA_BUNDLE}" +fi + +# List custom certs +echo -n " Custom certs: " +custom_dir="/usr/local/share/ca-certificates/custom" +llm_dir="/usr/local/share/ca-certificates/llm" +found_custom=0 +for dir in "$custom_dir" "$llm_dir"; do + if [ -d "$dir" ]; then + for f in "$dir"/*.pem "$dir"/*.crt; do + [ -f "$f" ] || continue + found_custom=$((found_custom+1)) + subject="$(openssl x509 -in "$f" -noout -subject 2>/dev/null | head -1 || echo "?")" + echo -e "\n ${PASS} $(basename "$f") — ${subject}" + done + fi +done +if [ "$found_custom" -eq 0 ]; then + echo -e "${WARN} none found (check CERTS_PATH mount and LLM_CA_CERT_URLS)" +fi + +# Check symlinks +echo -n " Hash symlinks: " +hash_links=$(find /etc/ssl/certs/ -maxdepth 1 -type l -name '[0-9a-f]*.[0-9]' 2>/dev/null | wc -l) +echo -e "${hash_links} found" + +# ══════════════════════════════════════════ +# SECTION 2: NSS Database (Chromium) +# ══════════════════════════════════════════ +echo "" +echo -e "${CYAN}── Layer 2: NSS Database (Chromium) ──${NC}" + +if command -v certutil &>/dev/null && [ -d "$NSS_DB_DIR" ]; then + nss_db="sql:${NSS_DB_DIR}" + nss_certs=$(certutil -L -d "$nss_db" 2>/dev/null | wc -l || echo "0") + echo -e " ${PASS} NSS DB: ${NSS_DB_DIR} (${nss_certs} certificates)" + + # Show custom certs + certutil -L -d "$nss_db" 2>/dev/null | grep -iE "rusal|llm-|custom-" | while read -r line; do + echo -e " ${PASS} ${line}" + done +elif ! command -v certutil &>/dev/null; then + echo -e " ${WARN} certutil not installed (libnss3-tools missing)" +elif [ ! -d "$NSS_DB_DIR" ]; then + echo -e " ${WARN} NSS DB not found at ${NSS_DB_DIR}" +fi + +# ══════════════════════════════════════════ +# SECTION 3: OpenSSL Connectivity Test +# ══════════════════════════════════════════ +echo "" +echo -e "${CYAN}── Layer 3: OpenSSL Connectivity ──${NC}" + +target_host="$(echo "${TARGET_URL}" | sed 's|https://||;s|/.*$||')" +echo -n " ${target_host}:443 — " + +if openssl s_client -connect "${target_host}:443" -CAfile "$CA_BUNDLE" -servername "$target_host" /dev/null \ + | grep -q "Verify return code: 0"; then + echo -e "${PASS} SSL handshake OK (verify via system CA)" +else + verify_code=$(openssl s_client -connect "${target_host}:443" -CAfile "$CA_BUNDLE" -servername "$target_host" &1 \ + | grep "Verify return code" | head -1 || echo "unknown") + echo -e "${FAIL} ${verify_code}" + echo -e " ${INFO} Check: missing intermediate CA or wrong CA bundle" +fi + +# ══════════════════════════════════════════ +# SECTION 4: Python httpx Test +# ══════════════════════════════════════════ +echo "" +echo -e "${CYAN}── Layer 4: Python httpx (AsyncOpenAI) ──${NC}" + +CA_BUNDLE_PY="${CA_BUNDLE}" +TARGET_URL_PY="${TARGET_URL}" +PASS_PY="${PASS}" +FAIL_PY="${FAIL}" +INFO_PY="${INFO}" +WARN_PY="${WARN}" +export CA_BUNDLE_PY TARGET_URL_PY PASS_PY FAIL_PY INFO_PY WARN_PY +python3 -c ' +import ssl, sys, os + +cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt") +target = os.environ.get("TARGET_URL_PY", "https://lite.ai.rusal.com") +pass_emoji = os.environ.get("PASS_PY", "✅") +fail_emoji = os.environ.get("FAIL_PY", "❌") +info_emoji = os.environ.get("INFO_PY", "ℹ") + +try: + ctx = ssl.create_default_context(cafile=cafile) + print(f" {pass_emoji} SSLContext created with cafile={cafile}") + print(f" {info_emoji} verify_mode={ctx.verify_mode}") +except Exception as e: + print(f" {fail_emoji} SSLContext error: {e}") + sys.exit(1) + +try: + import httpx + r = httpx.get(target, verify=ctx, timeout=10) + print(f" {pass_emoji} httpx GET {target} → {r.status_code}") +except httpx.ConnectError as e: + print(f" {fail_emoji} httpx connection failed: {e}") +except Exception as e: + print(f" {fail_emoji} httpx error: {e}") +' 2>&1 || echo -e " ${FAIL} Python test failed" + +# ══════════════════════════════════════════ +# SECTION 5: Python requests Test +# ══════════════════════════════════════════ +echo "" +echo -e "${CYAN}── Layer 5: Python requests ──${NC}" + +python3 -c ' +import os, sys +target = os.environ.get("TARGET_URL_PY", "https://lite.ai.rusal.com") +cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt") +pass_emoji = os.environ.get("PASS_PY", "✅") +fail_emoji = os.environ.get("FAIL_PY", "❌") +try: + import requests + r = requests.get(target, verify=cafile, timeout=10) + print(f" {pass_emoji} requests GET {target} → {r.status_code}") +except requests.exceptions.SSLError as e: + print(f" {fail_emoji} requests SSL error: {e}") +except Exception as e: + print(f" {fail_emoji} requests error: {e}") +' 2>&1 || echo -e " ${FAIL} Python requests test failed" + +# ══════════════════════════════════════════ +# SECTION 6: LLM_SSL_VERIFY env var check +# ══════════════════════════════════════════ +echo "" +echo -e "${CYAN}── Layer 6: LLM_SSL_VERIFY env var ──${NC}" + +python3 -c ' +import os +raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower() +disabled = raw in ("false", "0", "no", "off") +warn_emoji = os.environ.get("WARN_PY", "⚠") +pass_emoji = os.environ.get("PASS_PY", "✅") +info_emoji = os.environ.get("INFO_PY", "ℹ") +cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt") +if disabled: + print(f" {warn_emoji} LLM_SSL_VERIFY={raw} — SSL verification DISABLED") +else: + print(f" {pass_emoji} LLM_SSL_VERIFY={raw} — SSL verification ENABLED") + import ssl + if os.path.exists(cafile): + ctx = ssl.create_default_context(cafile=cafile) + print(f" {info_emoji} ssl.create_default_context(cafile=...) — OK") +' 2>&1 + +# ══════════════════════════════════════════ +# SUMMARY +# ══════════════════════════════════════════ +echo "" +echo -e "${CYAN}========================================${NC}" +echo -e "${CYAN} Summary${NC}" +echo -e "${CYAN}========================================${NC}" +echo "" +echo " CA Bundle: ${CA_BUNDLE} (${total_certs} certs)" +echo " NSS DB: ${NSS_DB_DIR}" +echo " Target: ${TARGET_URL}" +echo " LLM_SSL_VERIFY: ${LLM_SSL_VERIFY}" +echo "" +echo " If openssl test passes but Python fails — check certifi vs system CA" +echo " If all tests pass — remove LLM_SSL_VERIFY=false from .env and rerun" +echo ""