busya
2d4b60cef9
Runs from inside Docker container to verify all 4 SSL layers: 1. System CA store (openssl s_client) 2. NSS database (certutil) for Chromium 3. Python httpx (SSLContext with cafile) 4. Python requests (verify with system CA path) Reads LLM_SSL_VERIFY and CA paths from .env.enterprise-clean
252 lines
10 KiB
Bash
Executable File
252 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# #region check_llm_certs [C:4] [TYPE Module] [SEMANTICS ssl, certs, diagnostics, openssl, nss]
|
||
# @BRIEF Проверка установки корпоративных SSL-сертификатов для LLM-провайдеров.
|
||
# Читает конфигурацию из .env.enterprise-clean, проверяет все 4 слоя:
|
||
# OpenSSL store, httpx, requests, Chromium NSS.
|
||
# @RATIONALE После ручной/автоматической установки сертификатов нужно убедиться
|
||
# что все три HTTP-стека (httpx, requests, Chromium) доверяют корпоративным CA.
|
||
# По отдельности каждый слой имеет свою логику — скрипт собирает всё воедино.
|
||
# @USAGE ./scripts/check_llm_certs.sh [--env-file .env.enterprise-clean] [--target https://lite.ai.rusal.com]
|
||
# @LAYER Infrastructure
|
||
# @EXAMPLE:
|
||
# ./scripts/check_llm_certs.sh
|
||
# ./scripts/check_llm_certs.sh --target https://api.corp.com/v1 --env-file /opt/ss-tools/.env
|
||
# #endregion check_llm_certs
|
||
|
||
set -euo pipefail
|
||
|
||
# ── Colors ──
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
CYAN='\033[0;36m'
|
||
NC='\033[0m' # No Color
|
||
PASS="${GREEN}✅${NC}"
|
||
FAIL="${RED}❌${NC}"
|
||
WARN="${YELLOW}⚠${NC}"
|
||
INFO="${CYAN}ℹ${NC}"
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||
|
||
# ── Defaults ──
|
||
ENV_FILE="${PROJECT_ROOT}/.env.enterprise-clean"
|
||
TARGET_URL="https://lite.ai.rusal.com"
|
||
VERBOSE=false
|
||
|
||
# ── Parse args ──
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--env-file) ENV_FILE="$2"; shift 2 ;;
|
||
--target) TARGET_URL="$2"; shift 2 ;;
|
||
-v|--verbose) VERBOSE=true; shift ;;
|
||
--help|-h) echo "Usage: $0 [--env-file .env] [--target https://...] [-v]"; exit 0 ;;
|
||
*) echo "Unknown: $1"; exit 1 ;;
|
||
esac
|
||
done
|
||
|
||
# ── Load env ──
|
||
if [ -f "$ENV_FILE" ]; then
|
||
echo -e "${INFO} Loading env: ${ENV_FILE}"
|
||
set -a
|
||
source "$ENV_FILE"
|
||
set +a
|
||
else
|
||
echo -e "${WARN} Env file not found: ${ENV_FILE} (using defaults)"
|
||
fi
|
||
|
||
LLM_SSL_VERIFY="${LLM_SSL_VERIFY:-true}"
|
||
CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt"
|
||
NSS_DB_DIR="${HOME:-/root}/.pki/nssdb"
|
||
|
||
echo ""
|
||
echo -e "${CYAN}========================================${NC}"
|
||
echo -e "${CYAN} LLM SSL Certificate Diagnostics${NC}"
|
||
echo -e "${CYAN}========================================${NC}"
|
||
echo ""
|
||
echo "Target URL: ${TARGET_URL}"
|
||
echo "LLM_SSL_VERIFY: ${LLM_SSL_VERIFY}"
|
||
echo "Env file: ${ENV_FILE}"
|
||
echo ""
|
||
|
||
# ══════════════════════════════════════════
|
||
# SECTION 1: System CA Store
|
||
# ══════════════════════════════════════════
|
||
echo -e "${CYAN}── Layer 1: System CA Store (OpenSSL) ──${NC}"
|
||
|
||
# Check bundle exists
|
||
total_certs=0
|
||
if [ -f "$CA_BUNDLE" ]; then
|
||
total_certs=$(awk '/BEGIN CERTIFICATE/{c++} END{print c}' "$CA_BUNDLE" 2>/dev/null || echo "0")
|
||
echo -e " ${PASS} Bundle: ${CA_BUNDLE} (${total_certs} certificates)"
|
||
else
|
||
echo -e " ${FAIL} Bundle NOT FOUND: ${CA_BUNDLE}"
|
||
fi
|
||
|
||
# List custom certs
|
||
echo -n " Custom certs: "
|
||
custom_dir="/usr/local/share/ca-certificates/custom"
|
||
llm_dir="/usr/local/share/ca-certificates/llm"
|
||
found_custom=0
|
||
for dir in "$custom_dir" "$llm_dir"; do
|
||
if [ -d "$dir" ]; then
|
||
for f in "$dir"/*.pem "$dir"/*.crt; do
|
||
[ -f "$f" ] || continue
|
||
found_custom=$((found_custom+1))
|
||
subject="$(openssl x509 -in "$f" -noout -subject 2>/dev/null | head -1 || echo "?")"
|
||
echo -e "\n ${PASS} $(basename "$f") — ${subject}"
|
||
done
|
||
fi
|
||
done
|
||
if [ "$found_custom" -eq 0 ]; then
|
||
echo -e "${WARN} none found (check CERTS_PATH mount and LLM_CA_CERT_URLS)"
|
||
fi
|
||
|
||
# Check symlinks
|
||
echo -n " Hash symlinks: "
|
||
hash_links=$(find /etc/ssl/certs/ -maxdepth 1 -type l -name '[0-9a-f]*.[0-9]' 2>/dev/null | wc -l)
|
||
echo -e "${hash_links} found"
|
||
|
||
# ══════════════════════════════════════════
|
||
# SECTION 2: NSS Database (Chromium)
|
||
# ══════════════════════════════════════════
|
||
echo ""
|
||
echo -e "${CYAN}── Layer 2: NSS Database (Chromium) ──${NC}"
|
||
|
||
if command -v certutil &>/dev/null && [ -d "$NSS_DB_DIR" ]; then
|
||
nss_db="sql:${NSS_DB_DIR}"
|
||
nss_certs=$(certutil -L -d "$nss_db" 2>/dev/null | wc -l || echo "0")
|
||
echo -e " ${PASS} NSS DB: ${NSS_DB_DIR} (${nss_certs} certificates)"
|
||
|
||
# Show custom certs
|
||
certutil -L -d "$nss_db" 2>/dev/null | grep -iE "rusal|llm-|custom-" | while read -r line; do
|
||
echo -e " ${PASS} ${line}"
|
||
done
|
||
elif ! command -v certutil &>/dev/null; then
|
||
echo -e " ${WARN} certutil not installed (libnss3-tools missing)"
|
||
elif [ ! -d "$NSS_DB_DIR" ]; then
|
||
echo -e " ${WARN} NSS DB not found at ${NSS_DB_DIR}"
|
||
fi
|
||
|
||
# ══════════════════════════════════════════
|
||
# SECTION 3: OpenSSL Connectivity Test
|
||
# ══════════════════════════════════════════
|
||
echo ""
|
||
echo -e "${CYAN}── Layer 3: OpenSSL Connectivity ──${NC}"
|
||
|
||
target_host="$(echo "${TARGET_URL}" | sed 's|https://||;s|/.*$||')"
|
||
echo -n " ${target_host}:443 — "
|
||
|
||
if openssl s_client -connect "${target_host}:443" -CAfile "$CA_BUNDLE" -servername "$target_host" </dev/null 2>/dev/null \
|
||
| grep -q "Verify return code: 0"; then
|
||
echo -e "${PASS} SSL handshake OK (verify via system CA)"
|
||
else
|
||
verify_code=$(openssl s_client -connect "${target_host}:443" -CAfile "$CA_BUNDLE" -servername "$target_host" </dev/null 2>&1 \
|
||
| grep "Verify return code" | head -1 || echo "unknown")
|
||
echo -e "${FAIL} ${verify_code}"
|
||
echo -e " ${INFO} Check: missing intermediate CA or wrong CA bundle"
|
||
fi
|
||
|
||
# ══════════════════════════════════════════
|
||
# SECTION 4: Python httpx Test
|
||
# ══════════════════════════════════════════
|
||
echo ""
|
||
echo -e "${CYAN}── Layer 4: Python httpx (AsyncOpenAI) ──${NC}"
|
||
|
||
CA_BUNDLE_PY="${CA_BUNDLE}"
|
||
TARGET_URL_PY="${TARGET_URL}"
|
||
PASS_PY="${PASS}"
|
||
FAIL_PY="${FAIL}"
|
||
INFO_PY="${INFO}"
|
||
WARN_PY="${WARN}"
|
||
export CA_BUNDLE_PY TARGET_URL_PY PASS_PY FAIL_PY INFO_PY WARN_PY
|
||
python3 -c '
|
||
import ssl, sys, os
|
||
|
||
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
|
||
target = os.environ.get("TARGET_URL_PY", "https://lite.ai.rusal.com")
|
||
pass_emoji = os.environ.get("PASS_PY", "✅")
|
||
fail_emoji = os.environ.get("FAIL_PY", "❌")
|
||
info_emoji = os.environ.get("INFO_PY", "ℹ")
|
||
|
||
try:
|
||
ctx = ssl.create_default_context(cafile=cafile)
|
||
print(f" {pass_emoji} SSLContext created with cafile={cafile}")
|
||
print(f" {info_emoji} verify_mode={ctx.verify_mode}")
|
||
except Exception as e:
|
||
print(f" {fail_emoji} SSLContext error: {e}")
|
||
sys.exit(1)
|
||
|
||
try:
|
||
import httpx
|
||
r = httpx.get(target, verify=ctx, timeout=10)
|
||
print(f" {pass_emoji} httpx GET {target} → {r.status_code}")
|
||
except httpx.ConnectError as e:
|
||
print(f" {fail_emoji} httpx connection failed: {e}")
|
||
except Exception as e:
|
||
print(f" {fail_emoji} httpx error: {e}")
|
||
' 2>&1 || echo -e " ${FAIL} Python test failed"
|
||
|
||
# ══════════════════════════════════════════
|
||
# SECTION 5: Python requests Test
|
||
# ══════════════════════════════════════════
|
||
echo ""
|
||
echo -e "${CYAN}── Layer 5: Python requests ──${NC}"
|
||
|
||
python3 -c '
|
||
import os, sys
|
||
target = os.environ.get("TARGET_URL_PY", "https://lite.ai.rusal.com")
|
||
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
|
||
pass_emoji = os.environ.get("PASS_PY", "✅")
|
||
fail_emoji = os.environ.get("FAIL_PY", "❌")
|
||
try:
|
||
import requests
|
||
r = requests.get(target, verify=cafile, timeout=10)
|
||
print(f" {pass_emoji} requests GET {target} → {r.status_code}")
|
||
except requests.exceptions.SSLError as e:
|
||
print(f" {fail_emoji} requests SSL error: {e}")
|
||
except Exception as e:
|
||
print(f" {fail_emoji} requests error: {e}")
|
||
' 2>&1 || echo -e " ${FAIL} Python requests test failed"
|
||
|
||
# ══════════════════════════════════════════
|
||
# SECTION 6: LLM_SSL_VERIFY env var check
|
||
# ══════════════════════════════════════════
|
||
echo ""
|
||
echo -e "${CYAN}── Layer 6: LLM_SSL_VERIFY env var ──${NC}"
|
||
|
||
python3 -c '
|
||
import os
|
||
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
|
||
disabled = raw in ("false", "0", "no", "off")
|
||
warn_emoji = os.environ.get("WARN_PY", "⚠")
|
||
pass_emoji = os.environ.get("PASS_PY", "✅")
|
||
info_emoji = os.environ.get("INFO_PY", "ℹ")
|
||
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
|
||
if disabled:
|
||
print(f" {warn_emoji} LLM_SSL_VERIFY={raw} — SSL verification DISABLED")
|
||
else:
|
||
print(f" {pass_emoji} LLM_SSL_VERIFY={raw} — SSL verification ENABLED")
|
||
import ssl
|
||
if os.path.exists(cafile):
|
||
ctx = ssl.create_default_context(cafile=cafile)
|
||
print(f" {info_emoji} ssl.create_default_context(cafile=...) — OK")
|
||
' 2>&1
|
||
|
||
# ══════════════════════════════════════════
|
||
# SUMMARY
|
||
# ══════════════════════════════════════════
|
||
echo ""
|
||
echo -e "${CYAN}========================================${NC}"
|
||
echo -e "${CYAN} Summary${NC}"
|
||
echo -e "${CYAN}========================================${NC}"
|
||
echo ""
|
||
echo " CA Bundle: ${CA_BUNDLE} (${total_certs} certs)"
|
||
echo " NSS DB: ${NSS_DB_DIR}"
|
||
echo " Target: ${TARGET_URL}"
|
||
echo " LLM_SSL_VERIFY: ${LLM_SSL_VERIFY}"
|
||
echo ""
|
||
echo " If openssl test passes but Python fails — check certifi vs system CA"
|
||
echo " If all tests pass — remove LLM_SSL_VERIFY=false from .env and rerun"
|
||
echo ""
|