Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 75e40deccd | |||
| 9de4f57529 | |||
| 88241bed09 | |||
| d7c7924e62 | |||
| cb1a5a4f13 | |||
| 9186b2abe0 | |||
| 48aa0b27b8 | |||
| 21f3b29699 | |||
| 730ee1b165 | |||
| 2552cb7546 |
@@ -924,24 +924,27 @@ class LLMClient:
|
||||
|
||||
# region LLMClient._get_ssl_verify [TYPE Function]
|
||||
# @PURPOSE Resolve SSL verification flag from environment.
|
||||
# @POST Returns SSLContext with system CA bundle when enabled,
|
||||
# @POST Returns SSLContext with system CA dir when enabled,
|
||||
# False when LLM_SSL_VERIFY env var is "false"/"0"/"no"/"off".
|
||||
# @RATIONALE Возвращаем ssl.SSLContext вместо True, потому что httpx по
|
||||
# умолчанию использует certifi, в котором нет корпоративных CA.
|
||||
# ssl.create_default_context(cafile=...) гарантирует, что используются
|
||||
# сертификаты из системного хранилища (/etc/ssl/certs/ca-certificates.crt),
|
||||
# куда update-ca-certificates и наш fallback добавляют корпоративные CA.
|
||||
# @RATIONALE Используем capath=/etc/ssl/certs/ вместо cafile, потому что
|
||||
# OpenSSL 3.x не использует intermediate CA сертификаты из cafile для
|
||||
# построения цепочки (verify code 20). capath с хеш-симлинками работает
|
||||
# корректно (verify code 0). Оба пути — cafile и capath — указывают на
|
||||
# один и тот же набор сертификатов, но capath правильно обрабатывает
|
||||
# цепочку Root → Policy → Issuing.
|
||||
# @REJECTED verify=<string> отвергнут — httpx 0.28.x депрекейтит строковый
|
||||
# путь в verify=, требует SSLContext.
|
||||
# @REJECTED cafile отвергнут — OpenSSL 3.x не использует intermediate CA
|
||||
# из единого bundle-файла. Только capath с хеш-симлинками даёт code 0.
|
||||
@staticmethod
|
||||
def _get_ssl_verify() -> ssl.SSLContext | bool:
|
||||
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
|
||||
if raw in ("false", "0", "no", "off"):
|
||||
return False
|
||||
ca_path = "/etc/ssl/certs/ca-certificates.crt"
|
||||
if os.path.exists(ca_path):
|
||||
return ssl.create_default_context(cafile=ca_path)
|
||||
# fallback: если системного CA нет (редко), используем дефолтный
|
||||
ca_dir = "/etc/ssl/certs"
|
||||
if os.path.isdir(ca_dir):
|
||||
return ssl.create_default_context(capath=ca_dir)
|
||||
# fallback: если директории нет (редко), используем дефолтный
|
||||
return ssl.create_default_context()
|
||||
# endregion LLMClient._get_ssl_verify
|
||||
|
||||
|
||||
@@ -18,18 +18,21 @@ from typing import Any
|
||||
from ...core.logger import logger
|
||||
|
||||
|
||||
# #region _get_verify [C:1] [TYPE Function] [SEMANTICS translate, ssl, verify]
|
||||
# @BRIEF Resolve SSL verification path from LLM_SSL_VERIFY env var.
|
||||
# @RATIONALE Используем capath=/etc/ssl/certs/ вместо cafile, потому что
|
||||
# OpenSSL 3.x не использует intermediate CA сертификаты из cafile для
|
||||
# построения цепочки (verify code 20). capath с хеш-симлинками работает
|
||||
# корректно (verify code 0).
|
||||
# @REJECTED cafile отвергнут — OpenSSL 3.x не использует intermediate CA
|
||||
# из единого bundle-файла. Только capath с хеш-симлинками даёт code 0.
|
||||
# @POST Returns path to /etc/ssl/certs/ when enabled, False when disabled.
|
||||
def _get_verify() -> str | bool:
|
||||
"""Resolve SSL verify from LLM_SSL_VERIFY env var.
|
||||
|
||||
Returns:
|
||||
- Path to system CA bundle when verification enabled
|
||||
(certifi bundle doesn't include corporate CA certs)
|
||||
- False when LLM_SSL_VERIFY is set to false/0/no/off
|
||||
"""
|
||||
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
|
||||
if raw in ("false", "0", "no", "off"):
|
||||
return False
|
||||
return "/etc/ssl/certs/ca-certificates.crt"
|
||||
return "/etc/ssl/certs/"
|
||||
# #endregion _get_verify
|
||||
|
||||
|
||||
# #region call_openai_compatible [C:3] [TYPE Function] [SEMANTICS translate, llm, http, openai]
|
||||
@@ -200,7 +203,4 @@ def _handle_response_format_fallback(
|
||||
response.encoding = new_response.encoding
|
||||
response.headers = new_response.headers
|
||||
# #endregion _handle_response_format_fallback
|
||||
# #endregion LLMHttpClient
|
||||
response.headers = new_response.headers
|
||||
# #endregion _handle_response_format_fallback
|
||||
# #endregion LLMHttpClient
|
||||
|
||||
@@ -10,18 +10,21 @@ from typing import Any
|
||||
from ...core.logger import logger
|
||||
|
||||
|
||||
# #region _get_verify [C:1] [TYPE Function] [SEMANTICS translate, ssl, verify]
|
||||
# @BRIEF Resolve SSL verification path from LLM_SSL_VERIFY env var.
|
||||
# @RATIONALE Используем capath=/etc/ssl/certs/ вместо cafile, потому что
|
||||
# OpenSSL 3.x не использует intermediate CA сертификаты из cafile для
|
||||
# построения цепочки (verify code 20). capath с хеш-симлинками работает
|
||||
# корректно (verify code 0).
|
||||
# @REJECTED cafile отвергнут — OpenSSL 3.x не использует intermediate CA
|
||||
# из единого bundle-файла. Только capath с хеш-симлинками даёт code 0.
|
||||
# @POST Returns path to /etc/ssl/certs/ when enabled, False when disabled.
|
||||
def _get_verify() -> str | bool:
|
||||
"""Resolve SSL verify from LLM_SSL_VERIFY env var.
|
||||
|
||||
Returns:
|
||||
- Path to system CA bundle when verification enabled
|
||||
(certifi bundle doesn't include corporate CA certs)
|
||||
- False when LLM_SSL_VERIFY is set to false/0/no/off
|
||||
"""
|
||||
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
|
||||
if raw in ("false", "0", "no", "off"):
|
||||
return False
|
||||
return "/etc/ssl/certs/ca-certificates.crt"
|
||||
return "/etc/ssl/certs/"
|
||||
# #endregion _get_verify
|
||||
|
||||
|
||||
class LLMClient:
|
||||
|
||||
@@ -241,8 +241,31 @@ def validate_target_table_schema(
|
||||
safe_schema = (req.target_schema or "public").replace("'", "''")
|
||||
safe_table = req.target_table.replace("'", "''")
|
||||
|
||||
# Нормализуем backend через маппинг (как в get_dialect_from_database)
|
||||
# чтобы правильно обработать "clickhousedb" → "clickhouse",
|
||||
# "greenplum" → "postgresql" и другие варианты
|
||||
_backend_normalize = {
|
||||
"clickhouse": "clickhouse",
|
||||
"clickhousedb": "clickhouse",
|
||||
"postgresql": "postgresql",
|
||||
"greenplum": "postgresql",
|
||||
"mysql": "mysql",
|
||||
"mssql": "mssql",
|
||||
"sqlite": "sqlite",
|
||||
"oracle": "oracle",
|
||||
"snowflake": "snowflake",
|
||||
"bigquery": "bigquery",
|
||||
"redshift": "redshift",
|
||||
"presto": "presto",
|
||||
"trino": "trino",
|
||||
"druid": "druid",
|
||||
"hive": "hive",
|
||||
"spark": "spark",
|
||||
"databricks": "databricks",
|
||||
}
|
||||
normalized_backend = _backend_normalize.get(backend.lower().strip(), backend.lower().strip())
|
||||
# ClickHouse использует system.columns, всё остальное — information_schema.columns
|
||||
is_clickhouse = any(kw in backend.lower() for kw in ("clickhouse", "ch"))
|
||||
is_clickhouse = normalized_backend == "clickhouse"
|
||||
if is_clickhouse:
|
||||
sql = (
|
||||
f"SELECT name, type, default_expression AS data_default "
|
||||
|
||||
@@ -7,15 +7,27 @@ from ...schemas.translate import TranslateJobResponse
|
||||
|
||||
|
||||
# #region _extract_dialect [TYPE Function]
|
||||
# @BRIEF Extract database dialect from Superset backend URI.
|
||||
# @BRIEF Extract database dialect from Superset backend URI or engine name.
|
||||
def _extract_dialect(backend: str) -> str:
|
||||
"""Extract dialect name from a Superset database backend URI."""
|
||||
"""Extract dialect name from a Superset database backend URI or engine name.
|
||||
|
||||
Handles both URI formats (e.g. 'postgresql://...', 'clickhousedb://...')
|
||||
and plain engine names (e.g. 'postgresql', 'clickhousedb') returned by Superset.
|
||||
Normalises known variants like 'clickhousedb' → 'clickhouse'.
|
||||
"""
|
||||
if not backend:
|
||||
return "unknown"
|
||||
try:
|
||||
# Extract scheme from URI or use plain name
|
||||
scheme = backend.split("://")[0]
|
||||
dialect = scheme.split("+")[0]
|
||||
return dialect.lower()
|
||||
raw = dialect.lower()
|
||||
# Normalise known Superset backend variants
|
||||
dialect_map = {
|
||||
"clickhousedb": "clickhouse",
|
||||
"greenplum": "postgresql",
|
||||
}
|
||||
return dialect_map.get(raw, raw)
|
||||
except Exception:
|
||||
return "unknown"
|
||||
# #endregion _extract_dialect
|
||||
|
||||
@@ -18,7 +18,7 @@ from ...core.logger import belief_scope, logger
|
||||
# PostgreSQL/Greenplum dialects that support ON CONFLICT
|
||||
POSTGRESQL_DIALECTS = {"postgresql", "redshift", "greenplum"}
|
||||
# Dialects that use backtick or no quoting
|
||||
CLICKHOUSE_DIALECTS = {"clickhouse"}
|
||||
CLICKHOUSE_DIALECTS = {"clickhouse", "clickhousedb"}
|
||||
|
||||
|
||||
# #region _normalize_timestamp_value [C:2] [TYPE Function] [SEMANTICS translate,sql,timestamp]
|
||||
|
||||
@@ -136,7 +136,7 @@ install_llm_ca_certs() {
|
||||
fi
|
||||
|
||||
local raw_file="${work_dir}/${filename}"
|
||||
local pem_file="${target_dir}/$(echo "$filename" | sed 's/\.[^.]*$//').pem"
|
||||
local pem_file="${target_dir}/$(echo "$filename" | sed 's/\.[^.]*$//').crt"
|
||||
|
||||
echo "[entrypoint] [${index}] Скачивание: ${url}"
|
||||
|
||||
@@ -208,14 +208,14 @@ install_llm_ca_certs() {
|
||||
# Проверяем, попали ли сертификаты в бандл, и создаём хеш-симлинки
|
||||
echo "[entrypoint] --- Валидация установки сертификатов ---"
|
||||
|
||||
for pem in "$target_dir"/*.pem; do
|
||||
[ -f "$pem" ] || continue
|
||||
for cert_file in "$target_dir"/*.crt; do
|
||||
[ -f "$cert_file" ] || continue
|
||||
local cert_name
|
||||
cert_name="$(basename "$pem" .pem)"
|
||||
cert_name="$(basename "$cert_file" .crt)"
|
||||
local cert_subject
|
||||
cert_subject="$(openssl x509 -in "$pem" -noout -subject 2>/dev/null | head -1 || echo "unknown")"
|
||||
cert_subject="$(openssl x509 -in "$cert_file" -noout -subject 2>/dev/null | head -1 || echo "unknown")"
|
||||
local cert_fingerprint
|
||||
cert_fingerprint="$(openssl x509 -in "$pem" -noout -fingerprint -sha256 2>/dev/null | sed 's/.*=//' || echo "")"
|
||||
cert_fingerprint="$(openssl x509 -in "$cert_file" -noout -fingerprint -sha256 2>/dev/null | sed 's/.*=//' || echo "")"
|
||||
|
||||
# Проверяем есть ли в ca-certificates.crt по fingerprint (криптографически уникален)
|
||||
if [ -n "$cert_fingerprint" ] && \
|
||||
@@ -223,26 +223,26 @@ install_llm_ca_certs() {
|
||||
echo "[entrypoint] ✅ ${cert_name} — в ca-certificates.crt (${cert_subject})"
|
||||
else
|
||||
echo "[entrypoint] ⚠ ${cert_name} — НЕ в ca-certificates.crt. Добавляем напрямую..."
|
||||
cat "$pem" >> /etc/ssl/certs/ca-certificates.crt
|
||||
cat "$cert_file" >> /etc/ssl/certs/ca-certificates.crt
|
||||
echo "[entrypoint] ➕ Добавлен напрямую в ca-certificates.crt"
|
||||
fi
|
||||
|
||||
# Создаём хеш-симлинку с поддержкой коллизий (.0, .1, .2...)
|
||||
local hash_val
|
||||
hash_val="$(openssl x509 -in "$pem" -noout -hash 2>/dev/null || true)"
|
||||
hash_val="$(openssl x509 -in "$cert_file" -noout -hash 2>/dev/null || true)"
|
||||
if [ -n "$hash_val" ]; then
|
||||
local suffix=0
|
||||
local link_path="/etc/ssl/certs/${hash_val}.${suffix}"
|
||||
# Ищем первый свободный suffix или symlink уже указывающий на наш файл
|
||||
while [ -L "$link_path" ]; do
|
||||
if [ "$(readlink "$link_path")" = "$pem" ]; then
|
||||
if [ "$(readlink "$link_path")" = "$cert_file" ]; then
|
||||
break 2 # уже есть, ничего не делаем
|
||||
fi
|
||||
suffix=$((suffix + 1))
|
||||
link_path="/etc/ssl/certs/${hash_val}.${suffix}"
|
||||
done
|
||||
if [ ! -L "$link_path" ]; then
|
||||
ln -sf "$pem" "$link_path"
|
||||
ln -sf "$cert_file" "$link_path"
|
||||
echo "[entrypoint] 🔗 Создана симлинка: ${hash_val}.${suffix} → ${cert_name}"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
# @RELATION CALLS -> [backend/src/plugins/llm_analysis/service.py]
|
||||
# @RELATION CALLS -> [backend/src/plugins/translate/_llm_http.py]
|
||||
# @RELATION CALLS -> [backend/src/plugins/translate/preview_llm_client.py]
|
||||
# @RELATION CALLS -> [scripts/check_llm_certs.py]
|
||||
# @RATIONALE ss-tools operates in corporate environments with internal Certificate Authorities.
|
||||
# Three separate HTTP stacks are used: httpx (AsyncOpenAI in llm_analysis plugin),
|
||||
# requests (sync translate plugin), and Playwright Chromium (dashboard screenshots).
|
||||
@@ -23,32 +24,71 @@ Corporate SSL certificates installed via `update-ca-certificates` into `/etc/ssl
|
||||
1. **Python `requests` library** — uses bundled `certifi` CA bundle, not system store
|
||||
2. **Python `httpx` library** — uses `certifi` by default when `verify=True`
|
||||
3. **Playwright Chromium** — uses NSS Shared DB (`~/.pki/nssdb`), not OpenSSL store
|
||||
4. **`openssl s_client`** — hangs without stdin (needs `echo \|` or `input=""`)
|
||||
|
||||
This causes `SSLError: CERTIFICATE_VERIFY_FAILED` and `ERR_CERT_AUTHORITY_INVALID` even after correct system-wide CA installation.
|
||||
|
||||
## Discovery Journey
|
||||
|
||||
### Phase 1: certifi vs system CA (0.1.5)
|
||||
- `requests` and `httpx` use `certifi` bundle, not `/etc/ssl/certs/ca-certificates.crt`
|
||||
- Fix: return system CA path instead of `True` in all `_get_verify()` functions
|
||||
- Implemented in `service.py`, `_llm_http.py`, `preview_llm_client.py`
|
||||
|
||||
### Phase 2: .pem vs .crt extension (0.1.6)
|
||||
- `update-ca-certificates` only processes `.crt` files, silently ignores `.pem`
|
||||
- Downloaded certificates from PKI were saved as `.pem`, skipped by update-ca-certificates
|
||||
- Result: `ca-certificates.crt` was empty (0 certs), hash symlinks existed but bundle was broken
|
||||
- Fix: save downloaded certs with `.crt` extension, not `.pem`
|
||||
|
||||
### Phase 3: cafile vs capath (0.1.7) — KEY DISCOVERY
|
||||
- `openssl s_client -CAfile /etc/ssl/certs/ca-certificates.crt` → code 20 (intermediate CA ignored)
|
||||
- `openssl s_client -CApath /etc/ssl/certs/` → code 0 (chain built correctly)
|
||||
- `ssl.create_default_context(cafile=...)` → SSL error
|
||||
- `ssl.create_default_context(capath=...)` → HTTP 200
|
||||
- `requests.get(verify="/etc/ssl/certs/ca-certificates.crt")` → SSLError
|
||||
- `requests.get(verify="/etc/ssl/certs/")` → HTTP 200
|
||||
|
||||
**Root cause**: OpenSSL 3.x treats non-self-signed certificates in `-CAfile` as trust anchors,
|
||||
NOT as intermediates. `-CApath` (directory with hash symlinks) correctly builds chains using
|
||||
all certificates found. The intermediate CA certs (Policy CA, RGM Issuing CA) are not
|
||||
self-signed, so they are ignored in `-CAfile` but correctly used in `-CApath`.
|
||||
|
||||
Diagnostic matrix (verified on production server 2026-05-28):
|
||||
|
||||
| Method | cafile | capath |
|
||||
|--------|--------|--------|
|
||||
| openssl s_client | code 20 FAIL | code 0 OK |
|
||||
| httpx (SSLContext) | SSLError FAIL | HTTP 200 OK |
|
||||
| requests (verify=) | SSLError FAIL | HTTP 200 OK |
|
||||
|
||||
## Solution
|
||||
|
||||
### Layer 1: System CA Store (OpenSSL)
|
||||
|
||||
- Entrypoint `install_certificates()` copies `.crt/.pem` files from `CERTS_PATH` volume mount
|
||||
- Entrypoint `install_certificates()` copies `.crt` files from `CERTS_PATH` volume mount
|
||||
- Entrypoint `install_llm_ca_certs()` downloads PEM/DER certificates from `LLM_CA_CERT_URLS`
|
||||
- DER is auto-converted to PEM (`openssl x509 -inform DER -outform PEM`)
|
||||
- Certificates are saved with **`.crt` extension** (`.pem` is silently ignored by `update-ca-certificates`)
|
||||
- `update-ca-certificates --fresh` adds them to `/etc/ssl/certs/ca-certificates.crt`
|
||||
- **Fallback**: if `update-ca-certificates` misses certs (Debian Bookworm bug with subdirectories), they are appended to `ca-certificates.crt` with SHA256 fingerprint dedup
|
||||
- **Fallback**: if `update-ca-certificates` misses certs, they are appended to `ca-certificates.crt` with SHA256 fingerprint dedup
|
||||
- Hash symlinks created with collision support: `.0`, `.1`, `.2` suffixes
|
||||
|
||||
### Layer 2: Python HTTP Clients
|
||||
|
||||
| Library | File | Mechanism |
|
||||
|---------|------|-----------|
|
||||
| `httpx` (AsyncOpenAI) | `service.py:LLMClient._get_ssl_verify()` | Returns `ssl.create_default_context(cafile="/etc/ssl/certs/ca-certificates.crt")` |
|
||||
| `requests` (`_llm_http.py`) | `_get_verify()` | Returns `"/etc/ssl/certs/ca-certificates.crt"` (string path) |
|
||||
| `requests` (`preview_llm_client.py`) | `_get_verify()` | Returns `"/etc/ssl/certs/ca-certificates.crt"` (string path) |
|
||||
| Library | File | Mechanism | Status |
|
||||
|---------|------|-----------|--------|
|
||||
| `httpx` (AsyncOpenAI) | `service.py:LLMClient._get_ssl_verify()` | `ssl.create_default_context(capath="/etc/ssl/certs/")` | ✅ Works (0.1.7) |
|
||||
| `requests` (`_llm_http.py`) | `_get_verify()` | `"/etc/ssl/certs/"` (string path to dir) | ✅ Works (0.1.7) |
|
||||
| `requests` (`preview_llm_client.py`) | `_get_verify()` | `"/etc/ssl/certs/"` (string path to dir) | ✅ Works (0.1.7) |
|
||||
|
||||
Key insight: `verify=True` uses certifi, NOT system CA. All three functions now return the system CA path instead of boolean `True`.
|
||||
Key insight: `verify=True` uses certifi, NOT system CA. `verify=<cafile_path>` ignores
|
||||
intermediate CA in OpenSSL 3.x. `verify=<capath_dir>` works correctly.
|
||||
|
||||
### Layer 3: NSS Database (Playwright Chromium)
|
||||
|
||||
- Entrypoint `install_ca_to_nss()` imports PEM certs into `~/.pki/nssdb/` using `certutil`
|
||||
- NSS DB path format: `sql:$HOME/.pki/nssdb` (SQLite prefix required, DBM not supported by Chromium)
|
||||
- Nickname format: `{dir_prefix}-{filename}` (e.g., `llm-UC_RUSAL_Policy_CA`)
|
||||
- Dedup by SHA256 fingerprint (not nickname), preventing duplicate imports
|
||||
- Trust attributes: `"C,,"` (trusted CA for TLS server certs)
|
||||
@@ -58,8 +98,9 @@ Key insight: `verify=True` uses certifi, NOT system CA. All three functions now
|
||||
|
||||
- Env var `LLM_SSL_VERIFY=false` disables SSL verification entirely
|
||||
- Accepted values for false: `false`, `0`, `no`, `off` (case-insensitive)
|
||||
- Default: enabled (returns system CA path / SSLContext)
|
||||
- Default: enabled (returns capath-based SSLContext or path)
|
||||
- Used by all three HTTP client implementations
|
||||
- **WARNING**: `verify=False` is for DIAGNOSTIC USE ONLY. Never leave in production.
|
||||
|
||||
## Key Files
|
||||
|
||||
@@ -67,13 +108,59 @@ Key insight: `verify=True` uses certifi, NOT system CA. All three functions now
|
||||
|------|------|
|
||||
| `docker/backend.Dockerfile` | Installs `libnss3-tools` (certutil), Playwright Chromium |
|
||||
| `docker/backend.entrypoint.sh` | `install_certificates`, `install_llm_ca_certs`, `install_ca_to_nss` |
|
||||
| `plugins/llm_analysis/service.py` | `LLMClient._get_ssl_verify()` → `ssl.create_default_context(cafile=...)` |
|
||||
| `plugins/translate/_llm_http.py` | `_get_verify()` → `"/etc/ssl/certs/ca-certificates.crt"` |
|
||||
| `plugins/translate/preview_llm_client.py` | `_get_verify()` → `"/etc/ssl/certs/ca-certificates.crt"` |
|
||||
| `plugins/llm_analysis/service.py` | `LLMClient._get_ssl_verify()` → `ssl.create_default_context(capath=...)` |
|
||||
| `plugins/translate/_llm_http.py` | `_get_verify()` → `"/etc/ssl/certs/"` |
|
||||
| `plugins/translate/preview_llm_client.py` | `_get_verify()` → `"/etc/ssl/certs/"` |
|
||||
| `scripts/check_llm_certs.py` | Full diagnostic: openssl, httpx, requests, NSS |
|
||||
| `docker-compose.yml` | Passes `LLM_SSL_VERIFY`, `LLM_CA_CERT_URLS` |
|
||||
| `docker-compose.enterprise-clean.yml` | Same as above |
|
||||
| `.env.enterprise-clean` | Default values for both env vars |
|
||||
|
||||
## How to Test Certificate Installation
|
||||
|
||||
### On the server, after container restart:
|
||||
|
||||
```bash
|
||||
docker cp scripts/check_llm_certs.py ss-tools-backend-1:/tmp/
|
||||
|
||||
docker compose -f docker-compose.enterprise-clean.yml \
|
||||
--env-file .env.enterprise-clean exec backend \
|
||||
python3 /tmp/check_llm_certs.py --target https://lite.ai.rusal.com
|
||||
```
|
||||
|
||||
Expected diagnostic matrix:
|
||||
|
||||
```
|
||||
openssl_default: ✅ code 0
|
||||
openssl_capath: ✅ code 0
|
||||
openssl_cafile: ❌ code 20 (expected — OpenSSL 3.x limitation)
|
||||
httpx_capath: ✅ HTTP 200
|
||||
httpx_cafile: ❌ SSL error (expected)
|
||||
requests_capath: ✅ HTTP 200
|
||||
requests_cafile: ❌ SSL error (expected)
|
||||
```
|
||||
|
||||
## Manually testing chain validity
|
||||
|
||||
```bash
|
||||
# 1. Check individual certs
|
||||
openssl x509 -in /usr/local/share/ca-certificates/custom/RUSAL_ROOT.crt -noout -subject -issuer
|
||||
|
||||
# 2. Verify full chain
|
||||
openssl verify -CAfile /usr/local/share/ca-certificates/custom/RUSAL_ROOT.crt \
|
||||
-untrusted /usr/local/share/ca-certificates/llm/UC_RUSAL_Policy_CA.crt \
|
||||
/usr/local/share/ca-certificates/llm/UC_RUSAL_RGM_Issuing_CA.crt
|
||||
# → OK
|
||||
|
||||
# 3. Connect with capath
|
||||
echo | openssl s_client -connect lite.ai.rusal.com:443 \
|
||||
-servername lite.ai.rusal.com \
|
||||
-CApath /etc/ssl/certs/
|
||||
|
||||
# 4. Verify certificate count
|
||||
grep -c "BEGIN CERTIFICATE" /etc/ssl/certs/ca-certificates.crt
|
||||
```
|
||||
|
||||
## Discovered Findings
|
||||
|
||||
### Finding 1: certifi vs system CA
|
||||
@@ -82,30 +169,75 @@ Key insight: `verify=True` uses certifi, NOT system CA. All three functions now
|
||||
### Finding 2: Three separate HTTP stacks
|
||||
ss-tools has three LLM HTTP clients (httpx for async, requests for sync, Playwright for screenshots), each with independent CA trust configuration.
|
||||
|
||||
### Finding 3: update-ca-certificates inconsistency
|
||||
On Debian Bookworm (python:3.11-slim), `update-ca-certificates` occasionally skips symlinks for PEM files in subdirectories (`/usr/local/share/ca-certificates/custom/`, `llm/`), requiring a manual `cat >> ca-certificates.crt` fallback and explicit hash symlink creation.
|
||||
### Finding 3: .pem extension ignored by update-ca-certificates
|
||||
On Debian Bookworm (python:3.11-slim), `update-ca-certificates` only processes `.crt` files.
|
||||
Files with `.pem` extension in `/usr/local/share/ca-certificates/` are silently skipped.
|
||||
This left `ca-certificates.crt` empty (0 certs) while hash symlinks existed.
|
||||
Fix: save downloaded certificates with `.crt` extension.
|
||||
|
||||
### Finding 4: DER format from corporate PKI
|
||||
Corporate PKI servers (`pki.rusal.com`) often serve certificates in DER (binary) format. `update-ca-certificates` requires PEM. Auto-detection and conversion (`openssl x509 -inform DER -outform PEM`) is essential.
|
||||
Corporate PKI servers (`pki.rusal.com`) often serve certificates in DER (binary) format.
|
||||
`update-ca-certificates` requires PEM. Auto-detection and conversion
|
||||
(`openssl x509 -inform DER -outform PEM`) is essential.
|
||||
|
||||
### Finding 5: Chicken-and-egg TLS bootstrap
|
||||
Downloading a CA certificate from a PKI server that uses the same CA causes a TLS verification loop. Entrypoint uses `curl --insecure` as fallback when initial `curl` fails with TLS error.
|
||||
Downloading a CA certificate from a PKI server that uses the same CA causes a TLS
|
||||
verification loop. Entrypoint uses `curl --insecure` as fallback when initial `curl`
|
||||
fails with TLS error.
|
||||
|
||||
### Finding 6: NSS DB format
|
||||
Chromium uses NSS Shared DB in `~/.pki/nssdb/`. The `sql:` prefix is required for `certutil` to use the SQLite format. Without it, certutil defaults to the legacy DBM format which Chromium may not read.
|
||||
Chromium uses NSS Shared DB in `~/.pki/nssdb/`. The `sql:` prefix is required for
|
||||
`certutil` to use the SQLite format. Without it, certutil defaults to the legacy
|
||||
DBM format which Chromium may not read.
|
||||
|
||||
### Finding 7: OpenSSL 3.x cafile vs capath (CRITICAL)
|
||||
OpenSSL 3.x treats non-self-signed certificates in `-CAfile` as trust anchors,
|
||||
NOT as intermediates. This means intermediate CA certificates (Policy CA, RGM Issuing CA)
|
||||
are ignored when using `-CAfile /etc/ssl/certs/ca-certificates.crt`, producing
|
||||
verify code 20. Using `-CApath /etc/ssl/certs/` (directory with hash symlinks)
|
||||
correctly builds the full chain, producing verify code 0.
|
||||
|
||||
This affects ALL Python libraries:
|
||||
- `ssl.create_default_context(cafile=...)` → calls OpenSSL's `SSL_CTX_load_verify_locations`
|
||||
with the file, which exhibits the same limitation
|
||||
- `ssl.create_default_context(capath=...)` → works correctly
|
||||
- `requests.get(verify="/path/to/file")` → fails (uses cafile internally)
|
||||
- `requests.get(verify="/path/to/dir/")` → works (uses capath internally)
|
||||
|
||||
### Finding 8: openssl s_client hangs without stdin
|
||||
`openssl s_client` waits for input after TLS handshake. When run via
|
||||
`subprocess.run(cmd, capture_output=True)`, it hangs until timeout.
|
||||
Fix: pass `input=""` or `echo | openssl s_client ...`.
|
||||
|
||||
## Deploy
|
||||
|
||||
```bash
|
||||
# On target server:
|
||||
xz -dc ss-tools-backend.0.1.6.tar.xz | docker load
|
||||
xz -dc ss-tools-frontend.0.1.6.tar.xz | docker load
|
||||
xz -dc ss-tools-backend.0.1.7.tar.xz | docker load
|
||||
xz -dc ss-tools-frontend.0.1.7.tar.xz | docker load
|
||||
|
||||
# Enable capath-based verification (remove LLM_SSL_VERIFY=false)
|
||||
sed -i '/LLM_SSL_VERIFY=false/d' .env.enterprise-clean
|
||||
|
||||
docker compose -f docker-compose.enterprise-clean.yml \
|
||||
--env-file .env.enterprise-clean down
|
||||
|
||||
docker compose -f docker-compose.enterprise-clean.yml \
|
||||
--env-file .env.enterprise-clean up -d
|
||||
|
||||
# Verify
|
||||
docker cp scripts/check_llm_certs.py ss-tools-backend-1:/tmp/
|
||||
docker compose -f docker-compose.enterprise-clean.yml \
|
||||
--env-file .env.enterprise-clean exec backend \
|
||||
python3 /tmp/check_llm_certs.py --target https://lite.ai.rusal.com
|
||||
```
|
||||
|
||||
## Version History
|
||||
|
||||
| Version | Changes |
|
||||
|---------|---------|
|
||||
| 0.1.5 | Initial SSL support: `LLM_SSL_VERIFY`, `_format_connection_error()`, CA download via URL, NSS import |
|
||||
| 0.1.6 | QA fixes: fingerprint dedup, hash symlink collision, NSS collision, DER→PEM error handling, chicken-and-egg TLS, nullglob, translate plugin SSL |
|
||||
| 0.1.7 | **capath instead of cafile**: OpenSSL 3.x cafile limitation discovered and fixed. `.crt` extension for downloaded certs. Diagnostic script rewritten. |
|
||||
|
||||
# [/DEF:ADR-0009:ADR]
|
||||
|
||||
225
scripts/check_llm_certs.py
Executable file
225
scripts/check_llm_certs.py
Executable file
@@ -0,0 +1,225 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
SSL Certificate Diagnostics for ss-tools.
|
||||
|
||||
Tests all 4 layers with the actual libraries used in production.
|
||||
Key finding from ADR-0009: use capath, NOT cafile — OpenSSL 3.x
|
||||
ignores intermediate CA certs in cafile (verify code 20) but
|
||||
correctly builds chains with capath (verify code 0).
|
||||
|
||||
Output: human-readable + JSON summary with AI diagnosis hint.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# ── Constants ──
|
||||
CA_BUNDLE = Path("/etc/ssl/certs/ca-certificates.crt")
|
||||
CA_DIR = Path("/etc/ssl/certs")
|
||||
CUSTOM_DIRS = [
|
||||
Path("/usr/local/share/ca-certificates/custom"),
|
||||
Path("/usr/local/share/ca-certificates/llm"),
|
||||
]
|
||||
NSS_DB = f"sql:{Path(os.environ.get('HOME', '/root')) / '.pki' / 'nssdb'}"
|
||||
|
||||
results: dict = {}
|
||||
|
||||
|
||||
# ── Helpers ──
|
||||
|
||||
def _run(cmd: list[str], timeout: int = 15, input_data: str = "") -> tuple[int, str, str]:
|
||||
"""Run subprocess with optional stdin."""
|
||||
try:
|
||||
r = subprocess.run(cmd, input=input_data, capture_output=True, text=True, timeout=timeout)
|
||||
return r.returncode, r.stdout, r.stderr
|
||||
except FileNotFoundError:
|
||||
return -1, "", "command not found"
|
||||
except subprocess.TimeoutExpired:
|
||||
return -2, "", f"timeout ({timeout}s)"
|
||||
|
||||
|
||||
def _count_certs(path: str) -> int:
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
return f.read().count(b"-----BEGIN CERTIFICATE-----")
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
# ── Openssl tests ──
|
||||
|
||||
def test_openssl(label: str, extra_args: list[str]) -> None:
|
||||
"""Test openssl s_client with empty stdin to prevent hang."""
|
||||
cmd = ["openssl", "s_client", "-connect", f"{host}:443",
|
||||
"-servername", host, "-verify_return_error"] + extra_args
|
||||
rc, out, err = _run(cmd, timeout=20, input_data="")
|
||||
for line in out.split("\n"):
|
||||
if "Verify return code" in line:
|
||||
ok = "0 (ok)" in line
|
||||
results[f"openssl_{label}"] = f"{'OK' if ok else 'FAIL'}: {line.strip()}"
|
||||
print(f" openssl_{label}: {'✅' if ok else '❌'} {line.strip()}")
|
||||
return
|
||||
results[f"openssl_{label}"] = f"ERROR: {err.strip() or 'no output'}"
|
||||
print(f" openssl_{label}: ❌ {results[f'openssl_{label}']}")
|
||||
|
||||
|
||||
# ── Python library tests ──
|
||||
|
||||
def test_httpx(key: str, verify) -> None:
|
||||
try:
|
||||
import httpx
|
||||
r = httpx.get(TARGET_URL, verify=verify, timeout=10)
|
||||
results[key] = "OK"
|
||||
print(f" {key}: ✅ status={r.status_code}")
|
||||
except Exception as e:
|
||||
results[key] = f"FAIL: {type(e).__name__}"
|
||||
print(f" {key}: ❌ {type(e).__name__}")
|
||||
|
||||
|
||||
def test_requests(key: str, verify) -> None:
|
||||
try:
|
||||
import requests as req
|
||||
r = req.get(TARGET_URL, verify=verify, timeout=10)
|
||||
results[key] = "OK"
|
||||
print(f" {key}: ✅ status={r.status_code}")
|
||||
except Exception as e:
|
||||
results[key] = f"FAIL: {type(e).__name__}"
|
||||
print(f" {key}: ❌ {type(e).__name__}")
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════
|
||||
# MAIN
|
||||
# ══════════════════════════════════════════════
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--target", default=os.environ.get("TARGET_URL", "https://lite.ai.rusal.com"))
|
||||
args = parser.parse_args()
|
||||
TARGET_URL = args.target
|
||||
host = TARGET_URL.replace("https://", "").split("/")[0]
|
||||
LLM_SSL_VERIFY = os.environ.get("LLM_SSL_VERIFY", "true")
|
||||
|
||||
# ── 1. System ──
|
||||
print("\n=== 1. System ===")
|
||||
print(f" target={TARGET_URL} host={host}")
|
||||
print(f" platform={platform.platform()}")
|
||||
print(f" python={sys.version.split()[0]}")
|
||||
for lib in ("httpx", "requests", "openai", "certifi"):
|
||||
try:
|
||||
mod = __import__(lib)
|
||||
v = getattr(mod, "__version__", "installed")
|
||||
p = getattr(mod, "__file__", "")
|
||||
if lib == "certifi":
|
||||
print(f" {lib}={v} path={mod.where()} certs={_count_certs(mod.where())}")
|
||||
else:
|
||||
print(f" {lib}={v}")
|
||||
except ImportError:
|
||||
print(f" {lib}=NOT_INSTALLED")
|
||||
|
||||
# ── 2. System CA Store ──
|
||||
print("\n=== 2. System CA Store ===")
|
||||
bundle_certs = _count_certs(str(CA_BUNDLE)) if CA_BUNDLE.exists() else 0
|
||||
hash_links = len(list(CA_DIR.glob("[0-9a-f]*.[0-9]"))) if CA_DIR.is_dir() else 0
|
||||
print(f" bundle={CA_BUNDLE} certs={bundle_certs}")
|
||||
print(f" capath={CA_DIR} hash_links={hash_links}")
|
||||
|
||||
print(" custom certs:")
|
||||
for d in CUSTOM_DIRS:
|
||||
if not d.is_dir():
|
||||
continue
|
||||
for f in sorted(d.iterdir()):
|
||||
if f.suffix in (".pem", ".crt") and f.is_file():
|
||||
rc, out, _ = _run(["openssl", "x509", "-in", str(f), "-noout",
|
||||
"-subject", "-issuer"], timeout=3)
|
||||
if rc != 0:
|
||||
rc, out, _ = _run(["openssl", "x509", "-in", str(f), "-inform",
|
||||
"DER", "-noout", "-subject"], timeout=3)
|
||||
subj = out.replace("subject=", "").strip() if rc == 0 else "UNREADABLE"
|
||||
print(f" {f.parent.name}/{f.name} subj={subj[:60]}")
|
||||
|
||||
# ── 3. OpenSSL s_client (3 variants) ──
|
||||
print("\n=== 3. OpenSSL s_client ===")
|
||||
test_openssl("default", [])
|
||||
test_openssl("cafile", ["-CAfile", str(CA_BUNDLE)])
|
||||
test_openssl("capath", ["-CApath", str(CA_DIR)])
|
||||
|
||||
# ── 4. Python httpx ──
|
||||
print("\n=== 4. Python httpx (LLMClient) ===")
|
||||
try:
|
||||
import httpx
|
||||
test_httpx("httpx_True", verify=True)
|
||||
try:
|
||||
import ssl
|
||||
ctx_cafile = ssl.create_default_context(cafile=str(CA_BUNDLE)) if CA_BUNDLE.exists() else ssl.create_default_context()
|
||||
ctx_capath = ssl.create_default_context(capath=str(CA_DIR)) if CA_DIR.is_dir() else ssl.create_default_context()
|
||||
test_httpx("httpx_cafile", verify=ctx_cafile)
|
||||
test_httpx("httpx_capath", verify=ctx_capath)
|
||||
except Exception as e:
|
||||
print(f" SSLContext: ❌ {e}")
|
||||
test_httpx("httpx_False", verify=False)
|
||||
except ImportError:
|
||||
print(" httpx: NOT_INSTALLED")
|
||||
|
||||
# ── 5. Python requests ──
|
||||
print("\n=== 5. Python requests (translate plugin) ===")
|
||||
try:
|
||||
import requests as req
|
||||
test_requests("requests_True", verify=True)
|
||||
test_requests("requests_cafile", verify=str(CA_BUNDLE))
|
||||
test_requests("requests_capath", verify=str(CA_DIR))
|
||||
test_requests("requests_False", verify=False)
|
||||
except ImportError:
|
||||
print(" requests: NOT_INSTALLED")
|
||||
|
||||
# ── 6. NSS Chromium ──
|
||||
print("\n=== 6. NSS Chromium ===")
|
||||
rc, out, err = _run(["certutil", "-L", "-d", NSS_DB], timeout=5)
|
||||
if rc == 0:
|
||||
lines = [l for l in out.split("\n") if l.strip() and "Certificate Nickname" not in l]
|
||||
print(f" DB={NSS_DB} certs={len(lines)}")
|
||||
for l in lines:
|
||||
if any(x in l.lower() for x in ("rusal", "llm-", "custom-")):
|
||||
print(f" {l}")
|
||||
else:
|
||||
print(f" NSS: {err.strip() or 'unavailable'}")
|
||||
|
||||
# ── 7. LLM_SSL_VERIFY env ──
|
||||
print("\n=== 7. LLM_SSL_VERIFY ===")
|
||||
print(f" LLM_SSL_VERIFY={LLM_SSL_VERIFY} disabled={LLM_SSL_VERIFY.strip().lower() in ('false','0','no','off')}")
|
||||
|
||||
# ── 8. JSON Summary ──
|
||||
print("\n=== 8. JSON Summary ===")
|
||||
summary = {k: results[k] for k in [
|
||||
"openssl_default", "openssl_cafile", "openssl_capath",
|
||||
"httpx_True", "httpx_cafile", "httpx_capath", "httpx_False",
|
||||
"requests_True", "requests_cafile", "requests_capath", "requests_False",
|
||||
] if k in results}
|
||||
|
||||
# Diagnosis
|
||||
diag = []
|
||||
for k in ("openssl_default", "openssl_cafile", "openssl_capath"):
|
||||
v = str(summary.get(k, ""))
|
||||
diag.append(f"{k}={'OK' if v.startswith('OK') else 'FAIL'}")
|
||||
for k in sorted(summary):
|
||||
if k.startswith("httpx") or k.startswith("requests"):
|
||||
v = str(summary.get(k, ""))
|
||||
diag.append(f"{k}={'OK' if v.startswith('OK') else 'FAIL'}")
|
||||
|
||||
if str(results.get("httpx_capath", "")).startswith("OK") and not str(results.get("httpx_cafile", "")).startswith("OK"):
|
||||
diag.append("VERDICT=capath_works_cafile_fails_use_capath")
|
||||
elif str(summary.get("httpx_False", "")).startswith("OK"):
|
||||
diag.append("VERDICT=all_ssl_fails_verify_disabled_works")
|
||||
elif str(summary.get("openssl_default", "")).startswith("OK"):
|
||||
diag.append("VERDICT=all_ok")
|
||||
else:
|
||||
diag.append("VERDICT=needs_investigation")
|
||||
|
||||
summary["diagnosis"] = "; ".join(diag)
|
||||
print(json.dumps(summary, indent=2))
|
||||
@@ -1,251 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# #region check_llm_certs [C:4] [TYPE Module] [SEMANTICS ssl, certs, diagnostics, openssl, nss]
|
||||
# @BRIEF Проверка установки корпоративных SSL-сертификатов для LLM-провайдеров.
|
||||
# Читает конфигурацию из .env.enterprise-clean, проверяет все 4 слоя:
|
||||
# OpenSSL store, httpx, requests, Chromium NSS.
|
||||
# @RATIONALE После ручной/автоматической установки сертификатов нужно убедиться
|
||||
# что все три HTTP-стека (httpx, requests, Chromium) доверяют корпоративным CA.
|
||||
# По отдельности каждый слой имеет свою логику — скрипт собирает всё воедино.
|
||||
# @USAGE ./scripts/check_llm_certs.sh [--env-file .env.enterprise-clean] [--target https://lite.ai.rusal.com]
|
||||
# @LAYER Infrastructure
|
||||
# @EXAMPLE:
|
||||
# ./scripts/check_llm_certs.sh
|
||||
# ./scripts/check_llm_certs.sh --target https://api.corp.com/v1 --env-file /opt/ss-tools/.env
|
||||
# #endregion check_llm_certs
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── Colors ──
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
PASS="${GREEN}✅${NC}"
|
||||
FAIL="${RED}❌${NC}"
|
||||
WARN="${YELLOW}⚠${NC}"
|
||||
INFO="${CYAN}ℹ${NC}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
# ── Defaults ──
|
||||
ENV_FILE="${PROJECT_ROOT}/.env.enterprise-clean"
|
||||
TARGET_URL="https://lite.ai.rusal.com"
|
||||
VERBOSE=false
|
||||
|
||||
# ── Parse args ──
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env-file) ENV_FILE="$2"; shift 2 ;;
|
||||
--target) TARGET_URL="$2"; shift 2 ;;
|
||||
-v|--verbose) VERBOSE=true; shift ;;
|
||||
--help|-h) echo "Usage: $0 [--env-file .env] [--target https://...] [-v]"; exit 0 ;;
|
||||
*) echo "Unknown: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ── Load env ──
|
||||
if [ -f "$ENV_FILE" ]; then
|
||||
echo -e "${INFO} Loading env: ${ENV_FILE}"
|
||||
set -a
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
else
|
||||
echo -e "${WARN} Env file not found: ${ENV_FILE} (using defaults)"
|
||||
fi
|
||||
|
||||
LLM_SSL_VERIFY="${LLM_SSL_VERIFY:-true}"
|
||||
CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt"
|
||||
NSS_DB_DIR="${HOME:-/root}/.pki/nssdb"
|
||||
|
||||
echo ""
|
||||
echo -e "${CYAN}========================================${NC}"
|
||||
echo -e "${CYAN} LLM SSL Certificate Diagnostics${NC}"
|
||||
echo -e "${CYAN}========================================${NC}"
|
||||
echo ""
|
||||
echo "Target URL: ${TARGET_URL}"
|
||||
echo "LLM_SSL_VERIFY: ${LLM_SSL_VERIFY}"
|
||||
echo "Env file: ${ENV_FILE}"
|
||||
echo ""
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 1: System CA Store
|
||||
# ══════════════════════════════════════════
|
||||
echo -e "${CYAN}── Layer 1: System CA Store (OpenSSL) ──${NC}"
|
||||
|
||||
# Check bundle exists
|
||||
total_certs=0
|
||||
if [ -f "$CA_BUNDLE" ]; then
|
||||
total_certs=$(awk '/BEGIN CERTIFICATE/{c++} END{print c}' "$CA_BUNDLE" 2>/dev/null || echo "0")
|
||||
echo -e " ${PASS} Bundle: ${CA_BUNDLE} (${total_certs} certificates)"
|
||||
else
|
||||
echo -e " ${FAIL} Bundle NOT FOUND: ${CA_BUNDLE}"
|
||||
fi
|
||||
|
||||
# List custom certs
|
||||
echo -n " Custom certs: "
|
||||
custom_dir="/usr/local/share/ca-certificates/custom"
|
||||
llm_dir="/usr/local/share/ca-certificates/llm"
|
||||
found_custom=0
|
||||
for dir in "$custom_dir" "$llm_dir"; do
|
||||
if [ -d "$dir" ]; then
|
||||
for f in "$dir"/*.pem "$dir"/*.crt; do
|
||||
[ -f "$f" ] || continue
|
||||
found_custom=$((found_custom+1))
|
||||
subject="$(openssl x509 -in "$f" -noout -subject 2>/dev/null | head -1 || echo "?")"
|
||||
echo -e "\n ${PASS} $(basename "$f") — ${subject}"
|
||||
done
|
||||
fi
|
||||
done
|
||||
if [ "$found_custom" -eq 0 ]; then
|
||||
echo -e "${WARN} none found (check CERTS_PATH mount and LLM_CA_CERT_URLS)"
|
||||
fi
|
||||
|
||||
# Check symlinks
|
||||
echo -n " Hash symlinks: "
|
||||
hash_links=$(find /etc/ssl/certs/ -maxdepth 1 -type l -name '[0-9a-f]*.[0-9]' 2>/dev/null | wc -l)
|
||||
echo -e "${hash_links} found"
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 2: NSS Database (Chromium)
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 2: NSS Database (Chromium) ──${NC}"
|
||||
|
||||
if command -v certutil &>/dev/null && [ -d "$NSS_DB_DIR" ]; then
|
||||
nss_db="sql:${NSS_DB_DIR}"
|
||||
nss_certs=$(certutil -L -d "$nss_db" 2>/dev/null | wc -l || echo "0")
|
||||
echo -e " ${PASS} NSS DB: ${NSS_DB_DIR} (${nss_certs} certificates)"
|
||||
|
||||
# Show custom certs
|
||||
certutil -L -d "$nss_db" 2>/dev/null | grep -iE "rusal|llm-|custom-" | while read -r line; do
|
||||
echo -e " ${PASS} ${line}"
|
||||
done
|
||||
elif ! command -v certutil &>/dev/null; then
|
||||
echo -e " ${WARN} certutil not installed (libnss3-tools missing)"
|
||||
elif [ ! -d "$NSS_DB_DIR" ]; then
|
||||
echo -e " ${WARN} NSS DB not found at ${NSS_DB_DIR}"
|
||||
fi
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 3: OpenSSL Connectivity Test
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 3: OpenSSL Connectivity ──${NC}"
|
||||
|
||||
target_host="$(echo "${TARGET_URL}" | sed 's|https://||;s|/.*$||')"
|
||||
echo -n " ${target_host}:443 — "
|
||||
|
||||
if openssl s_client -connect "${target_host}:443" -CAfile "$CA_BUNDLE" -servername "$target_host" </dev/null 2>/dev/null \
|
||||
| grep -q "Verify return code: 0"; then
|
||||
echo -e "${PASS} SSL handshake OK (verify via system CA)"
|
||||
else
|
||||
verify_code=$(openssl s_client -connect "${target_host}:443" -CAfile "$CA_BUNDLE" -servername "$target_host" </dev/null 2>&1 \
|
||||
| grep "Verify return code" | head -1 || echo "unknown")
|
||||
echo -e "${FAIL} ${verify_code}"
|
||||
echo -e " ${INFO} Check: missing intermediate CA or wrong CA bundle"
|
||||
fi
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 4: Python httpx Test
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 4: Python httpx (AsyncOpenAI) ──${NC}"
|
||||
|
||||
CA_BUNDLE_PY="${CA_BUNDLE}"
|
||||
TARGET_URL_PY="${TARGET_URL}"
|
||||
PASS_PY="${PASS}"
|
||||
FAIL_PY="${FAIL}"
|
||||
INFO_PY="${INFO}"
|
||||
WARN_PY="${WARN}"
|
||||
export CA_BUNDLE_PY TARGET_URL_PY PASS_PY FAIL_PY INFO_PY WARN_PY
|
||||
python3 -c '
|
||||
import ssl, sys, os
|
||||
|
||||
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
|
||||
target = os.environ.get("TARGET_URL_PY", "https://lite.ai.rusal.com")
|
||||
pass_emoji = os.environ.get("PASS_PY", "✅")
|
||||
fail_emoji = os.environ.get("FAIL_PY", "❌")
|
||||
info_emoji = os.environ.get("INFO_PY", "ℹ")
|
||||
|
||||
try:
|
||||
ctx = ssl.create_default_context(cafile=cafile)
|
||||
print(f" {pass_emoji} SSLContext created with cafile={cafile}")
|
||||
print(f" {info_emoji} verify_mode={ctx.verify_mode}")
|
||||
except Exception as e:
|
||||
print(f" {fail_emoji} SSLContext error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
import httpx
|
||||
r = httpx.get(target, verify=ctx, timeout=10)
|
||||
print(f" {pass_emoji} httpx GET {target} → {r.status_code}")
|
||||
except httpx.ConnectError as e:
|
||||
print(f" {fail_emoji} httpx connection failed: {e}")
|
||||
except Exception as e:
|
||||
print(f" {fail_emoji} httpx error: {e}")
|
||||
' 2>&1 || echo -e " ${FAIL} Python test failed"
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 5: Python requests Test
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 5: Python requests ──${NC}"
|
||||
|
||||
python3 -c '
|
||||
import os, sys
|
||||
target = os.environ.get("TARGET_URL_PY", "https://lite.ai.rusal.com")
|
||||
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
|
||||
pass_emoji = os.environ.get("PASS_PY", "✅")
|
||||
fail_emoji = os.environ.get("FAIL_PY", "❌")
|
||||
try:
|
||||
import requests
|
||||
r = requests.get(target, verify=cafile, timeout=10)
|
||||
print(f" {pass_emoji} requests GET {target} → {r.status_code}")
|
||||
except requests.exceptions.SSLError as e:
|
||||
print(f" {fail_emoji} requests SSL error: {e}")
|
||||
except Exception as e:
|
||||
print(f" {fail_emoji} requests error: {e}")
|
||||
' 2>&1 || echo -e " ${FAIL} Python requests test failed"
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SECTION 6: LLM_SSL_VERIFY env var check
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}── Layer 6: LLM_SSL_VERIFY env var ──${NC}"
|
||||
|
||||
python3 -c '
|
||||
import os
|
||||
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
|
||||
disabled = raw in ("false", "0", "no", "off")
|
||||
warn_emoji = os.environ.get("WARN_PY", "⚠")
|
||||
pass_emoji = os.environ.get("PASS_PY", "✅")
|
||||
info_emoji = os.environ.get("INFO_PY", "ℹ")
|
||||
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
|
||||
if disabled:
|
||||
print(f" {warn_emoji} LLM_SSL_VERIFY={raw} — SSL verification DISABLED")
|
||||
else:
|
||||
print(f" {pass_emoji} LLM_SSL_VERIFY={raw} — SSL verification ENABLED")
|
||||
import ssl
|
||||
if os.path.exists(cafile):
|
||||
ctx = ssl.create_default_context(cafile=cafile)
|
||||
print(f" {info_emoji} ssl.create_default_context(cafile=...) — OK")
|
||||
' 2>&1
|
||||
|
||||
# ══════════════════════════════════════════
|
||||
# SUMMARY
|
||||
# ══════════════════════════════════════════
|
||||
echo ""
|
||||
echo -e "${CYAN}========================================${NC}"
|
||||
echo -e "${CYAN} Summary${NC}"
|
||||
echo -e "${CYAN}========================================${NC}"
|
||||
echo ""
|
||||
echo " CA Bundle: ${CA_BUNDLE} (${total_certs} certs)"
|
||||
echo " NSS DB: ${NSS_DB_DIR}"
|
||||
echo " Target: ${TARGET_URL}"
|
||||
echo " LLM_SSL_VERIFY: ${LLM_SSL_VERIFY}"
|
||||
echo ""
|
||||
echo " If openssl test passes but Python fails — check certifi vs system CA"
|
||||
echo " If all tests pass — remove LLM_SSL_VERIFY=false from .env and rerun"
|
||||
echo ""
|
||||
Reference in New Issue
Block a user