10 Commits
0.1.6 ... 0.1.7

Author SHA1 Message Date
75e40deccd fix: .pem -> .crt extension for downloaded certs, remove stale check_llm_certs.txt 2026-05-29 14:32:45 +03:00
9de4f57529 fix(translate): normalize backend dialect detection with explicit mapping
- Replace fragile substring check (any(kw in backend for kw in ('clickhouse', 'ch')))
  with a proper _backend_normalize mapping + exact comparison.
  Fixes misdetection of Greenplum (postgresql) and other backends.
- Add 'clickhousedb' to CLICKHOUSE_DIALECTS so SQL generator uses
  backtick quoting and correct INSERT strategy for ClickHouse.
- Normalize _extract_dialect to map 'clickhousedb' -> 'clickhouse'
  and 'greenplum' -> 'postgresql'.
2026-05-29 14:32:00 +03:00
88241bed09 docs: update ADR-0009 with complete discovery journey
Added Phase 1-3 discovery timeline (certifi → .pem→.crt → cafile→capath),
OpenSSL 3.x cafile limitation analysis, diagnostic matrix,
test commands, version history. All 8 findings documented.
2026-05-29 14:30:33 +03:00
d7c7924e62 fix: QA findings — GRACE contracts for _get_verify, fix _llm_http.py structure
- Added #region/#endregion contracts with @RATIONALE/@REJECTED to
  _get_verify() in both translate plugins (P1 HIGH, P2 MEDIUM)
- Removed orphaned duplicates #endregion in _llm_http.py (P1 HIGH, P6 HIGH)
2026-05-29 11:10:11 +03:00
cb1a5a4f13 fix: capath instead of cafile for all LLM clients
OpenSSL 3.x ignores intermediate CA certs in -CAfile (verify code 20)
but correctly builds chains with -CApath (verify code 0).
All three clients now use capath:
  - service.py: ssl.create_default_context(capath=...)
  - _llm_http.py: verify='/etc/ssl/certs/'
  - preview_llm_client.py: verify='/etc/ssl/certs/'
check_llm_certs.py rewritten for clarity.
2026-05-29 11:02:48 +03:00
9186b2abe0 fix: check_llm_certs.py — fixes from user review
- openssl s_client: stdin=EOF to prevent hang (input='')
- argparse for --target CLI argument
- in_bundle check: awk+openssl per-cert fingerprint comparison
- NamedTemporaryFile: mode='wb' for binary writes
- Added httpx_capath and requests_cafile_capath tests
- Graceful handling of empty ca-certificates.crt
2026-05-28 23:22:53 +03:00
48aa0b27b8 fix: check_llm_certs.py — handle DER files, openssl timeout 20s, binary PEM reads 2026-05-28 23:10:18 +03:00
21f3b29699 fix: check_llm_certs.py — clean main() structure, safe imports 2026-05-28 22:58:37 +03:00
730ee1b165 feat: rewrite SSL diag in Python — tests all real libraries
Tests all 4 SSL methods with actual project libraries:
  - httpx (verify=True/False/SSLContext cafile/cafile+capath)
  - requests (verify=True/False/cafile/capath)
  - openssl s_client (default/CAfile/CApath/chain bundle)
  - certutil (NSS/Chromium)
  - certifi vs system CA comparison
Outputs JSON summary with AI diagnosis hint.
2026-05-28 22:57:08 +03:00
2552cb7546 fix: rewrite check_llm_certs.sh — tests all methods AI needs
Tests 4 variants: openssl s_client (CAfile, CApath, default),
Python httpx (cafile, cafile+capath), requests (cafile, capath),
NSS certutil. Output format is machine-parseable for AI diagnosis.
2026-05-28 22:56:19 +03:00
10 changed files with 461 additions and 314 deletions

View File

@@ -924,24 +924,27 @@ class LLMClient:
# region LLMClient._get_ssl_verify [TYPE Function] # region LLMClient._get_ssl_verify [TYPE Function]
# @PURPOSE Resolve SSL verification flag from environment. # @PURPOSE Resolve SSL verification flag from environment.
# @POST Returns SSLContext with system CA bundle when enabled, # @POST Returns SSLContext with system CA dir when enabled,
# False when LLM_SSL_VERIFY env var is "false"/"0"/"no"/"off". # False when LLM_SSL_VERIFY env var is "false"/"0"/"no"/"off".
# @RATIONALE Возвращаем ssl.SSLContext вместо True, потому что httpx по # @RATIONALE Используем capath=/etc/ssl/certs/ вместо cafile, потому что
# умолчанию использует certifi, в котором нет корпоративных CA. # OpenSSL 3.x не использует intermediate CA сертификаты из cafile для
# ssl.create_default_context(cafile=...) гарантирует, что используются # построения цепочки (verify code 20). capath с хеш-симлинками работает
# сертификаты из системного хранилища (/etc/ssl/certs/ca-certificates.crt), # корректно (verify code 0). Оба пути — cafile и capath — указывают на
# куда update-ca-certificates и наш fallback добавляют корпоративные CA. # один и тот же набор сертификатов, но capath правильно обрабатывает
# цепочку Root → Policy → Issuing.
# @REJECTED verify=<string> отвергнут — httpx 0.28.x депрекейтит строковый # @REJECTED verify=<string> отвергнут — httpx 0.28.x депрекейтит строковый
# путь в verify=, требует SSLContext. # путь в verify=, требует SSLContext.
# @REJECTED cafile отвергнут — OpenSSL 3.x не использует intermediate CA
# из единого bundle-файла. Только capath с хеш-симлинками даёт code 0.
@staticmethod @staticmethod
def _get_ssl_verify() -> ssl.SSLContext | bool: def _get_ssl_verify() -> ssl.SSLContext | bool:
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower() raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
if raw in ("false", "0", "no", "off"): if raw in ("false", "0", "no", "off"):
return False return False
ca_path = "/etc/ssl/certs/ca-certificates.crt" ca_dir = "/etc/ssl/certs"
if os.path.exists(ca_path): if os.path.isdir(ca_dir):
return ssl.create_default_context(cafile=ca_path) return ssl.create_default_context(capath=ca_dir)
# fallback: если системного CA нет (редко), используем дефолтный # fallback: если директории нет (редко), используем дефолтный
return ssl.create_default_context() return ssl.create_default_context()
# endregion LLMClient._get_ssl_verify # endregion LLMClient._get_ssl_verify

View File

@@ -18,18 +18,21 @@ from typing import Any
from ...core.logger import logger from ...core.logger import logger
# #region _get_verify [C:1] [TYPE Function] [SEMANTICS translate, ssl, verify]
# @BRIEF Resolve SSL verification path from LLM_SSL_VERIFY env var.
# @RATIONALE Используем capath=/etc/ssl/certs/ вместо cafile, потому что
# OpenSSL 3.x не использует intermediate CA сертификаты из cafile для
# построения цепочки (verify code 20). capath с хеш-симлинками работает
# корректно (verify code 0).
# @REJECTED cafile отвергнут — OpenSSL 3.x не использует intermediate CA
# из единого bundle-файла. Только capath с хеш-симлинками даёт code 0.
# @POST Returns path to /etc/ssl/certs/ when enabled, False when disabled.
def _get_verify() -> str | bool: def _get_verify() -> str | bool:
"""Resolve SSL verify from LLM_SSL_VERIFY env var.
Returns:
- Path to system CA bundle when verification enabled
(certifi bundle doesn't include corporate CA certs)
- False when LLM_SSL_VERIFY is set to false/0/no/off
"""
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower() raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
if raw in ("false", "0", "no", "off"): if raw in ("false", "0", "no", "off"):
return False return False
return "/etc/ssl/certs/ca-certificates.crt" return "/etc/ssl/certs/"
# #endregion _get_verify
# #region call_openai_compatible [C:3] [TYPE Function] [SEMANTICS translate, llm, http, openai] # #region call_openai_compatible [C:3] [TYPE Function] [SEMANTICS translate, llm, http, openai]
@@ -200,7 +203,4 @@ def _handle_response_format_fallback(
response.encoding = new_response.encoding response.encoding = new_response.encoding
response.headers = new_response.headers response.headers = new_response.headers
# #endregion _handle_response_format_fallback # #endregion _handle_response_format_fallback
# #endregion LLMHttpClient
response.headers = new_response.headers
# #endregion _handle_response_format_fallback
# #endregion LLMHttpClient # #endregion LLMHttpClient

View File

@@ -10,18 +10,21 @@ from typing import Any
from ...core.logger import logger from ...core.logger import logger
# #region _get_verify [C:1] [TYPE Function] [SEMANTICS translate, ssl, verify]
# @BRIEF Resolve SSL verification path from LLM_SSL_VERIFY env var.
# @RATIONALE Используем capath=/etc/ssl/certs/ вместо cafile, потому что
# OpenSSL 3.x не использует intermediate CA сертификаты из cafile для
# построения цепочки (verify code 20). capath с хеш-симлинками работает
# корректно (verify code 0).
# @REJECTED cafile отвергнут — OpenSSL 3.x не использует intermediate CA
# из единого bundle-файла. Только capath с хеш-симлинками даёт code 0.
# @POST Returns path to /etc/ssl/certs/ when enabled, False when disabled.
def _get_verify() -> str | bool: def _get_verify() -> str | bool:
"""Resolve SSL verify from LLM_SSL_VERIFY env var.
Returns:
- Path to system CA bundle when verification enabled
(certifi bundle doesn't include corporate CA certs)
- False when LLM_SSL_VERIFY is set to false/0/no/off
"""
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower() raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
if raw in ("false", "0", "no", "off"): if raw in ("false", "0", "no", "off"):
return False return False
return "/etc/ssl/certs/ca-certificates.crt" return "/etc/ssl/certs/"
# #endregion _get_verify
class LLMClient: class LLMClient:

View File

@@ -241,8 +241,31 @@ def validate_target_table_schema(
safe_schema = (req.target_schema or "public").replace("'", "''") safe_schema = (req.target_schema or "public").replace("'", "''")
safe_table = req.target_table.replace("'", "''") safe_table = req.target_table.replace("'", "''")
# Нормализуем backend через маппинг (как в get_dialect_from_database)
# чтобы правильно обработать "clickhousedb" → "clickhouse",
# "greenplum" → "postgresql" и другие варианты
_backend_normalize = {
"clickhouse": "clickhouse",
"clickhousedb": "clickhouse",
"postgresql": "postgresql",
"greenplum": "postgresql",
"mysql": "mysql",
"mssql": "mssql",
"sqlite": "sqlite",
"oracle": "oracle",
"snowflake": "snowflake",
"bigquery": "bigquery",
"redshift": "redshift",
"presto": "presto",
"trino": "trino",
"druid": "druid",
"hive": "hive",
"spark": "spark",
"databricks": "databricks",
}
normalized_backend = _backend_normalize.get(backend.lower().strip(), backend.lower().strip())
# ClickHouse использует system.columns, всё остальное — information_schema.columns # ClickHouse использует system.columns, всё остальное — information_schema.columns
is_clickhouse = any(kw in backend.lower() for kw in ("clickhouse", "ch")) is_clickhouse = normalized_backend == "clickhouse"
if is_clickhouse: if is_clickhouse:
sql = ( sql = (
f"SELECT name, type, default_expression AS data_default " f"SELECT name, type, default_expression AS data_default "

View File

@@ -7,15 +7,27 @@ from ...schemas.translate import TranslateJobResponse
# #region _extract_dialect [TYPE Function] # #region _extract_dialect [TYPE Function]
# @BRIEF Extract database dialect from Superset backend URI. # @BRIEF Extract database dialect from Superset backend URI or engine name.
def _extract_dialect(backend: str) -> str: def _extract_dialect(backend: str) -> str:
"""Extract dialect name from a Superset database backend URI.""" """Extract dialect name from a Superset database backend URI or engine name.
Handles both URI formats (e.g. 'postgresql://...', 'clickhousedb://...')
and plain engine names (e.g. 'postgresql', 'clickhousedb') returned by Superset.
Normalises known variants like 'clickhousedb''clickhouse'.
"""
if not backend: if not backend:
return "unknown" return "unknown"
try: try:
# Extract scheme from URI or use plain name
scheme = backend.split("://")[0] scheme = backend.split("://")[0]
dialect = scheme.split("+")[0] dialect = scheme.split("+")[0]
return dialect.lower() raw = dialect.lower()
# Normalise known Superset backend variants
dialect_map = {
"clickhousedb": "clickhouse",
"greenplum": "postgresql",
}
return dialect_map.get(raw, raw)
except Exception: except Exception:
return "unknown" return "unknown"
# #endregion _extract_dialect # #endregion _extract_dialect

View File

@@ -18,7 +18,7 @@ from ...core.logger import belief_scope, logger
# PostgreSQL/Greenplum dialects that support ON CONFLICT # PostgreSQL/Greenplum dialects that support ON CONFLICT
POSTGRESQL_DIALECTS = {"postgresql", "redshift", "greenplum"} POSTGRESQL_DIALECTS = {"postgresql", "redshift", "greenplum"}
# Dialects that use backtick or no quoting # Dialects that use backtick or no quoting
CLICKHOUSE_DIALECTS = {"clickhouse"} CLICKHOUSE_DIALECTS = {"clickhouse", "clickhousedb"}
# #region _normalize_timestamp_value [C:2] [TYPE Function] [SEMANTICS translate,sql,timestamp] # #region _normalize_timestamp_value [C:2] [TYPE Function] [SEMANTICS translate,sql,timestamp]

View File

@@ -136,7 +136,7 @@ install_llm_ca_certs() {
fi fi
local raw_file="${work_dir}/${filename}" local raw_file="${work_dir}/${filename}"
local pem_file="${target_dir}/$(echo "$filename" | sed 's/\.[^.]*$//').pem" local pem_file="${target_dir}/$(echo "$filename" | sed 's/\.[^.]*$//').crt"
echo "[entrypoint] [${index}] Скачивание: ${url}" echo "[entrypoint] [${index}] Скачивание: ${url}"
@@ -208,14 +208,14 @@ install_llm_ca_certs() {
# Проверяем, попали ли сертификаты в бандл, и создаём хеш-симлинки # Проверяем, попали ли сертификаты в бандл, и создаём хеш-симлинки
echo "[entrypoint] --- Валидация установки сертификатов ---" echo "[entrypoint] --- Валидация установки сертификатов ---"
for pem in "$target_dir"/*.pem; do for cert_file in "$target_dir"/*.crt; do
[ -f "$pem" ] || continue [ -f "$cert_file" ] || continue
local cert_name local cert_name
cert_name="$(basename "$pem" .pem)" cert_name="$(basename "$cert_file" .crt)"
local cert_subject local cert_subject
cert_subject="$(openssl x509 -in "$pem" -noout -subject 2>/dev/null | head -1 || echo "unknown")" cert_subject="$(openssl x509 -in "$cert_file" -noout -subject 2>/dev/null | head -1 || echo "unknown")"
local cert_fingerprint local cert_fingerprint
cert_fingerprint="$(openssl x509 -in "$pem" -noout -fingerprint -sha256 2>/dev/null | sed 's/.*=//' || echo "")" cert_fingerprint="$(openssl x509 -in "$cert_file" -noout -fingerprint -sha256 2>/dev/null | sed 's/.*=//' || echo "")"
# Проверяем есть ли в ca-certificates.crt по fingerprint (криптографически уникален) # Проверяем есть ли в ca-certificates.crt по fingerprint (криптографически уникален)
if [ -n "$cert_fingerprint" ] && \ if [ -n "$cert_fingerprint" ] && \
@@ -223,26 +223,26 @@ install_llm_ca_certs() {
echo "[entrypoint] ✅ ${cert_name} — в ca-certificates.crt (${cert_subject})" echo "[entrypoint] ✅ ${cert_name} — в ca-certificates.crt (${cert_subject})"
else else
echo "[entrypoint] ⚠ ${cert_name}НЕ в ca-certificates.crt. Добавляем напрямую..." echo "[entrypoint] ⚠ ${cert_name}НЕ в ca-certificates.crt. Добавляем напрямую..."
cat "$pem" >> /etc/ssl/certs/ca-certificates.crt cat "$cert_file" >> /etc/ssl/certs/ca-certificates.crt
echo "[entrypoint] Добавлен напрямую в ca-certificates.crt" echo "[entrypoint] Добавлен напрямую в ca-certificates.crt"
fi fi
# Создаём хеш-симлинку с поддержкой коллизий (.0, .1, .2...) # Создаём хеш-симлинку с поддержкой коллизий (.0, .1, .2...)
local hash_val local hash_val
hash_val="$(openssl x509 -in "$pem" -noout -hash 2>/dev/null || true)" hash_val="$(openssl x509 -in "$cert_file" -noout -hash 2>/dev/null || true)"
if [ -n "$hash_val" ]; then if [ -n "$hash_val" ]; then
local suffix=0 local suffix=0
local link_path="/etc/ssl/certs/${hash_val}.${suffix}" local link_path="/etc/ssl/certs/${hash_val}.${suffix}"
# Ищем первый свободный suffix или symlink уже указывающий на наш файл # Ищем первый свободный suffix или symlink уже указывающий на наш файл
while [ -L "$link_path" ]; do while [ -L "$link_path" ]; do
if [ "$(readlink "$link_path")" = "$pem" ]; then if [ "$(readlink "$link_path")" = "$cert_file" ]; then
break 2 # уже есть, ничего не делаем break 2 # уже есть, ничего не делаем
fi fi
suffix=$((suffix + 1)) suffix=$((suffix + 1))
link_path="/etc/ssl/certs/${hash_val}.${suffix}" link_path="/etc/ssl/certs/${hash_val}.${suffix}"
done done
if [ ! -L "$link_path" ]; then if [ ! -L "$link_path" ]; then
ln -sf "$pem" "$link_path" ln -sf "$cert_file" "$link_path"
echo "[entrypoint] 🔗 Создана симлинка: ${hash_val}.${suffix}${cert_name}" echo "[entrypoint] 🔗 Создана симлинка: ${hash_val}.${suffix}${cert_name}"
fi fi
fi fi

View File

@@ -7,6 +7,7 @@
# @RELATION CALLS -> [backend/src/plugins/llm_analysis/service.py] # @RELATION CALLS -> [backend/src/plugins/llm_analysis/service.py]
# @RELATION CALLS -> [backend/src/plugins/translate/_llm_http.py] # @RELATION CALLS -> [backend/src/plugins/translate/_llm_http.py]
# @RELATION CALLS -> [backend/src/plugins/translate/preview_llm_client.py] # @RELATION CALLS -> [backend/src/plugins/translate/preview_llm_client.py]
# @RELATION CALLS -> [scripts/check_llm_certs.py]
# @RATIONALE ss-tools operates in corporate environments with internal Certificate Authorities. # @RATIONALE ss-tools operates in corporate environments with internal Certificate Authorities.
# Three separate HTTP stacks are used: httpx (AsyncOpenAI in llm_analysis plugin), # Three separate HTTP stacks are used: httpx (AsyncOpenAI in llm_analysis plugin),
# requests (sync translate plugin), and Playwright Chromium (dashboard screenshots). # requests (sync translate plugin), and Playwright Chromium (dashboard screenshots).
@@ -23,32 +24,71 @@ Corporate SSL certificates installed via `update-ca-certificates` into `/etc/ssl
1. **Python `requests` library** — uses bundled `certifi` CA bundle, not system store 1. **Python `requests` library** — uses bundled `certifi` CA bundle, not system store
2. **Python `httpx` library** — uses `certifi` by default when `verify=True` 2. **Python `httpx` library** — uses `certifi` by default when `verify=True`
3. **Playwright Chromium** — uses NSS Shared DB (`~/.pki/nssdb`), not OpenSSL store 3. **Playwright Chromium** — uses NSS Shared DB (`~/.pki/nssdb`), not OpenSSL store
4. **`openssl s_client`** — hangs without stdin (needs `echo \|` or `input=""`)
This causes `SSLError: CERTIFICATE_VERIFY_FAILED` and `ERR_CERT_AUTHORITY_INVALID` even after correct system-wide CA installation. This causes `SSLError: CERTIFICATE_VERIFY_FAILED` and `ERR_CERT_AUTHORITY_INVALID` even after correct system-wide CA installation.
## Discovery Journey
### Phase 1: certifi vs system CA (0.1.5)
- `requests` and `httpx` use `certifi` bundle, not `/etc/ssl/certs/ca-certificates.crt`
- Fix: return system CA path instead of `True` in all `_get_verify()` functions
- Implemented in `service.py`, `_llm_http.py`, `preview_llm_client.py`
### Phase 2: .pem vs .crt extension (0.1.6)
- `update-ca-certificates` only processes `.crt` files, silently ignores `.pem`
- Downloaded certificates from PKI were saved as `.pem`, skipped by update-ca-certificates
- Result: `ca-certificates.crt` was empty (0 certs), hash symlinks existed but bundle was broken
- Fix: save downloaded certs with `.crt` extension, not `.pem`
### Phase 3: cafile vs capath (0.1.7) — KEY DISCOVERY
- `openssl s_client -CAfile /etc/ssl/certs/ca-certificates.crt` → code 20 (intermediate CA ignored)
- `openssl s_client -CApath /etc/ssl/certs/` → code 0 (chain built correctly)
- `ssl.create_default_context(cafile=...)` → SSL error
- `ssl.create_default_context(capath=...)` → HTTP 200
- `requests.get(verify="/etc/ssl/certs/ca-certificates.crt")` → SSLError
- `requests.get(verify="/etc/ssl/certs/")` → HTTP 200
**Root cause**: OpenSSL 3.x treats non-self-signed certificates in `-CAfile` as trust anchors,
NOT as intermediates. `-CApath` (directory with hash symlinks) correctly builds chains using
all certificates found. The intermediate CA certs (Policy CA, RGM Issuing CA) are not
self-signed, so they are ignored in `-CAfile` but correctly used in `-CApath`.
Diagnostic matrix (verified on production server 2026-05-28):
| Method | cafile | capath |
|--------|--------|--------|
| openssl s_client | code 20 FAIL | code 0 OK |
| httpx (SSLContext) | SSLError FAIL | HTTP 200 OK |
| requests (verify=) | SSLError FAIL | HTTP 200 OK |
## Solution ## Solution
### Layer 1: System CA Store (OpenSSL) ### Layer 1: System CA Store (OpenSSL)
- Entrypoint `install_certificates()` copies `.crt/.pem` files from `CERTS_PATH` volume mount - Entrypoint `install_certificates()` copies `.crt` files from `CERTS_PATH` volume mount
- Entrypoint `install_llm_ca_certs()` downloads PEM/DER certificates from `LLM_CA_CERT_URLS` - Entrypoint `install_llm_ca_certs()` downloads PEM/DER certificates from `LLM_CA_CERT_URLS`
- DER is auto-converted to PEM (`openssl x509 -inform DER -outform PEM`) - DER is auto-converted to PEM (`openssl x509 -inform DER -outform PEM`)
- Certificates are saved with **`.crt` extension** (`.pem` is silently ignored by `update-ca-certificates`)
- `update-ca-certificates --fresh` adds them to `/etc/ssl/certs/ca-certificates.crt` - `update-ca-certificates --fresh` adds them to `/etc/ssl/certs/ca-certificates.crt`
- **Fallback**: if `update-ca-certificates` misses certs (Debian Bookworm bug with subdirectories), they are appended to `ca-certificates.crt` with SHA256 fingerprint dedup - **Fallback**: if `update-ca-certificates` misses certs, they are appended to `ca-certificates.crt` with SHA256 fingerprint dedup
- Hash symlinks created with collision support: `.0`, `.1`, `.2` suffixes
### Layer 2: Python HTTP Clients ### Layer 2: Python HTTP Clients
| Library | File | Mechanism | | Library | File | Mechanism | Status |
|---------|------|-----------| |---------|------|-----------|--------|
| `httpx` (AsyncOpenAI) | `service.py:LLMClient._get_ssl_verify()` | Returns `ssl.create_default_context(cafile="/etc/ssl/certs/ca-certificates.crt")` | | `httpx` (AsyncOpenAI) | `service.py:LLMClient._get_ssl_verify()` | `ssl.create_default_context(capath="/etc/ssl/certs/")` | ✅ Works (0.1.7) |
| `requests` (`_llm_http.py`) | `_get_verify()` | Returns `"/etc/ssl/certs/ca-certificates.crt"` (string path) | | `requests` (`_llm_http.py`) | `_get_verify()` | `"/etc/ssl/certs/"` (string path to dir) | ✅ Works (0.1.7) |
| `requests` (`preview_llm_client.py`) | `_get_verify()` | Returns `"/etc/ssl/certs/ca-certificates.crt"` (string path) | | `requests` (`preview_llm_client.py`) | `_get_verify()` | `"/etc/ssl/certs/"` (string path to dir) | ✅ Works (0.1.7) |
Key insight: `verify=True` uses certifi, NOT system CA. All three functions now return the system CA path instead of boolean `True`. Key insight: `verify=True` uses certifi, NOT system CA. `verify=<cafile_path>` ignores
intermediate CA in OpenSSL 3.x. `verify=<capath_dir>` works correctly.
### Layer 3: NSS Database (Playwright Chromium) ### Layer 3: NSS Database (Playwright Chromium)
- Entrypoint `install_ca_to_nss()` imports PEM certs into `~/.pki/nssdb/` using `certutil` - Entrypoint `install_ca_to_nss()` imports PEM certs into `~/.pki/nssdb/` using `certutil`
- NSS DB path format: `sql:$HOME/.pki/nssdb` (SQLite prefix required, DBM not supported by Chromium)
- Nickname format: `{dir_prefix}-{filename}` (e.g., `llm-UC_RUSAL_Policy_CA`) - Nickname format: `{dir_prefix}-{filename}` (e.g., `llm-UC_RUSAL_Policy_CA`)
- Dedup by SHA256 fingerprint (not nickname), preventing duplicate imports - Dedup by SHA256 fingerprint (not nickname), preventing duplicate imports
- Trust attributes: `"C,,"` (trusted CA for TLS server certs) - Trust attributes: `"C,,"` (trusted CA for TLS server certs)
@@ -58,8 +98,9 @@ Key insight: `verify=True` uses certifi, NOT system CA. All three functions now
- Env var `LLM_SSL_VERIFY=false` disables SSL verification entirely - Env var `LLM_SSL_VERIFY=false` disables SSL verification entirely
- Accepted values for false: `false`, `0`, `no`, `off` (case-insensitive) - Accepted values for false: `false`, `0`, `no`, `off` (case-insensitive)
- Default: enabled (returns system CA path / SSLContext) - Default: enabled (returns capath-based SSLContext or path)
- Used by all three HTTP client implementations - Used by all three HTTP client implementations
- **WARNING**: `verify=False` is for DIAGNOSTIC USE ONLY. Never leave in production.
## Key Files ## Key Files
@@ -67,13 +108,59 @@ Key insight: `verify=True` uses certifi, NOT system CA. All three functions now
|------|------| |------|------|
| `docker/backend.Dockerfile` | Installs `libnss3-tools` (certutil), Playwright Chromium | | `docker/backend.Dockerfile` | Installs `libnss3-tools` (certutil), Playwright Chromium |
| `docker/backend.entrypoint.sh` | `install_certificates`, `install_llm_ca_certs`, `install_ca_to_nss` | | `docker/backend.entrypoint.sh` | `install_certificates`, `install_llm_ca_certs`, `install_ca_to_nss` |
| `plugins/llm_analysis/service.py` | `LLMClient._get_ssl_verify()``ssl.create_default_context(cafile=...)` | | `plugins/llm_analysis/service.py` | `LLMClient._get_ssl_verify()``ssl.create_default_context(capath=...)` |
| `plugins/translate/_llm_http.py` | `_get_verify()``"/etc/ssl/certs/ca-certificates.crt"` | | `plugins/translate/_llm_http.py` | `_get_verify()``"/etc/ssl/certs/"` |
| `plugins/translate/preview_llm_client.py` | `_get_verify()``"/etc/ssl/certs/ca-certificates.crt"` | | `plugins/translate/preview_llm_client.py` | `_get_verify()``"/etc/ssl/certs/"` |
| `scripts/check_llm_certs.py` | Full diagnostic: openssl, httpx, requests, NSS |
| `docker-compose.yml` | Passes `LLM_SSL_VERIFY`, `LLM_CA_CERT_URLS` | | `docker-compose.yml` | Passes `LLM_SSL_VERIFY`, `LLM_CA_CERT_URLS` |
| `docker-compose.enterprise-clean.yml` | Same as above | | `docker-compose.enterprise-clean.yml` | Same as above |
| `.env.enterprise-clean` | Default values for both env vars | | `.env.enterprise-clean` | Default values for both env vars |
## How to Test Certificate Installation
### On the server, after container restart:
```bash
docker cp scripts/check_llm_certs.py ss-tools-backend-1:/tmp/
docker compose -f docker-compose.enterprise-clean.yml \
--env-file .env.enterprise-clean exec backend \
python3 /tmp/check_llm_certs.py --target https://lite.ai.rusal.com
```
Expected diagnostic matrix:
```
openssl_default: ✅ code 0
openssl_capath: ✅ code 0
openssl_cafile: ❌ code 20 (expected — OpenSSL 3.x limitation)
httpx_capath: ✅ HTTP 200
httpx_cafile: ❌ SSL error (expected)
requests_capath: ✅ HTTP 200
requests_cafile: ❌ SSL error (expected)
```
## Manually testing chain validity
```bash
# 1. Check individual certs
openssl x509 -in /usr/local/share/ca-certificates/custom/RUSAL_ROOT.crt -noout -subject -issuer
# 2. Verify full chain
openssl verify -CAfile /usr/local/share/ca-certificates/custom/RUSAL_ROOT.crt \
-untrusted /usr/local/share/ca-certificates/llm/UC_RUSAL_Policy_CA.crt \
/usr/local/share/ca-certificates/llm/UC_RUSAL_RGM_Issuing_CA.crt
# → OK
# 3. Connect with capath
echo | openssl s_client -connect lite.ai.rusal.com:443 \
-servername lite.ai.rusal.com \
-CApath /etc/ssl/certs/
# 4. Verify certificate count
grep -c "BEGIN CERTIFICATE" /etc/ssl/certs/ca-certificates.crt
```
## Discovered Findings ## Discovered Findings
### Finding 1: certifi vs system CA ### Finding 1: certifi vs system CA
@@ -82,30 +169,75 @@ Key insight: `verify=True` uses certifi, NOT system CA. All three functions now
### Finding 2: Three separate HTTP stacks ### Finding 2: Three separate HTTP stacks
ss-tools has three LLM HTTP clients (httpx for async, requests for sync, Playwright for screenshots), each with independent CA trust configuration. ss-tools has three LLM HTTP clients (httpx for async, requests for sync, Playwright for screenshots), each with independent CA trust configuration.
### Finding 3: update-ca-certificates inconsistency ### Finding 3: .pem extension ignored by update-ca-certificates
On Debian Bookworm (python:3.11-slim), `update-ca-certificates` occasionally skips symlinks for PEM files in subdirectories (`/usr/local/share/ca-certificates/custom/`, `llm/`), requiring a manual `cat >> ca-certificates.crt` fallback and explicit hash symlink creation. On Debian Bookworm (python:3.11-slim), `update-ca-certificates` only processes `.crt` files.
Files with `.pem` extension in `/usr/local/share/ca-certificates/` are silently skipped.
This left `ca-certificates.crt` empty (0 certs) while hash symlinks existed.
Fix: save downloaded certificates with `.crt` extension.
### Finding 4: DER format from corporate PKI ### Finding 4: DER format from corporate PKI
Corporate PKI servers (`pki.rusal.com`) often serve certificates in DER (binary) format. `update-ca-certificates` requires PEM. Auto-detection and conversion (`openssl x509 -inform DER -outform PEM`) is essential. Corporate PKI servers (`pki.rusal.com`) often serve certificates in DER (binary) format.
`update-ca-certificates` requires PEM. Auto-detection and conversion
(`openssl x509 -inform DER -outform PEM`) is essential.
### Finding 5: Chicken-and-egg TLS bootstrap ### Finding 5: Chicken-and-egg TLS bootstrap
Downloading a CA certificate from a PKI server that uses the same CA causes a TLS verification loop. Entrypoint uses `curl --insecure` as fallback when initial `curl` fails with TLS error. Downloading a CA certificate from a PKI server that uses the same CA causes a TLS
verification loop. Entrypoint uses `curl --insecure` as fallback when initial `curl`
fails with TLS error.
### Finding 6: NSS DB format ### Finding 6: NSS DB format
Chromium uses NSS Shared DB in `~/.pki/nssdb/`. The `sql:` prefix is required for `certutil` to use the SQLite format. Without it, certutil defaults to the legacy DBM format which Chromium may not read. Chromium uses NSS Shared DB in `~/.pki/nssdb/`. The `sql:` prefix is required for
`certutil` to use the SQLite format. Without it, certutil defaults to the legacy
DBM format which Chromium may not read.
### Finding 7: OpenSSL 3.x cafile vs capath (CRITICAL)
OpenSSL 3.x treats non-self-signed certificates in `-CAfile` as trust anchors,
NOT as intermediates. This means intermediate CA certificates (Policy CA, RGM Issuing CA)
are ignored when using `-CAfile /etc/ssl/certs/ca-certificates.crt`, producing
verify code 20. Using `-CApath /etc/ssl/certs/` (directory with hash symlinks)
correctly builds the full chain, producing verify code 0.
This affects ALL Python libraries:
- `ssl.create_default_context(cafile=...)` → calls OpenSSL's `SSL_CTX_load_verify_locations`
with the file, which exhibits the same limitation
- `ssl.create_default_context(capath=...)` → works correctly
- `requests.get(verify="/path/to/file")` → fails (uses cafile internally)
- `requests.get(verify="/path/to/dir/")` → works (uses capath internally)
### Finding 8: openssl s_client hangs without stdin
`openssl s_client` waits for input after TLS handshake. When run via
`subprocess.run(cmd, capture_output=True)`, it hangs until timeout.
Fix: pass `input=""` or `echo | openssl s_client ...`.
## Deploy ## Deploy
```bash ```bash
# On target server: # On target server:
xz -dc ss-tools-backend.0.1.6.tar.xz | docker load xz -dc ss-tools-backend.0.1.7.tar.xz | docker load
xz -dc ss-tools-frontend.0.1.6.tar.xz | docker load xz -dc ss-tools-frontend.0.1.7.tar.xz | docker load
# Enable capath-based verification (remove LLM_SSL_VERIFY=false)
sed -i '/LLM_SSL_VERIFY=false/d' .env.enterprise-clean
docker compose -f docker-compose.enterprise-clean.yml \ docker compose -f docker-compose.enterprise-clean.yml \
--env-file .env.enterprise-clean down --env-file .env.enterprise-clean down
docker compose -f docker-compose.enterprise-clean.yml \ docker compose -f docker-compose.enterprise-clean.yml \
--env-file .env.enterprise-clean up -d --env-file .env.enterprise-clean up -d
# Verify
docker cp scripts/check_llm_certs.py ss-tools-backend-1:/tmp/
docker compose -f docker-compose.enterprise-clean.yml \
--env-file .env.enterprise-clean exec backend \
python3 /tmp/check_llm_certs.py --target https://lite.ai.rusal.com
``` ```
## Version History
| Version | Changes |
|---------|---------|
| 0.1.5 | Initial SSL support: `LLM_SSL_VERIFY`, `_format_connection_error()`, CA download via URL, NSS import |
| 0.1.6 | QA fixes: fingerprint dedup, hash symlink collision, NSS collision, DER→PEM error handling, chicken-and-egg TLS, nullglob, translate plugin SSL |
| 0.1.7 | **capath instead of cafile**: OpenSSL 3.x cafile limitation discovered and fixed. `.crt` extension for downloaded certs. Diagnostic script rewritten. |
# [/DEF:ADR-0009:ADR] # [/DEF:ADR-0009:ADR]

225
scripts/check_llm_certs.py Executable file
View File

@@ -0,0 +1,225 @@
#!/usr/bin/env python3
"""
SSL Certificate Diagnostics for ss-tools.
Tests all 4 layers with the actual libraries used in production.
Key finding from ADR-0009: use capath, NOT cafile — OpenSSL 3.x
ignores intermediate CA certs in cafile (verify code 20) but
correctly builds chains with capath (verify code 0).
Output: human-readable + JSON summary with AI diagnosis hint.
"""
from __future__ import annotations
import argparse
import json
import os
import platform
import subprocess
import sys
from pathlib import Path
# ── Constants ──
CA_BUNDLE = Path("/etc/ssl/certs/ca-certificates.crt")
CA_DIR = Path("/etc/ssl/certs")
CUSTOM_DIRS = [
Path("/usr/local/share/ca-certificates/custom"),
Path("/usr/local/share/ca-certificates/llm"),
]
NSS_DB = f"sql:{Path(os.environ.get('HOME', '/root')) / '.pki' / 'nssdb'}"
results: dict = {}
# ── Helpers ──
def _run(cmd: list[str], timeout: int = 15, input_data: str = "") -> tuple[int, str, str]:
"""Run subprocess with optional stdin."""
try:
r = subprocess.run(cmd, input=input_data, capture_output=True, text=True, timeout=timeout)
return r.returncode, r.stdout, r.stderr
except FileNotFoundError:
return -1, "", "command not found"
except subprocess.TimeoutExpired:
return -2, "", f"timeout ({timeout}s)"
def _count_certs(path: str) -> int:
try:
with open(path, "rb") as f:
return f.read().count(b"-----BEGIN CERTIFICATE-----")
except Exception:
return 0
# ── Openssl tests ──
def test_openssl(label: str, extra_args: list[str]) -> None:
"""Test openssl s_client with empty stdin to prevent hang."""
cmd = ["openssl", "s_client", "-connect", f"{host}:443",
"-servername", host, "-verify_return_error"] + extra_args
rc, out, err = _run(cmd, timeout=20, input_data="")
for line in out.split("\n"):
if "Verify return code" in line:
ok = "0 (ok)" in line
results[f"openssl_{label}"] = f"{'OK' if ok else 'FAIL'}: {line.strip()}"
print(f" openssl_{label}: {'' if ok else ''} {line.strip()}")
return
results[f"openssl_{label}"] = f"ERROR: {err.strip() or 'no output'}"
print(f" openssl_{label}: ❌ {results[f'openssl_{label}']}")
# ── Python library tests ──
def test_httpx(key: str, verify) -> None:
try:
import httpx
r = httpx.get(TARGET_URL, verify=verify, timeout=10)
results[key] = "OK"
print(f" {key}: ✅ status={r.status_code}")
except Exception as e:
results[key] = f"FAIL: {type(e).__name__}"
print(f" {key}: ❌ {type(e).__name__}")
def test_requests(key: str, verify) -> None:
try:
import requests as req
r = req.get(TARGET_URL, verify=verify, timeout=10)
results[key] = "OK"
print(f" {key}: ✅ status={r.status_code}")
except Exception as e:
results[key] = f"FAIL: {type(e).__name__}"
print(f" {key}: ❌ {type(e).__name__}")
# ══════════════════════════════════════════════
# MAIN
# ══════════════════════════════════════════════
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--target", default=os.environ.get("TARGET_URL", "https://lite.ai.rusal.com"))
args = parser.parse_args()
TARGET_URL = args.target
host = TARGET_URL.replace("https://", "").split("/")[0]
LLM_SSL_VERIFY = os.environ.get("LLM_SSL_VERIFY", "true")
# ── 1. System ──
print("\n=== 1. System ===")
print(f" target={TARGET_URL} host={host}")
print(f" platform={platform.platform()}")
print(f" python={sys.version.split()[0]}")
for lib in ("httpx", "requests", "openai", "certifi"):
try:
mod = __import__(lib)
v = getattr(mod, "__version__", "installed")
p = getattr(mod, "__file__", "")
if lib == "certifi":
print(f" {lib}={v} path={mod.where()} certs={_count_certs(mod.where())}")
else:
print(f" {lib}={v}")
except ImportError:
print(f" {lib}=NOT_INSTALLED")
# ── 2. System CA Store ──
print("\n=== 2. System CA Store ===")
bundle_certs = _count_certs(str(CA_BUNDLE)) if CA_BUNDLE.exists() else 0
hash_links = len(list(CA_DIR.glob("[0-9a-f]*.[0-9]"))) if CA_DIR.is_dir() else 0
print(f" bundle={CA_BUNDLE} certs={bundle_certs}")
print(f" capath={CA_DIR} hash_links={hash_links}")
print(" custom certs:")
for d in CUSTOM_DIRS:
if not d.is_dir():
continue
for f in sorted(d.iterdir()):
if f.suffix in (".pem", ".crt") and f.is_file():
rc, out, _ = _run(["openssl", "x509", "-in", str(f), "-noout",
"-subject", "-issuer"], timeout=3)
if rc != 0:
rc, out, _ = _run(["openssl", "x509", "-in", str(f), "-inform",
"DER", "-noout", "-subject"], timeout=3)
subj = out.replace("subject=", "").strip() if rc == 0 else "UNREADABLE"
print(f" {f.parent.name}/{f.name} subj={subj[:60]}")
# ── 3. OpenSSL s_client (3 variants) ──
print("\n=== 3. OpenSSL s_client ===")
test_openssl("default", [])
test_openssl("cafile", ["-CAfile", str(CA_BUNDLE)])
test_openssl("capath", ["-CApath", str(CA_DIR)])
# ── 4. Python httpx ──
print("\n=== 4. Python httpx (LLMClient) ===")
try:
import httpx
test_httpx("httpx_True", verify=True)
try:
import ssl
ctx_cafile = ssl.create_default_context(cafile=str(CA_BUNDLE)) if CA_BUNDLE.exists() else ssl.create_default_context()
ctx_capath = ssl.create_default_context(capath=str(CA_DIR)) if CA_DIR.is_dir() else ssl.create_default_context()
test_httpx("httpx_cafile", verify=ctx_cafile)
test_httpx("httpx_capath", verify=ctx_capath)
except Exception as e:
print(f" SSLContext: ❌ {e}")
test_httpx("httpx_False", verify=False)
except ImportError:
print(" httpx: NOT_INSTALLED")
# ── 5. Python requests ──
print("\n=== 5. Python requests (translate plugin) ===")
try:
import requests as req
test_requests("requests_True", verify=True)
test_requests("requests_cafile", verify=str(CA_BUNDLE))
test_requests("requests_capath", verify=str(CA_DIR))
test_requests("requests_False", verify=False)
except ImportError:
print(" requests: NOT_INSTALLED")
# ── 6. NSS Chromium ──
print("\n=== 6. NSS Chromium ===")
rc, out, err = _run(["certutil", "-L", "-d", NSS_DB], timeout=5)
if rc == 0:
lines = [l for l in out.split("\n") if l.strip() and "Certificate Nickname" not in l]
print(f" DB={NSS_DB} certs={len(lines)}")
for l in lines:
if any(x in l.lower() for x in ("rusal", "llm-", "custom-")):
print(f" {l}")
else:
print(f" NSS: {err.strip() or 'unavailable'}")
# ── 7. LLM_SSL_VERIFY env ──
print("\n=== 7. LLM_SSL_VERIFY ===")
print(f" LLM_SSL_VERIFY={LLM_SSL_VERIFY} disabled={LLM_SSL_VERIFY.strip().lower() in ('false','0','no','off')}")
# ── 8. JSON Summary ──
print("\n=== 8. JSON Summary ===")
summary = {k: results[k] for k in [
"openssl_default", "openssl_cafile", "openssl_capath",
"httpx_True", "httpx_cafile", "httpx_capath", "httpx_False",
"requests_True", "requests_cafile", "requests_capath", "requests_False",
] if k in results}
# Diagnosis
diag = []
for k in ("openssl_default", "openssl_cafile", "openssl_capath"):
v = str(summary.get(k, ""))
diag.append(f"{k}={'OK' if v.startswith('OK') else 'FAIL'}")
for k in sorted(summary):
if k.startswith("httpx") or k.startswith("requests"):
v = str(summary.get(k, ""))
diag.append(f"{k}={'OK' if v.startswith('OK') else 'FAIL'}")
if str(results.get("httpx_capath", "")).startswith("OK") and not str(results.get("httpx_cafile", "")).startswith("OK"):
diag.append("VERDICT=capath_works_cafile_fails_use_capath")
elif str(summary.get("httpx_False", "")).startswith("OK"):
diag.append("VERDICT=all_ssl_fails_verify_disabled_works")
elif str(summary.get("openssl_default", "")).startswith("OK"):
diag.append("VERDICT=all_ok")
else:
diag.append("VERDICT=needs_investigation")
summary["diagnosis"] = "; ".join(diag)
print(json.dumps(summary, indent=2))

View File

@@ -1,251 +0,0 @@
#!/usr/bin/env bash
# #region check_llm_certs [C:4] [TYPE Module] [SEMANTICS ssl, certs, diagnostics, openssl, nss]
# @BRIEF Проверка установки корпоративных SSL-сертификатов для LLM-провайдеров.
# Читает конфигурацию из .env.enterprise-clean, проверяет все 4 слоя:
# OpenSSL store, httpx, requests, Chromium NSS.
# @RATIONALE После ручной/автоматической установки сертификатов нужно убедиться
# что все три HTTP-стека (httpx, requests, Chromium) доверяют корпоративным CA.
# По отдельности каждый слой имеет свою логику — скрипт собирает всё воедино.
# @USAGE ./scripts/check_llm_certs.sh [--env-file .env.enterprise-clean] [--target https://lite.ai.rusal.com]
# @LAYER Infrastructure
# @EXAMPLE:
# ./scripts/check_llm_certs.sh
# ./scripts/check_llm_certs.sh --target https://api.corp.com/v1 --env-file /opt/ss-tools/.env
# #endregion check_llm_certs
set -euo pipefail
# ── Colors ──
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color
PASS="${GREEN}${NC}"
FAIL="${RED}${NC}"
WARN="${YELLOW}${NC}"
INFO="${CYAN}${NC}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
# ── Defaults ──
ENV_FILE="${PROJECT_ROOT}/.env.enterprise-clean"
TARGET_URL="https://lite.ai.rusal.com"
VERBOSE=false
# ── Parse args ──
while [[ $# -gt 0 ]]; do
case "$1" in
--env-file) ENV_FILE="$2"; shift 2 ;;
--target) TARGET_URL="$2"; shift 2 ;;
-v|--verbose) VERBOSE=true; shift ;;
--help|-h) echo "Usage: $0 [--env-file .env] [--target https://...] [-v]"; exit 0 ;;
*) echo "Unknown: $1"; exit 1 ;;
esac
done
# ── Load env ──
if [ -f "$ENV_FILE" ]; then
echo -e "${INFO} Loading env: ${ENV_FILE}"
set -a
source "$ENV_FILE"
set +a
else
echo -e "${WARN} Env file not found: ${ENV_FILE} (using defaults)"
fi
LLM_SSL_VERIFY="${LLM_SSL_VERIFY:-true}"
CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt"
NSS_DB_DIR="${HOME:-/root}/.pki/nssdb"
echo ""
echo -e "${CYAN}========================================${NC}"
echo -e "${CYAN} LLM SSL Certificate Diagnostics${NC}"
echo -e "${CYAN}========================================${NC}"
echo ""
echo "Target URL: ${TARGET_URL}"
echo "LLM_SSL_VERIFY: ${LLM_SSL_VERIFY}"
echo "Env file: ${ENV_FILE}"
echo ""
# ══════════════════════════════════════════
# SECTION 1: System CA Store
# ══════════════════════════════════════════
echo -e "${CYAN}── Layer 1: System CA Store (OpenSSL) ──${NC}"
# Check bundle exists
total_certs=0
if [ -f "$CA_BUNDLE" ]; then
total_certs=$(awk '/BEGIN CERTIFICATE/{c++} END{print c}' "$CA_BUNDLE" 2>/dev/null || echo "0")
echo -e " ${PASS} Bundle: ${CA_BUNDLE} (${total_certs} certificates)"
else
echo -e " ${FAIL} Bundle NOT FOUND: ${CA_BUNDLE}"
fi
# List custom certs
echo -n " Custom certs: "
custom_dir="/usr/local/share/ca-certificates/custom"
llm_dir="/usr/local/share/ca-certificates/llm"
found_custom=0
for dir in "$custom_dir" "$llm_dir"; do
if [ -d "$dir" ]; then
for f in "$dir"/*.pem "$dir"/*.crt; do
[ -f "$f" ] || continue
found_custom=$((found_custom+1))
subject="$(openssl x509 -in "$f" -noout -subject 2>/dev/null | head -1 || echo "?")"
echo -e "\n ${PASS} $(basename "$f")${subject}"
done
fi
done
if [ "$found_custom" -eq 0 ]; then
echo -e "${WARN} none found (check CERTS_PATH mount and LLM_CA_CERT_URLS)"
fi
# Check symlinks
echo -n " Hash symlinks: "
hash_links=$(find /etc/ssl/certs/ -maxdepth 1 -type l -name '[0-9a-f]*.[0-9]' 2>/dev/null | wc -l)
echo -e "${hash_links} found"
# ══════════════════════════════════════════
# SECTION 2: NSS Database (Chromium)
# ══════════════════════════════════════════
echo ""
echo -e "${CYAN}── Layer 2: NSS Database (Chromium) ──${NC}"
if command -v certutil &>/dev/null && [ -d "$NSS_DB_DIR" ]; then
nss_db="sql:${NSS_DB_DIR}"
nss_certs=$(certutil -L -d "$nss_db" 2>/dev/null | wc -l || echo "0")
echo -e " ${PASS} NSS DB: ${NSS_DB_DIR} (${nss_certs} certificates)"
# Show custom certs
certutil -L -d "$nss_db" 2>/dev/null | grep -iE "rusal|llm-|custom-" | while read -r line; do
echo -e " ${PASS} ${line}"
done
elif ! command -v certutil &>/dev/null; then
echo -e " ${WARN} certutil not installed (libnss3-tools missing)"
elif [ ! -d "$NSS_DB_DIR" ]; then
echo -e " ${WARN} NSS DB not found at ${NSS_DB_DIR}"
fi
# ══════════════════════════════════════════
# SECTION 3: OpenSSL Connectivity Test
# ══════════════════════════════════════════
echo ""
echo -e "${CYAN}── Layer 3: OpenSSL Connectivity ──${NC}"
target_host="$(echo "${TARGET_URL}" | sed 's|https://||;s|/.*$||')"
echo -n " ${target_host}:443 — "
if openssl s_client -connect "${target_host}:443" -CAfile "$CA_BUNDLE" -servername "$target_host" </dev/null 2>/dev/null \
| grep -q "Verify return code: 0"; then
echo -e "${PASS} SSL handshake OK (verify via system CA)"
else
verify_code=$(openssl s_client -connect "${target_host}:443" -CAfile "$CA_BUNDLE" -servername "$target_host" </dev/null 2>&1 \
| grep "Verify return code" | head -1 || echo "unknown")
echo -e "${FAIL} ${verify_code}"
echo -e " ${INFO} Check: missing intermediate CA or wrong CA bundle"
fi
# ══════════════════════════════════════════
# SECTION 4: Python httpx Test
# ══════════════════════════════════════════
echo ""
echo -e "${CYAN}── Layer 4: Python httpx (AsyncOpenAI) ──${NC}"
CA_BUNDLE_PY="${CA_BUNDLE}"
TARGET_URL_PY="${TARGET_URL}"
PASS_PY="${PASS}"
FAIL_PY="${FAIL}"
INFO_PY="${INFO}"
WARN_PY="${WARN}"
export CA_BUNDLE_PY TARGET_URL_PY PASS_PY FAIL_PY INFO_PY WARN_PY
python3 -c '
import ssl, sys, os
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
target = os.environ.get("TARGET_URL_PY", "https://lite.ai.rusal.com")
pass_emoji = os.environ.get("PASS_PY", "✅")
fail_emoji = os.environ.get("FAIL_PY", "❌")
info_emoji = os.environ.get("INFO_PY", "")
try:
ctx = ssl.create_default_context(cafile=cafile)
print(f" {pass_emoji} SSLContext created with cafile={cafile}")
print(f" {info_emoji} verify_mode={ctx.verify_mode}")
except Exception as e:
print(f" {fail_emoji} SSLContext error: {e}")
sys.exit(1)
try:
import httpx
r = httpx.get(target, verify=ctx, timeout=10)
print(f" {pass_emoji} httpx GET {target} → {r.status_code}")
except httpx.ConnectError as e:
print(f" {fail_emoji} httpx connection failed: {e}")
except Exception as e:
print(f" {fail_emoji} httpx error: {e}")
' 2>&1 || echo -e " ${FAIL} Python test failed"
# ══════════════════════════════════════════
# SECTION 5: Python requests Test
# ══════════════════════════════════════════
echo ""
echo -e "${CYAN}── Layer 5: Python requests ──${NC}"
python3 -c '
import os, sys
target = os.environ.get("TARGET_URL_PY", "https://lite.ai.rusal.com")
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
pass_emoji = os.environ.get("PASS_PY", "✅")
fail_emoji = os.environ.get("FAIL_PY", "❌")
try:
import requests
r = requests.get(target, verify=cafile, timeout=10)
print(f" {pass_emoji} requests GET {target} → {r.status_code}")
except requests.exceptions.SSLError as e:
print(f" {fail_emoji} requests SSL error: {e}")
except Exception as e:
print(f" {fail_emoji} requests error: {e}")
' 2>&1 || echo -e " ${FAIL} Python requests test failed"
# ══════════════════════════════════════════
# SECTION 6: LLM_SSL_VERIFY env var check
# ══════════════════════════════════════════
echo ""
echo -e "${CYAN}── Layer 6: LLM_SSL_VERIFY env var ──${NC}"
python3 -c '
import os
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
disabled = raw in ("false", "0", "no", "off")
warn_emoji = os.environ.get("WARN_PY", "⚠")
pass_emoji = os.environ.get("PASS_PY", "✅")
info_emoji = os.environ.get("INFO_PY", "")
cafile = os.environ.get("CA_BUNDLE_PY", "/etc/ssl/certs/ca-certificates.crt")
if disabled:
print(f" {warn_emoji} LLM_SSL_VERIFY={raw} — SSL verification DISABLED")
else:
print(f" {pass_emoji} LLM_SSL_VERIFY={raw} — SSL verification ENABLED")
import ssl
if os.path.exists(cafile):
ctx = ssl.create_default_context(cafile=cafile)
print(f" {info_emoji} ssl.create_default_context(cafile=...) — OK")
' 2>&1
# ══════════════════════════════════════════
# SUMMARY
# ══════════════════════════════════════════
echo ""
echo -e "${CYAN}========================================${NC}"
echo -e "${CYAN} Summary${NC}"
echo -e "${CYAN}========================================${NC}"
echo ""
echo " CA Bundle: ${CA_BUNDLE} (${total_certs} certs)"
echo " NSS DB: ${NSS_DB_DIR}"
echo " Target: ${TARGET_URL}"
echo " LLM_SSL_VERIFY: ${LLM_SSL_VERIFY}"
echo ""
echo " If openssl test passes but Python fails — check certifi vs system CA"
echo " If all tests pass — remove LLM_SSL_VERIFY=false from .env and rerun"
echo ""