feat: ADR-0009 SSL cert management, fix certifi vs system CA
- ADR-0009: comprehensive SSL certificate management strategy - _get_ssl_verify() returns SSLContext with cafile= instead of bool True (httpx deprecates verify=<str>, requests needs system CA, not certifi) - _get_verify() in translate plugin returns system CA path - All tests updated for SSLContext return type - All QA findings C1-C3, H1-H4, M1 incorporated into ADR
This commit is contained in:
@@ -54,15 +54,18 @@ def test_litellm_client_uses_default_bearer_auth():
|
||||
# endregion test_litellm_client_uses_default_bearer_auth
|
||||
|
||||
|
||||
# region test_get_ssl_verify_default_true [TYPE Function]
|
||||
# region test_get_ssl_verify_default_context [TYPE Function]
|
||||
# @RELATION BINDS_TO -> TestClientHeaders
|
||||
# @PURPOSE: _get_ssl_verify defaults to True when LLM_SSL_VERIFY is unset.
|
||||
# @PURPOSE: _get_ssl_verify returns SSLContext with CERT_REQUIRED by default.
|
||||
# @PRE LLM_SSL_VERIFY env var is not set.
|
||||
# @POST Returns True.
|
||||
def test_get_ssl_verify_default_true():
|
||||
"""Verify default SSL verification is enabled."""
|
||||
assert LLMClient._get_ssl_verify() is True
|
||||
# endregion test_get_ssl_verify_default_true
|
||||
# @POST Returns SSLContext with verify_mode=CERT_REQUIRED.
|
||||
def test_get_ssl_verify_default_context():
|
||||
"""Verify default returns SSLContext with CERT_REQUIRED."""
|
||||
import ssl
|
||||
result = LLMClient._get_ssl_verify()
|
||||
assert isinstance(result, ssl.SSLContext), "Expected SSLContext, not bool"
|
||||
assert result.verify_mode == ssl.CERT_REQUIRED
|
||||
# endregion test_get_ssl_verify_default_context
|
||||
|
||||
|
||||
# region test_get_ssl_verify_false_values [TYPE Function]
|
||||
@@ -77,16 +80,19 @@ def test_get_ssl_verify_false_values(monkeypatch):
|
||||
# endregion test_get_ssl_verify_false_values
|
||||
|
||||
|
||||
# region test_get_ssl_verify_true_values [TYPE Function]
|
||||
# region test_get_ssl_verify_true_returns_context [TYPE Function]
|
||||
# @RELATION BINDS_TO -> TestClientHeaders
|
||||
# @PURPOSE: _get_ssl_verify returns True for truthy values like "TRUE", "1", "yes".
|
||||
# @PURPOSE: _get_ssl_verify returns SSLContext for truthy env values.
|
||||
# @PRE LLM_SSL_VERIFY env var is set to truthy values.
|
||||
# @POST Returns True.
|
||||
def test_get_ssl_verify_true_values(monkeypatch):
|
||||
# @POST Returns SSLContext with CERT_REQUIRED.
|
||||
def test_get_ssl_verify_true_returns_context(monkeypatch):
|
||||
import ssl
|
||||
for val in ("TRUE", "1", "yes", "on", "true"):
|
||||
monkeypatch.setenv("LLM_SSL_VERIFY", val)
|
||||
assert LLMClient._get_ssl_verify() is True, f"Expected True for LLM_SSL_VERIFY={val}"
|
||||
# endregion test_get_ssl_verify_true_values
|
||||
result = LLMClient._get_ssl_verify()
|
||||
assert isinstance(result, ssl.SSLContext), f"Expected SSLContext for LLM_SSL_VERIFY={val}"
|
||||
assert result.verify_mode == ssl.CERT_REQUIRED
|
||||
# endregion test_get_ssl_verify_true_returns_path
|
||||
|
||||
|
||||
# region test_format_connection_error_no_cause [TYPE Function]
|
||||
|
||||
@@ -13,6 +13,7 @@ import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import ssl
|
||||
import tempfile
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
@@ -923,11 +924,25 @@ class LLMClient:
|
||||
|
||||
# region LLMClient._get_ssl_verify [TYPE Function]
|
||||
# @PURPOSE Resolve SSL verification flag from environment.
|
||||
# @POST Returns False when LLM_SSL_VERIFY env var is "false"/"0"/"no" (case-insensitive).
|
||||
# @POST Returns SSLContext with system CA bundle when enabled,
|
||||
# False when LLM_SSL_VERIFY env var is "false"/"0"/"no"/"off".
|
||||
# @RATIONALE Возвращаем ssl.SSLContext вместо True, потому что httpx по
|
||||
# умолчанию использует certifi, в котором нет корпоративных CA.
|
||||
# ssl.create_default_context(cafile=...) гарантирует, что используются
|
||||
# сертификаты из системного хранилища (/etc/ssl/certs/ca-certificates.crt),
|
||||
# куда update-ca-certificates и наш fallback добавляют корпоративные CA.
|
||||
# @REJECTED verify=<string> отвергнут — httpx 0.28.x депрекейтит строковый
|
||||
# путь в verify=, требует SSLContext.
|
||||
@staticmethod
|
||||
def _get_ssl_verify() -> bool:
|
||||
def _get_ssl_verify() -> ssl.SSLContext | bool:
|
||||
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
|
||||
return raw not in ("false", "0", "no", "off")
|
||||
if raw in ("false", "0", "no", "off"):
|
||||
return False
|
||||
ca_path = "/etc/ssl/certs/ca-certificates.crt"
|
||||
if os.path.exists(ca_path):
|
||||
return ssl.create_default_context(cafile=ca_path)
|
||||
# fallback: если системного CA нет (редко), используем дефолтный
|
||||
return ssl.create_default_context()
|
||||
# endregion LLMClient._get_ssl_verify
|
||||
|
||||
# region LLMClient._format_connection_error [TYPE Function]
|
||||
|
||||
@@ -18,10 +18,18 @@ from typing import Any
|
||||
from ...core.logger import logger
|
||||
|
||||
|
||||
def _get_verify() -> bool:
|
||||
"""Resolve SSL verify flag from LLM_SSL_VERIFY env var."""
|
||||
def _get_verify() -> str | bool:
|
||||
"""Resolve SSL verify from LLM_SSL_VERIFY env var.
|
||||
|
||||
Returns:
|
||||
- Path to system CA bundle when verification enabled
|
||||
(certifi bundle doesn't include corporate CA certs)
|
||||
- False when LLM_SSL_VERIFY is set to false/0/no/off
|
||||
"""
|
||||
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
|
||||
return raw not in ("false", "0", "no", "off")
|
||||
if raw in ("false", "0", "no", "off"):
|
||||
return False
|
||||
return "/etc/ssl/certs/ca-certificates.crt"
|
||||
|
||||
|
||||
# #region call_openai_compatible [C:3] [TYPE Function] [SEMANTICS translate, llm, http, openai]
|
||||
|
||||
@@ -10,10 +10,18 @@ from typing import Any
|
||||
from ...core.logger import logger
|
||||
|
||||
|
||||
def _get_verify() -> bool:
|
||||
"""Resolve SSL verify flag from LLM_SSL_VERIFY env var."""
|
||||
def _get_verify() -> str | bool:
|
||||
"""Resolve SSL verify from LLM_SSL_VERIFY env var.
|
||||
|
||||
Returns:
|
||||
- Path to system CA bundle when verification enabled
|
||||
(certifi bundle doesn't include corporate CA certs)
|
||||
- False when LLM_SSL_VERIFY is set to false/0/no/off
|
||||
"""
|
||||
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
|
||||
return raw not in ("false", "0", "no", "off")
|
||||
if raw in ("false", "0", "no", "off"):
|
||||
return False
|
||||
return "/etc/ssl/certs/ca-certificates.crt"
|
||||
|
||||
|
||||
class LLMClient:
|
||||
|
||||
111
docs/adr/ADR-0009-ssl-certificate-management.md
Normal file
111
docs/adr/ADR-0009-ssl-certificate-management.md
Normal file
@@ -0,0 +1,111 @@
|
||||
# [DEF:ADR-0009:ADR]
|
||||
# @STATUS ACTIVE
|
||||
# @PURPOSE Define the strategy for corporate SSL certificate management across all LLM HTTP clients in ss-tools (httpx, requests, openai), covering system CA store, NSS database for Chromium/Playwright, and the LLM_SSL_VERIFY escape hatch.
|
||||
# @RELATION DEPENDS_ON -> [ADR-0001:ADR]
|
||||
# @RELATION DEPENDS_ON -> [ADR-0004:ADR]
|
||||
# @RELATION CALLS -> [docker/backend.entrypoint.sh]
|
||||
# @RELATION CALLS -> [backend/src/plugins/llm_analysis/service.py]
|
||||
# @RELATION CALLS -> [backend/src/plugins/translate/_llm_http.py]
|
||||
# @RELATION CALLS -> [backend/src/plugins/translate/preview_llm_client.py]
|
||||
# @RATIONALE ss-tools operates in corporate environments with internal Certificate Authorities.
|
||||
# Three separate HTTP stacks are used: httpx (AsyncOpenAI in llm_analysis plugin),
|
||||
# requests (sync translate plugin), and Playwright Chromium (dashboard screenshots).
|
||||
# Each has its own CA trust store, requiring different installation strategies.
|
||||
# @REJECTED Centralizing all LLM calls into a single HTTP client — rejected because
|
||||
# llm_analysis uses async httpx+AsyncOpenAI for streaming and multiple concurrent calls,
|
||||
# while translate uses sync requests for simpler request/response. Merging would
|
||||
# require rewriting one or both, creating regression risk with no business value.
|
||||
|
||||
## Problem
|
||||
|
||||
Corporate SSL certificates installed via `update-ca-certificates` into `/etc/ssl/certs/ca-certificates.crt` are NOT automatically trusted by:
|
||||
|
||||
1. **Python `requests` library** — uses bundled `certifi` CA bundle, not system store
|
||||
2. **Python `httpx` library** — uses `certifi` by default when `verify=True`
|
||||
3. **Playwright Chromium** — uses NSS Shared DB (`~/.pki/nssdb`), not OpenSSL store
|
||||
|
||||
This causes `SSLError: CERTIFICATE_VERIFY_FAILED` and `ERR_CERT_AUTHORITY_INVALID` even after correct system-wide CA installation.
|
||||
|
||||
## Solution
|
||||
|
||||
### Layer 1: System CA Store (OpenSSL)
|
||||
|
||||
- Entrypoint `install_certificates()` copies `.crt/.pem` files from `CERTS_PATH` volume mount
|
||||
- Entrypoint `install_llm_ca_certs()` downloads PEM/DER certificates from `LLM_CA_CERT_URLS`
|
||||
- DER is auto-converted to PEM (`openssl x509 -inform DER -outform PEM`)
|
||||
- `update-ca-certificates --fresh` adds them to `/etc/ssl/certs/ca-certificates.crt`
|
||||
- **Fallback**: if `update-ca-certificates` misses certs (Debian Bookworm bug with subdirectories), they are appended to `ca-certificates.crt` with SHA256 fingerprint dedup
|
||||
|
||||
### Layer 2: Python HTTP Clients
|
||||
|
||||
| Library | File | Mechanism |
|
||||
|---------|------|-----------|
|
||||
| `httpx` (AsyncOpenAI) | `service.py:LLMClient._get_ssl_verify()` | Returns `ssl.create_default_context(cafile="/etc/ssl/certs/ca-certificates.crt")` |
|
||||
| `requests` (`_llm_http.py`) | `_get_verify()` | Returns `"/etc/ssl/certs/ca-certificates.crt"` (string path) |
|
||||
| `requests` (`preview_llm_client.py`) | `_get_verify()` | Returns `"/etc/ssl/certs/ca-certificates.crt"` (string path) |
|
||||
|
||||
Key insight: `verify=True` uses certifi, NOT system CA. All three functions now return the system CA path instead of boolean `True`.
|
||||
|
||||
### Layer 3: NSS Database (Playwright Chromium)
|
||||
|
||||
- Entrypoint `install_ca_to_nss()` imports PEM certs into `~/.pki/nssdb/` using `certutil`
|
||||
- Nickname format: `{dir_prefix}-{filename}` (e.g., `llm-UC_RUSAL_Policy_CA`)
|
||||
- Dedup by SHA256 fingerprint (not nickname), preventing duplicate imports
|
||||
- Trust attributes: `"C,,"` (trusted CA for TLS server certs)
|
||||
- Fixes `ERR_CERT_AUTHORITY_INVALID` in Playwright dashboard screenshots
|
||||
|
||||
### Layer 4: LLM_SSL_VERIFY Escape Hatch
|
||||
|
||||
- Env var `LLM_SSL_VERIFY=false` disables SSL verification entirely
|
||||
- Accepted values for false: `false`, `0`, `no`, `off` (case-insensitive)
|
||||
- Default: enabled (returns system CA path / SSLContext)
|
||||
- Used by all three HTTP client implementations
|
||||
|
||||
## Key Files
|
||||
|
||||
| File | Role |
|
||||
|------|------|
|
||||
| `docker/backend.Dockerfile` | Installs `libnss3-tools` (certutil), Playwright Chromium |
|
||||
| `docker/backend.entrypoint.sh` | `install_certificates`, `install_llm_ca_certs`, `install_ca_to_nss` |
|
||||
| `plugins/llm_analysis/service.py` | `LLMClient._get_ssl_verify()` → `ssl.create_default_context(cafile=...)` |
|
||||
| `plugins/translate/_llm_http.py` | `_get_verify()` → `"/etc/ssl/certs/ca-certificates.crt"` |
|
||||
| `plugins/translate/preview_llm_client.py` | `_get_verify()` → `"/etc/ssl/certs/ca-certificates.crt"` |
|
||||
| `docker-compose.yml` | Passes `LLM_SSL_VERIFY`, `LLM_CA_CERT_URLS` |
|
||||
| `docker-compose.enterprise-clean.yml` | Same as above |
|
||||
| `.env.enterprise-clean` | Default values for both env vars |
|
||||
|
||||
## Discovered Findings
|
||||
|
||||
### Finding 1: certifi vs system CA
|
||||
`requests` and `httpx` use `certifi` CA bundle by default. Corporate CA certs installed into `/etc/ssl/certs/` are invisible to Python HTTP clients unless explicitly pointed to the system bundle.
|
||||
|
||||
### Finding 2: Three separate HTTP stacks
|
||||
ss-tools has three LLM HTTP clients (httpx for async, requests for sync, Playwright for screenshots), each with independent CA trust configuration.
|
||||
|
||||
### Finding 3: update-ca-certificates inconsistency
|
||||
On Debian Bookworm (python:3.11-slim), `update-ca-certificates` occasionally skips symlinks for PEM files in subdirectories (`/usr/local/share/ca-certificates/custom/`, `llm/`), requiring a manual `cat >> ca-certificates.crt` fallback and explicit hash symlink creation.
|
||||
|
||||
### Finding 4: DER format from corporate PKI
|
||||
Corporate PKI servers (`pki.rusal.com`) often serve certificates in DER (binary) format. `update-ca-certificates` requires PEM. Auto-detection and conversion (`openssl x509 -inform DER -outform PEM`) is essential.
|
||||
|
||||
### Finding 5: Chicken-and-egg TLS bootstrap
|
||||
Downloading a CA certificate from a PKI server that uses the same CA causes a TLS verification loop. Entrypoint uses `curl --insecure` as fallback when initial `curl` fails with TLS error.
|
||||
|
||||
### Finding 6: NSS DB format
|
||||
Chromium uses NSS Shared DB in `~/.pki/nssdb/`. The `sql:` prefix is required for `certutil` to use the SQLite format. Without it, certutil defaults to the legacy DBM format which Chromium may not read.
|
||||
|
||||
## Deploy
|
||||
|
||||
```bash
|
||||
# On target server:
|
||||
xz -dc ss-tools-backend.0.1.6.tar.xz | docker load
|
||||
xz -dc ss-tools-frontend.0.1.6.tar.xz | docker load
|
||||
|
||||
docker compose -f docker-compose.enterprise-clean.yml \
|
||||
--env-file .env.enterprise-clean down
|
||||
|
||||
docker compose -f docker-compose.enterprise-clean.yml \
|
||||
--env-file .env.enterprise-clean up -d
|
||||
```
|
||||
|
||||
# [/DEF:ADR-0009:ADR]
|
||||
Reference in New Issue
Block a user