From 19dd4473458457a3f7e492c7e2e5e1aba115a817 Mon Sep 17 00:00:00 2001 From: busya Date: Thu, 28 May 2026 12:06:42 +0300 Subject: [PATCH] feat: ADR-0009 SSL cert management, fix certifi vs system CA - ADR-0009: comprehensive SSL certificate management strategy - _get_ssl_verify() returns SSLContext with cafile= instead of bool True (httpx deprecates verify=, requests needs system CA, not certifi) - _get_verify() in translate plugin returns system CA path - All tests updated for SSLContext return type - All QA findings C1-C3, H1-H4, M1 incorporated into ADR --- .../__tests__/test_client_headers.py | 32 +++-- backend/src/plugins/llm_analysis/service.py | 21 +++- backend/src/plugins/translate/_llm_http.py | 14 ++- .../plugins/translate/preview_llm_client.py | 14 ++- .../ADR-0009-ssl-certificate-management.md | 111 ++++++++++++++++++ 5 files changed, 170 insertions(+), 22 deletions(-) create mode 100644 docs/adr/ADR-0009-ssl-certificate-management.md diff --git a/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py b/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py index ec280aa2..0f1d1bf1 100644 --- a/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py +++ b/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py @@ -54,15 +54,18 @@ def test_litellm_client_uses_default_bearer_auth(): # endregion test_litellm_client_uses_default_bearer_auth -# region test_get_ssl_verify_default_true [TYPE Function] +# region test_get_ssl_verify_default_context [TYPE Function] # @RELATION BINDS_TO -> TestClientHeaders -# @PURPOSE: _get_ssl_verify defaults to True when LLM_SSL_VERIFY is unset. +# @PURPOSE: _get_ssl_verify returns SSLContext with CERT_REQUIRED by default. # @PRE LLM_SSL_VERIFY env var is not set. -# @POST Returns True. -def test_get_ssl_verify_default_true(): - """Verify default SSL verification is enabled.""" - assert LLMClient._get_ssl_verify() is True -# endregion test_get_ssl_verify_default_true +# @POST Returns SSLContext with verify_mode=CERT_REQUIRED. +def test_get_ssl_verify_default_context(): + """Verify default returns SSLContext with CERT_REQUIRED.""" + import ssl + result = LLMClient._get_ssl_verify() + assert isinstance(result, ssl.SSLContext), "Expected SSLContext, not bool" + assert result.verify_mode == ssl.CERT_REQUIRED +# endregion test_get_ssl_verify_default_context # region test_get_ssl_verify_false_values [TYPE Function] @@ -77,16 +80,19 @@ def test_get_ssl_verify_false_values(monkeypatch): # endregion test_get_ssl_verify_false_values -# region test_get_ssl_verify_true_values [TYPE Function] +# region test_get_ssl_verify_true_returns_context [TYPE Function] # @RELATION BINDS_TO -> TestClientHeaders -# @PURPOSE: _get_ssl_verify returns True for truthy values like "TRUE", "1", "yes". +# @PURPOSE: _get_ssl_verify returns SSLContext for truthy env values. # @PRE LLM_SSL_VERIFY env var is set to truthy values. -# @POST Returns True. -def test_get_ssl_verify_true_values(monkeypatch): +# @POST Returns SSLContext with CERT_REQUIRED. +def test_get_ssl_verify_true_returns_context(monkeypatch): + import ssl for val in ("TRUE", "1", "yes", "on", "true"): monkeypatch.setenv("LLM_SSL_VERIFY", val) - assert LLMClient._get_ssl_verify() is True, f"Expected True for LLM_SSL_VERIFY={val}" -# endregion test_get_ssl_verify_true_values + result = LLMClient._get_ssl_verify() + assert isinstance(result, ssl.SSLContext), f"Expected SSLContext for LLM_SSL_VERIFY={val}" + assert result.verify_mode == ssl.CERT_REQUIRED +# endregion test_get_ssl_verify_true_returns_path # region test_format_connection_error_no_cause [TYPE Function] diff --git a/backend/src/plugins/llm_analysis/service.py b/backend/src/plugins/llm_analysis/service.py index 750577fc..c194d3a4 100644 --- a/backend/src/plugins/llm_analysis/service.py +++ b/backend/src/plugins/llm_analysis/service.py @@ -13,6 +13,7 @@ import json import os import re import shutil +import ssl import tempfile from typing import Any from urllib.parse import urlsplit @@ -923,11 +924,25 @@ class LLMClient: # region LLMClient._get_ssl_verify [TYPE Function] # @PURPOSE Resolve SSL verification flag from environment. - # @POST Returns False when LLM_SSL_VERIFY env var is "false"/"0"/"no" (case-insensitive). + # @POST Returns SSLContext with system CA bundle when enabled, + # False when LLM_SSL_VERIFY env var is "false"/"0"/"no"/"off". + # @RATIONALE Возвращаем ssl.SSLContext вместо True, потому что httpx по + # умолчанию использует certifi, в котором нет корпоративных CA. + # ssl.create_default_context(cafile=...) гарантирует, что используются + # сертификаты из системного хранилища (/etc/ssl/certs/ca-certificates.crt), + # куда update-ca-certificates и наш fallback добавляют корпоративные CA. + # @REJECTED verify= отвергнут — httpx 0.28.x депрекейтит строковый + # путь в verify=, требует SSLContext. @staticmethod - def _get_ssl_verify() -> bool: + def _get_ssl_verify() -> ssl.SSLContext | bool: raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower() - return raw not in ("false", "0", "no", "off") + if raw in ("false", "0", "no", "off"): + return False + ca_path = "/etc/ssl/certs/ca-certificates.crt" + if os.path.exists(ca_path): + return ssl.create_default_context(cafile=ca_path) + # fallback: если системного CA нет (редко), используем дефолтный + return ssl.create_default_context() # endregion LLMClient._get_ssl_verify # region LLMClient._format_connection_error [TYPE Function] diff --git a/backend/src/plugins/translate/_llm_http.py b/backend/src/plugins/translate/_llm_http.py index d3127edd..d034852e 100644 --- a/backend/src/plugins/translate/_llm_http.py +++ b/backend/src/plugins/translate/_llm_http.py @@ -18,10 +18,18 @@ from typing import Any from ...core.logger import logger -def _get_verify() -> bool: - """Resolve SSL verify flag from LLM_SSL_VERIFY env var.""" +def _get_verify() -> str | bool: + """Resolve SSL verify from LLM_SSL_VERIFY env var. + + Returns: + - Path to system CA bundle when verification enabled + (certifi bundle doesn't include corporate CA certs) + - False when LLM_SSL_VERIFY is set to false/0/no/off + """ raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower() - return raw not in ("false", "0", "no", "off") + if raw in ("false", "0", "no", "off"): + return False + return "/etc/ssl/certs/ca-certificates.crt" # #region call_openai_compatible [C:3] [TYPE Function] [SEMANTICS translate, llm, http, openai] diff --git a/backend/src/plugins/translate/preview_llm_client.py b/backend/src/plugins/translate/preview_llm_client.py index e3fd58a8..eed7725b 100644 --- a/backend/src/plugins/translate/preview_llm_client.py +++ b/backend/src/plugins/translate/preview_llm_client.py @@ -10,10 +10,18 @@ from typing import Any from ...core.logger import logger -def _get_verify() -> bool: - """Resolve SSL verify flag from LLM_SSL_VERIFY env var.""" +def _get_verify() -> str | bool: + """Resolve SSL verify from LLM_SSL_VERIFY env var. + + Returns: + - Path to system CA bundle when verification enabled + (certifi bundle doesn't include corporate CA certs) + - False when LLM_SSL_VERIFY is set to false/0/no/off + """ raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower() - return raw not in ("false", "0", "no", "off") + if raw in ("false", "0", "no", "off"): + return False + return "/etc/ssl/certs/ca-certificates.crt" class LLMClient: diff --git a/docs/adr/ADR-0009-ssl-certificate-management.md b/docs/adr/ADR-0009-ssl-certificate-management.md new file mode 100644 index 00000000..3109e154 --- /dev/null +++ b/docs/adr/ADR-0009-ssl-certificate-management.md @@ -0,0 +1,111 @@ +# [DEF:ADR-0009:ADR] +# @STATUS ACTIVE +# @PURPOSE Define the strategy for corporate SSL certificate management across all LLM HTTP clients in ss-tools (httpx, requests, openai), covering system CA store, NSS database for Chromium/Playwright, and the LLM_SSL_VERIFY escape hatch. +# @RELATION DEPENDS_ON -> [ADR-0001:ADR] +# @RELATION DEPENDS_ON -> [ADR-0004:ADR] +# @RELATION CALLS -> [docker/backend.entrypoint.sh] +# @RELATION CALLS -> [backend/src/plugins/llm_analysis/service.py] +# @RELATION CALLS -> [backend/src/plugins/translate/_llm_http.py] +# @RELATION CALLS -> [backend/src/plugins/translate/preview_llm_client.py] +# @RATIONALE ss-tools operates in corporate environments with internal Certificate Authorities. +# Three separate HTTP stacks are used: httpx (AsyncOpenAI in llm_analysis plugin), +# requests (sync translate plugin), and Playwright Chromium (dashboard screenshots). +# Each has its own CA trust store, requiring different installation strategies. +# @REJECTED Centralizing all LLM calls into a single HTTP client — rejected because +# llm_analysis uses async httpx+AsyncOpenAI for streaming and multiple concurrent calls, +# while translate uses sync requests for simpler request/response. Merging would +# require rewriting one or both, creating regression risk with no business value. + +## Problem + +Corporate SSL certificates installed via `update-ca-certificates` into `/etc/ssl/certs/ca-certificates.crt` are NOT automatically trusted by: + +1. **Python `requests` library** — uses bundled `certifi` CA bundle, not system store +2. **Python `httpx` library** — uses `certifi` by default when `verify=True` +3. **Playwright Chromium** — uses NSS Shared DB (`~/.pki/nssdb`), not OpenSSL store + +This causes `SSLError: CERTIFICATE_VERIFY_FAILED` and `ERR_CERT_AUTHORITY_INVALID` even after correct system-wide CA installation. + +## Solution + +### Layer 1: System CA Store (OpenSSL) + +- Entrypoint `install_certificates()` copies `.crt/.pem` files from `CERTS_PATH` volume mount +- Entrypoint `install_llm_ca_certs()` downloads PEM/DER certificates from `LLM_CA_CERT_URLS` +- DER is auto-converted to PEM (`openssl x509 -inform DER -outform PEM`) +- `update-ca-certificates --fresh` adds them to `/etc/ssl/certs/ca-certificates.crt` +- **Fallback**: if `update-ca-certificates` misses certs (Debian Bookworm bug with subdirectories), they are appended to `ca-certificates.crt` with SHA256 fingerprint dedup + +### Layer 2: Python HTTP Clients + +| Library | File | Mechanism | +|---------|------|-----------| +| `httpx` (AsyncOpenAI) | `service.py:LLMClient._get_ssl_verify()` | Returns `ssl.create_default_context(cafile="/etc/ssl/certs/ca-certificates.crt")` | +| `requests` (`_llm_http.py`) | `_get_verify()` | Returns `"/etc/ssl/certs/ca-certificates.crt"` (string path) | +| `requests` (`preview_llm_client.py`) | `_get_verify()` | Returns `"/etc/ssl/certs/ca-certificates.crt"` (string path) | + +Key insight: `verify=True` uses certifi, NOT system CA. All three functions now return the system CA path instead of boolean `True`. + +### Layer 3: NSS Database (Playwright Chromium) + +- Entrypoint `install_ca_to_nss()` imports PEM certs into `~/.pki/nssdb/` using `certutil` +- Nickname format: `{dir_prefix}-{filename}` (e.g., `llm-UC_RUSAL_Policy_CA`) +- Dedup by SHA256 fingerprint (not nickname), preventing duplicate imports +- Trust attributes: `"C,,"` (trusted CA for TLS server certs) +- Fixes `ERR_CERT_AUTHORITY_INVALID` in Playwright dashboard screenshots + +### Layer 4: LLM_SSL_VERIFY Escape Hatch + +- Env var `LLM_SSL_VERIFY=false` disables SSL verification entirely +- Accepted values for false: `false`, `0`, `no`, `off` (case-insensitive) +- Default: enabled (returns system CA path / SSLContext) +- Used by all three HTTP client implementations + +## Key Files + +| File | Role | +|------|------| +| `docker/backend.Dockerfile` | Installs `libnss3-tools` (certutil), Playwright Chromium | +| `docker/backend.entrypoint.sh` | `install_certificates`, `install_llm_ca_certs`, `install_ca_to_nss` | +| `plugins/llm_analysis/service.py` | `LLMClient._get_ssl_verify()` → `ssl.create_default_context(cafile=...)` | +| `plugins/translate/_llm_http.py` | `_get_verify()` → `"/etc/ssl/certs/ca-certificates.crt"` | +| `plugins/translate/preview_llm_client.py` | `_get_verify()` → `"/etc/ssl/certs/ca-certificates.crt"` | +| `docker-compose.yml` | Passes `LLM_SSL_VERIFY`, `LLM_CA_CERT_URLS` | +| `docker-compose.enterprise-clean.yml` | Same as above | +| `.env.enterprise-clean` | Default values for both env vars | + +## Discovered Findings + +### Finding 1: certifi vs system CA +`requests` and `httpx` use `certifi` CA bundle by default. Corporate CA certs installed into `/etc/ssl/certs/` are invisible to Python HTTP clients unless explicitly pointed to the system bundle. + +### Finding 2: Three separate HTTP stacks +ss-tools has three LLM HTTP clients (httpx for async, requests for sync, Playwright for screenshots), each with independent CA trust configuration. + +### Finding 3: update-ca-certificates inconsistency +On Debian Bookworm (python:3.11-slim), `update-ca-certificates` occasionally skips symlinks for PEM files in subdirectories (`/usr/local/share/ca-certificates/custom/`, `llm/`), requiring a manual `cat >> ca-certificates.crt` fallback and explicit hash symlink creation. + +### Finding 4: DER format from corporate PKI +Corporate PKI servers (`pki.rusal.com`) often serve certificates in DER (binary) format. `update-ca-certificates` requires PEM. Auto-detection and conversion (`openssl x509 -inform DER -outform PEM`) is essential. + +### Finding 5: Chicken-and-egg TLS bootstrap +Downloading a CA certificate from a PKI server that uses the same CA causes a TLS verification loop. Entrypoint uses `curl --insecure` as fallback when initial `curl` fails with TLS error. + +### Finding 6: NSS DB format +Chromium uses NSS Shared DB in `~/.pki/nssdb/`. The `sql:` prefix is required for `certutil` to use the SQLite format. Without it, certutil defaults to the legacy DBM format which Chromium may not read. + +## Deploy + +```bash +# On target server: +xz -dc ss-tools-backend.0.1.6.tar.xz | docker load +xz -dc ss-tools-frontend.0.1.6.tar.xz | docker load + +docker compose -f docker-compose.enterprise-clean.yml \ + --env-file .env.enterprise-clean down + +docker compose -f docker-compose.enterprise-clean.yml \ + --env-file .env.enterprise-clean up -d +``` + +# [/DEF:ADR-0009:ADR]