refactor(ssl): centralize SSL trust management, remove LLM_SSL_VERIFY
Centralized SSL via one contract: CERTS_PATH=/opt/certs mounted into all containers.
Backend:
- NEW: backend/src/core/ssl.py — system_ssl_context(), httpx_verify(),
cert_dir_inventory()
- LLMClient._get_ssl_verify() → delegates to core.ssl
- _llm_async_http._get_verify() → delegates to core.ssl
- Removed LLM_SSL_VERIFY env reading from all runtime code
Docker:
- NEW: docker/certs.sh — shared cert installer (PEM/DER/cer to .crt conversion,
update-ca-certificates, hash symlinks, NSS import)
- NEW: docker/agent.entrypoint.sh — agent entrypoint with cert installation
- backend.entrypoint.sh → uses certs.sh instead of install_llm_ca_certs
- Dockerfile.agent → adds ca-certificates, openssl, entrypoint
Compose:
- Removed LLM_CA_CERT_URLS and LLM_SSL_VERIFY from all compose files
- Added CERTS_PATH volume mount to agent (dev + enterprise)
- Added certs volume mount to backend/agent in dev compose
Env examples:
- Removed LLM_SSL_VERIFY, LLM_CA_CERT_URLS from .env.example,
.env.enterprise-clean.example, .env.current.example, .env.master.example,
backend/.env.example
- Enhanced CERTS_PATH comments with accepted formats
Diagnostics:
- diag_container.py: removed LLM_* checks, added CERTS_PATH inventory,
uses core.ssl for context creation
Tests:
- Updated test_llm_analysis_service, test_llm_async_http,
test_client_headers to verify centralized ssl context (no env disable)
- 4/4 SSL tests pass
This commit is contained in:
@@ -46,12 +46,7 @@ def section(title: str) -> None:
|
||||
def check_env() -> None:
|
||||
section("1. Environment variables")
|
||||
|
||||
for var in (
|
||||
"ENCRYPTION_KEY",
|
||||
"AUTH_SECRET_KEY",
|
||||
"LLM_SSL_VERIFY",
|
||||
"LLM_CA_CERT_URLS",
|
||||
):
|
||||
for var in ("ENCRYPTION_KEY", "AUTH_SECRET_KEY", "CERTS_PATH"):
|
||||
val = os.getenv(var, "")
|
||||
if not val:
|
||||
warn(f"{var} is NOT set")
|
||||
@@ -64,25 +59,36 @@ def check_env() -> None:
|
||||
fp = hashlib.sha256(os.getenv("ENCRYPTION_KEY", "").encode()).hexdigest()[:8]
|
||||
print(f"\n Current key fingerprint: {BOLD}sha256:{fp}{RESET}")
|
||||
|
||||
if os.getenv("LLM_SSL_VERIFY", "").lower() in ("false", "0", "no", "off"):
|
||||
fail(
|
||||
"LLM_SSL_VERIFY is DISABLED — SSL verification bypassed.\n"
|
||||
" This masks the root cause. Remove LLM_SSL_VERIFY=false\n"
|
||||
" and fix cert installation instead."
|
||||
)
|
||||
else:
|
||||
ok(
|
||||
"LLM_SSL_VERIFY is ENABLED (capath approach).\n"
|
||||
" If LLM calls fail, the certificate for the target server\n"
|
||||
" is not in /etc/ssl/certs/."
|
||||
)
|
||||
|
||||
# ── Section 2: CERTS_PATH inventory ──────────────────────────────────
|
||||
|
||||
|
||||
# ── Section 2: System CA store ──────────────────────────────────────
|
||||
def check_certs_inventory() -> None:
|
||||
section("2. CERTS_PATH inventory (/opt/certs)")
|
||||
|
||||
from src.core.ssl import cert_dir_inventory
|
||||
|
||||
inv = cert_dir_inventory("/opt/certs")
|
||||
if not inv["trust_candidates"] and not inv["server_files"]:
|
||||
warn("CERTS_PATH is empty or not mounted — no corporate CA certs found")
|
||||
print(
|
||||
" Put .crt/.pem/.cer/.der CA files in ./certs/ on the host and restart containers."
|
||||
)
|
||||
return
|
||||
|
||||
for name in inv["trust_candidates"]:
|
||||
ok(f"Trust candidate: {Path(name).name}")
|
||||
for name in inv["server_files"]:
|
||||
ok(f"Server file (skipped as CA): {Path(name).name}")
|
||||
for name in inv["invalid"]:
|
||||
warn(f"Unrecognized file (skipped): {Path(name).name}")
|
||||
|
||||
|
||||
# ── Section 3: System CA store ──────────────────────────────────────
|
||||
|
||||
|
||||
def check_system_certs() -> None:
|
||||
section("2. System CA store (/etc/ssl/certs/)")
|
||||
section("3. System CA store (/etc/ssl/certs/)")
|
||||
|
||||
bundle = Path("/etc/ssl/certs/ca-certificates.crt")
|
||||
if not bundle.exists():
|
||||
@@ -96,78 +102,38 @@ def check_system_certs() -> None:
|
||||
).stdout.strip()
|
||||
ok(f"ca-certificates.crt exists, {cert_count} certificates in bundle")
|
||||
|
||||
certs_dir = Path("/etc/ssl/certs")
|
||||
custom_pems = list(Path("/usr/local/share/ca-certificates").rglob("*.crt"))
|
||||
llm_pems = list(Path("/usr/local/share/ca-certificates/llm").rglob("*.crt"))
|
||||
|
||||
custom_pems = list(Path("/usr/local/share/ca-certificates/custom").rglob("*.crt"))
|
||||
if custom_pems:
|
||||
names = [p.name for p in custom_pems]
|
||||
ok(f"Installed CA certs: {', '.join(names)}")
|
||||
else:
|
||||
warn("No custom .crt files in /usr/local/share/ca-certificates/")
|
||||
warn("No custom .crt files in /usr/local/share/ca-certificates/custom/")
|
||||
|
||||
if llm_pems:
|
||||
names = [p.name for p in llm_pems]
|
||||
ok(f"LLM CA certs: {', '.join(names)}")
|
||||
else:
|
||||
warn(
|
||||
"No LLM CA certs downloaded.\n"
|
||||
" LLM_CA_CERT_URLS is not set — set it in .env.enterprise-clean\n"
|
||||
" or place the CA .crt in ./certs/ on the host."
|
||||
)
|
||||
|
||||
# Check hash symlinks
|
||||
certs_dir = Path("/etc/ssl/certs")
|
||||
hash_links = [p for p in certs_dir.iterdir() if p.is_symlink()]
|
||||
llm_links = [l for l in hash_links if "llm" in str(Path(os.readlink(str(l))))]
|
||||
if llm_links:
|
||||
ok(f"Hash symlinks for LLM certs: {len(llm_links)}")
|
||||
custom_links = [l for l in hash_links if "custom" in str(Path(os.readlink(str(l))))]
|
||||
if custom_links:
|
||||
ok(f"Hash symlinks for custom certs: {len(custom_links)}")
|
||||
else:
|
||||
warn("No hash symlinks pointing to LLM cert dir — capath may not find them")
|
||||
warn("No hash symlinks for custom certs — capath may not find them")
|
||||
|
||||
|
||||
# ── Section 3: OpenSSL connectivity ──────────────────────────────────
|
||||
# ── Section 4: OpenSSL connectivity ──────────────────────────────────
|
||||
|
||||
|
||||
def check_openssl(target: str) -> None:
|
||||
section(f"3. OpenSSL connectivity ({target})")
|
||||
section(f"4. OpenSSL connectivity ({target})")
|
||||
|
||||
# cafile approach (expected to FAIL with code 20 per ADR-0009 Finding 7)
|
||||
cafile = "/etc/ssl/certs/ca-certificates.crt"
|
||||
r = subprocess.run(
|
||||
[
|
||||
"openssl",
|
||||
"s_client",
|
||||
"-connect",
|
||||
target,
|
||||
"-servername",
|
||||
target.split(":")[0],
|
||||
"-CAfile",
|
||||
cafile,
|
||||
],
|
||||
input="",
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
)
|
||||
if "Verify return code: 0" in r.stderr or "Verify return code: 0" in r.stdout:
|
||||
ok(f"cafile: verify OK (unexpected — usually fails per ADR-0009 Finding 7)")
|
||||
elif "Verify return code: 20" in (r.stderr + r.stdout):
|
||||
warn(
|
||||
f"cafile: verify code 20 (EXPECTED per ADR-0009 — intermediate CA ignored)"
|
||||
)
|
||||
else:
|
||||
fail(f"cafile: unexpected output, returncode={r.returncode}")
|
||||
|
||||
# capath approach (should succeed)
|
||||
capath = "/etc/ssl/certs/"
|
||||
host, port = _parse_target(target)
|
||||
r = subprocess.run(
|
||||
[
|
||||
"openssl",
|
||||
"s_client",
|
||||
"-connect",
|
||||
target,
|
||||
f"{host}:{port}",
|
||||
"-servername",
|
||||
target.split(":")[0],
|
||||
host,
|
||||
"-CApath",
|
||||
capath,
|
||||
],
|
||||
@@ -181,21 +147,31 @@ def check_openssl(target: str) -> None:
|
||||
else:
|
||||
fail(
|
||||
f"capath: verify FAILED.\n"
|
||||
f" The CA certificate for {target} is NOT in /etc/ssl/certs/.\n"
|
||||
f" Add it via LLM_CA_CERT_URLS or ./certs/ on the host."
|
||||
f" The CA certificate for {host} is NOT in /etc/ssl/certs/.\n"
|
||||
f" Put the issuing/root CA .crt into CERTS_PATH (./certs)\n"
|
||||
f" and restart the container."
|
||||
)
|
||||
|
||||
|
||||
# ── Section 4: Python SSL context ────────────────────────────────────
|
||||
def _parse_target(target: str) -> tuple[str, int]:
|
||||
if "://" in target:
|
||||
target = target.split("://")[1]
|
||||
if ":" in target:
|
||||
host, port = target.rsplit(":", 1)
|
||||
return host, int(port)
|
||||
return target, 443
|
||||
|
||||
|
||||
# ── Section 5: Python SSL context ────────────────────────────────────
|
||||
|
||||
|
||||
def check_python_ssl(target: str) -> None:
|
||||
section(f"4. Python SSL context ({target})")
|
||||
section(f"5. Python SSL context ({target})")
|
||||
|
||||
# capath approach (what _get_ssl_verify() uses)
|
||||
ctx = ssl.create_default_context(capath="/etc/ssl/certs/")
|
||||
host, port = target.split(":") if ":" in target else (target, 443)
|
||||
port = int(port)
|
||||
from src.core.ssl import system_ssl_context
|
||||
|
||||
ctx = system_ssl_context()
|
||||
host, port = _parse_target(target)
|
||||
|
||||
try:
|
||||
with ctx.wrap_socket(__import__("socket").socket(), server_hostname=host) as s:
|
||||
@@ -208,16 +184,17 @@ def check_python_ssl(target: str) -> None:
|
||||
fail(f"SSLContext(capath): FAILED — {e}")
|
||||
|
||||
|
||||
# ── Section 5: httpx connectivity ────────────────────────────────────
|
||||
# ── Section 6: httpx connectivity ────────────────────────────────────
|
||||
|
||||
|
||||
def check_httpx(target: str) -> None:
|
||||
section(f"5. httpx connectivity ({target})")
|
||||
section(f"6. httpx connectivity ({target})")
|
||||
|
||||
try:
|
||||
import httpx
|
||||
from src.core.ssl import httpx_verify
|
||||
|
||||
ctx = ssl.create_default_context(capath="/etc/ssl/certs/")
|
||||
ctx = httpx_verify()
|
||||
url = f"https://{target}" if not target.startswith("http") else target
|
||||
r = httpx.get(url, verify=ctx, timeout=10, follow_redirects=True)
|
||||
ok(f"httpx(capath): HTTP {r.status_code}")
|
||||
@@ -227,11 +204,11 @@ def check_httpx(target: str) -> None:
|
||||
fail(f"httpx(capath): FAILED — {e}")
|
||||
|
||||
|
||||
# ── Section 6: Encryption health ─────────────────────────────────────
|
||||
# ── Section 7: Encryption health ─────────────────────────────────────
|
||||
|
||||
|
||||
def check_encryption_health() -> None:
|
||||
section("6. Encryption key health (internal)")
|
||||
section("7. Encryption key health (internal)")
|
||||
|
||||
sys.path.insert(0, "/app/backend")
|
||||
try:
|
||||
@@ -246,17 +223,15 @@ def check_encryption_health() -> None:
|
||||
ok(f"LLM providers found: {len(providers)}")
|
||||
for p in providers:
|
||||
if not p.api_key:
|
||||
warn(f" {p.name}: api_key is EMPTY — no key stored")
|
||||
warn(f" {p.name}: api_key is EMPTY")
|
||||
continue
|
||||
if not is_fernet_token(p.api_key):
|
||||
warn(
|
||||
f" {p.name}: api_key is NOT a fernet token — stored unencrypted?"
|
||||
)
|
||||
warn(f" {p.name}: api_key is NOT a fernet token")
|
||||
continue
|
||||
try:
|
||||
mgr.decrypt(p.api_key)
|
||||
ok(f" {p.name}: api_key decrypts OK (healthy)")
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
fail(
|
||||
f" {p.name}: api_key decrypt FAILED — "
|
||||
f"encrypted with different ENCRYPTION_KEY"
|
||||
@@ -291,6 +266,7 @@ def main() -> None:
|
||||
print(f"Target: {target}\n")
|
||||
|
||||
check_env()
|
||||
check_certs_inventory()
|
||||
check_system_certs()
|
||||
check_openssl(target)
|
||||
check_python_ssl(target)
|
||||
@@ -299,16 +275,11 @@ def main() -> None:
|
||||
|
||||
section("Summary")
|
||||
print(
|
||||
"\nExpected (per ADR-0009):\n"
|
||||
" - openssl CAfile: code 20 FAIL (intermediate CA ignored)\n"
|
||||
" - openssl CApath: code 0 OK\n"
|
||||
" - Python SSLContext(capath): OK\n"
|
||||
" - httpx(capath): HTTP 200 OK\n"
|
||||
"\nIf CApath/httpx failures:\n"
|
||||
" → LLM_CA_CERT_URLS not set → certs not downloaded\n"
|
||||
" → set LLM_CA_CERT_URLS or add .crt to ./certs/\n"
|
||||
"\nIf cert verification failures:\n"
|
||||
" → Put the issuing/root CA for target into CERTS_PATH (./certs)\n"
|
||||
" → Restart affected containers\n"
|
||||
" → Rerun this diagnostic\n"
|
||||
)
|
||||
|
||||
print(
|
||||
"\nIf ENCRYPTION_KEY health failures:\n"
|
||||
" → Stored secrets encrypted with old key\n"
|
||||
|
||||
Reference in New Issue
Block a user