fix(docker): add missing certs.sh COPY to backend/all-in-one Dockerfile

Backend entrypoint sources certs.sh for CA certificate installation,
but the file was never copied into the Docker image — causing
container crash loop on startup (regression in 8fd23f7e).

Changes:
- docker/backend.Dockerfile, docker/all-in-one.Dockerfile: COPY certs.sh
- docker/backend.entrypoint.sh: remove dead install_llm_ca_certs,
  install_certificates (replaced by docker/certs.sh); update @INVARIANT
- backend/src/core/ssl.py: fix describe_context() to report actual
  system cert count (SSLContext.capath attr does not exist in Python 3)
- __tests__: rewrite stale tests asserting LLM_SSL_VERIFY=false →
  verify=False; behaviour is permanently removed — always CERT_REQUIRED
- backend/tests/integration/test_backend_container.py [new]: 5 tests
  verifying certs.sh presence, sourceability, and full-stack smoke
  (testcontainers PG → entrypoint → migrations → health)
- conftest.py: restore health-wait loop and fixtures in superset_container
- ADR-0009, README.md, scripts, .env: align docs with centralized SSL
This commit is contained in:
2026-07-07 00:58:41 +03:00
parent 98d91bb738
commit 1e61f098bd
12 changed files with 645 additions and 371 deletions

View File

@@ -303,24 +303,19 @@ Entrypoint автоматически:
Положите `.crt`/`.pem` файлы в `./certs/` — entrypoint установит их в системное хранилище Alpine и NSS (Chromium/Playwright). Поддерживаются цепочки из нескольких CA (Root → Intermediate).
### LLM CA-сертификаты
Если LLM-провайдер использует корпоративный PKI — укажите URL для скачивания CA-сертификатов:
```bash
LLM_CA_CERT_URLS="http://pki.company.com/root-ca.crt http://pki.company.com/intermediate-ca.crt"
```
Сертификаты скачиваются на старте backend-контейнера, конвертируются из DER в PEM при необходимости и устанавливаются в системное хранилище. Подробнее — в [ADR-0009](docs/adr/ADR-0009-ssl-certificate-management.md).
### Диагностика
### Диагностика SSL
```bash
# Проверка доверия сертификата
# Полная проверка цепочки доверия
scripts/check_llm_certs.py
# Принудительное отключение TLS verify (только для отладки)
LLM_SSL_VERIFY=false
# Контейнерная диагностика
scripts/diag_container.py --target your-llm-provider.com:443
```
Сертификаты и централизованный SSL — подробнее в [ADR-0009](docs/adr/ADR-0009-ssl-certificate-management.md).
`LLM_SSL_VERIFY` и `LLM_CA_CERT_URLS` удалены в 0.2.x — используйте `CERTS_PATH=./certs`.
## 🏢 Enterprise Clean Deployment
Для разворота в корпоративной сети с очищенным дистрибутивом (без тестовых данных, с запретом внешних источников и обязательной compliance-проверкой) используется профиль **enterprise clean**.

View File

@@ -51,21 +51,22 @@ def describe_context(ctx: ssl.SSLContext | bool | None) -> str:
"""Return a diagnostic description of the SSL verification state.
Does NOT leak key material or secret values.
SSLContext in Python 3.13 does not expose capath/cafile as public
attributes — verify_load_locations info is internal to OpenSSL.
Instead, reports whether /etc/ssl/certs (DEFAULT_CAPATH) is available
and the cert count in the system bundle.
"""
if ctx is False:
return "DISABLED (verify=False — should not happen in production)"
if ctx is None:
return "DEFAULT (certifi)"
if isinstance(ctx, ssl.SSLContext):
capath = getattr(ctx, "capath", None) or getattr(ctx, "_capath", None)
cafile = getattr(ctx, "cafile", None) or getattr(ctx, "_cafile", None)
parts = []
if cafile:
parts.append(f"cafile={cafile}")
if capath:
parts.append(f"capath={capath}")
parts.append(f"verify_mode={'CERT_REQUIRED' if ctx.verify_mode == ssl.CERT_REQUIRED else ctx.verify_mode}")
return "SSLContext(" + ", ".join(parts) + ")"
bundle = Path(DEFAULT_CAPATH) / "ca-certificates.crt"
cert_count = 0
if bundle.exists():
cert_count = bundle.read_bytes().count(b"-----BEGIN CERTIFICATE-----")
mode = "CERT_REQUIRED" if ctx.verify_mode == ssl.CERT_REQUIRED else str(ctx.verify_mode)
return f"SSLContext(verify_mode={mode}, system_certs={cert_count})"
return f"unknown({type(ctx).__name__})"

View File

@@ -27,6 +27,8 @@ def test_openrouter_client_includes_referer_and_title_headers(monkeypatch):
assert headers["Authorization"] == "Bearer sk-test-provider-key-123456"
assert headers["HTTP-Referer"] == "http://localhost:8000"
assert headers["X-Title"] == "superset-tools-test"
# endregion test_openrouter_client_includes_referer_and_title_headers
@@ -51,48 +53,44 @@ def test_litellm_client_uses_default_bearer_auth():
assert "X-Title" not in headers
assert "Authentication" not in headers
assert "X-API-Key" not in headers
# endregion test_litellm_client_uses_default_bearer_auth
# region test_get_ssl_verify_default_context [TYPE Function]
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: _get_ssl_verify returns SSLContext with CERT_REQUIRED by default.
# @PRE LLM_SSL_VERIFY env var is not set.
# @PRE Centralized SSL is active (no LLM_SSL_VERIFY dependency).
# @POST Returns SSLContext with verify_mode=CERT_REQUIRED.
def test_get_ssl_verify_default_context():
"""Verify default returns SSLContext with CERT_REQUIRED."""
import ssl
result = LLMClient._get_ssl_verify()
assert isinstance(result, ssl.SSLContext), "Expected SSLContext, not bool"
assert result.verify_mode == ssl.CERT_REQUIRED
# endregion test_get_ssl_verify_default_context
# region test_get_ssl_verify_false_values [TYPE Function]
# region test_get_ssl_verify_ignores_llm_env [TYPE Function]
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: _get_ssl_verify returns False for "false"/"0"/"no"/"off".
# @PRE LLM_SSL_VERIFY env var is set to each falsy value.
# @POST Returns False.
def test_get_ssl_verify_false_values(monkeypatch):
for val in ("false", "False", "0", "no", "off"):
monkeypatch.setenv("LLM_SSL_VERIFY", val)
assert LLMClient._get_ssl_verify() is False, f"Expected False for LLM_SSL_VERIFY={val}"
# endregion test_get_ssl_verify_false_values
# region test_get_ssl_verify_true_returns_context [TYPE Function]
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: _get_ssl_verify returns SSLContext for truthy env values.
# @PRE LLM_SSL_VERIFY env var is set to truthy values.
# @POST Returns SSLContext with CERT_REQUIRED.
def test_get_ssl_verify_true_returns_context(monkeypatch):
# @PURPOSE: _get_ssl_verify ignores LLM_SSL_VERIFY env (centralized SSL — no verify=False escape hatch).
# @PRE LLM_SSL_VERIFY env var is set to each falsy/truthy value.
# @POST Always returns SSLContext with CERT_REQUIRED — env var is ignored.
def test_get_ssl_verify_ignores_llm_env(monkeypatch):
import ssl
for val in ("TRUE", "1", "yes", "on", "true"):
for val in ("false", "False", "0", "no", "off", "true", "1", "yes"):
monkeypatch.setenv("LLM_SSL_VERIFY", val)
result = LLMClient._get_ssl_verify()
assert isinstance(result, ssl.SSLContext), f"Expected SSLContext for LLM_SSL_VERIFY={val}"
assert result.verify_mode == ssl.CERT_REQUIRED
# endregion test_get_ssl_verify_true_returns_path
assert isinstance(result, ssl.SSLContext), f"Expected SSLContext for LLM_SSL_VERIFY={val}, got {type(result)}"
assert result.verify_mode == ssl.CERT_REQUIRED, f"Expected CERT_REQUIRED for LLM_SSL_VERIFY={val}"
# endregion test_get_ssl_verify_ignores_llm_env
# region test_format_connection_error_no_cause [TYPE Function]
@@ -105,6 +103,8 @@ def test_format_connection_error_no_cause():
result = LLMClient._format_connection_error(exc)
assert "ValueError: test error" in result
assert "└─" not in result
# endregion test_format_connection_error_no_cause
@@ -121,6 +121,8 @@ def test_format_connection_error_with_cause():
assert "RuntimeError: API call failed" in result
assert "└─" in result
assert "ConnectionRefusedError: Connection refused" in result
# endregion test_format_connection_error_with_cause
@@ -139,17 +141,20 @@ def test_format_connection_error_deep_chain():
assert "RuntimeError: Connection error" in result
assert "ConnectionError: SSL: CERTIFICATE_VERIFY_FAILED" in result
assert "Exception: Name or service not known" in result
# endregion test_format_connection_error_deep_chain
# region test_litellm_client_verify_default [TYPE Function]
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: LLMClient passes verify=True to httpx.AsyncClient by default.
# @PRE LLM_SSL_VERIFY env var is not set.
# @PURPOSE: LLMClient passes verify context to httpx.AsyncClient by default.
# @PRE Centralized SSL is active (no LLM_SSL_VERIFY dependency).
# @POST httpcore pool SSL context has verify_mode=CERT_REQUIRED (2).
def test_litellm_client_verify_default():
"""Verify default SSL context with CERT_REQUIRED when LLM_SSL_VERIFY unset."""
"""Verify default SSL context with CERT_REQUIRED — centralized SSL in use."""
import ssl
client = LLMClient(
provider_type=LLMProviderType.LITELLM,
api_key="sk-test-verify-key",
@@ -157,17 +162,19 @@ def test_litellm_client_verify_default():
default_model="gpt-4o",
)
pool = client.client._client._transport._pool
assert pool._ssl_context.verify_mode == ssl.CERT_REQUIRED, \
"verify=True should set verify_mode to CERT_REQUIRED"
assert pool._ssl_context.verify_mode == ssl.CERT_REQUIRED, "verify=True should set verify_mode to CERT_REQUIRED"
# endregion test_litellm_client_verify_default
# region test_litellm_client_verify_disabled [TYPE Function]
# region test_litellm_client_verify_always_required [TYPE Function]
# @RELATION BINDS_TO -> TestClientHeaders
# @PURPOSE: LLMClient passes verify=False when LLM_SSL_VERIFY=false.
# @PURPOSE: LLMClient always uses CERT_REQUIRED — centralized SSL ignores LLM_SSL_VERIFY.
# @PRE LLM_SSL_VERIFY is set to "false".
# @POST httpcore pool SSL context has verify_mode=CERT_NONE (0).
def test_litellm_client_verify_disabled(monkeypatch):
# @POST httpcore pool SSL context has verify_mode=CERT_REQUIRED (2).
def test_litellm_client_verify_always_required(monkeypatch):
"""Verify LLM_SSL_VERIFY=false is ignored — verify mode always CERT_REQUIRED."""
monkeypatch.setenv("LLM_SSL_VERIFY", "false")
client = LLMClient(
provider_type=LLMProviderType.LITELLM,
@@ -176,8 +183,10 @@ def test_litellm_client_verify_disabled(monkeypatch):
default_model="gpt-4o",
)
import ssl
pool = client.client._client._transport._pool
assert pool._ssl_context.verify_mode == ssl.CERT_NONE, \
"verify=False should set verify_mode to CERT_NONE"
# endregion test_litellm_client_verify_disabled
assert pool._ssl_context.verify_mode == ssl.CERT_REQUIRED, "LLM_SSL_VERIFY=false must be ignored — verify_mode stays CERT_REQUIRED"
# endregion test_litellm_client_verify_always_required
# endregion TestClientHeaders

View File

@@ -911,9 +911,9 @@ class LLMClient:
# is sufficient. No additional headers like HTTP-Referer or X-API-Key are required.
ssl_verify = self._get_ssl_verify()
ssl_desc = str(ssl_verify)
if isinstance(ssl_verify, ssl.SSLContext):
ssl_desc = f"SSLContext(capath={ssl_verify.capath if hasattr(ssl_verify, 'capath') else 'default'})"
from ...core.ssl import describe_context
ssl_desc = describe_context(ssl_verify)
logger.reason("LLM client SSL verification configured", payload={"ssl_verify": ssl_desc})
http_client = httpx.AsyncClient(
@@ -932,8 +932,7 @@ class LLMClient:
# #region LLMClient._get_ssl_verify [TYPE Function]
# @PURPOSE Resolve SSL verification flag from environment.
# @POST Returns SSLContext with system CA dir when enabled,
# False when LLM_SSL_VERIFY env var is "false"/"0"/"no"/"off".
# @POST Returns SSLContext with system CA dir (never False — centralized SSL).
# @RATIONALE Используем capath=/etc/ssl/certs/ вместо cafile, потому что
# OpenSSL 3.x не использует intermediate CA сертификаты из cafile для
# построения цепочки (verify code 20). capath с хеш-симлинками работает

View File

@@ -1,5 +1,5 @@
# #region LLMAsyncHttpClient [C:4] [TYPE Module] [SEMANTICS translate, llm, http, openai, async, retry, rate-limit]
# @defgroup Translate Module group.
# @ingroup Translate
# @BRIEF Async HTTP client for OpenAI-compatible LLM API calls using httpx.AsyncClient
# with rate-limit handling and structured output fallback.
# @LAYER Infrastructure
@@ -34,7 +34,7 @@ DEFAULT_MAX_TOKENS: int = 8192
# #region _get_verify [C:1] [TYPE Function] [SEMANTICS translate, ssl, verify]
# @BRIEF Resolve SSL verification context from LLM_SSL_VERIFY env var.
# @BRIEF Resolve SSL verification context via centralized core.ssl helper.
# @RATIONALE Используем capath=/etc/ssl/certs/ через ssl.create_default_context,
# потому что OpenSSL 3.x не использует intermediate CA сертификаты из cafile
# для построения цепочки (verify code 20). capath с хеш-симлинками работает
@@ -44,8 +44,10 @@ DEFAULT_MAX_TOKENS: int = 8192
# из единого bundle-файла. Только capath с хеш-симлинками даёт code 0.
# @REJECTED Строковый путь "/etc/ssl/certs/" отвергнут — httpx 0.28.x депрекейтит
# строки в verify=, требует SSLContext.
# @POST Returns ssl.SSLContext with capath when enabled, False when disabled.
def _get_verify() -> ssl.SSLContext | bool:
# @REJECTED LLM_SSL_VERIFY=false escape hatch отвергнут — централизованный SSL
# всегда использует системное хранилище без возможности отключения.
# @POST Returns ssl.SSLContext with capath (never False).
def _get_verify() -> ssl.SSLContext:
from ...core.ssl import httpx_verify
return httpx_verify()
@@ -61,8 +63,6 @@ async def _get_http_client() -> httpx.AsyncClient:
global _http_client
if _http_client is None:
ssl_verify = _get_verify()
if ssl_verify is False:
log("LLMAsyncHttpClient", "EXPLORE", "TLS verification disabled via LLM_SSL_VERIFY=false", error="TLS verification disabled — traffic to LLM provider is unencrypted")
_http_client = httpx.AsyncClient(
verify=ssl_verify,
timeout=httpx.Timeout(180.0),

View File

@@ -485,11 +485,11 @@ def superset_container(request, superset_db_url, superset_secret_key, superset_a
key_pem = ca_chain["server_key_encrypted"]
key_pass = ca_chain["server_key_passphrase"]
key_b64 = base64.b64encode(key_pem.encode()).decode()
_write_certs = f"echo '{fullchain_b64}' | base64 -d > /tmp/server.crt && echo '{key_b64}' | base64 -d > /tmp/server.key"
_decrypt_key = f"export SSL_KEY_PASSPHRASE='{key_pass}' && openssl rsa -in /tmp/server.key -passin env:SSL_KEY_PASSPHRASE -out /tmp/server.key 2>/dev/null; if [ $? -ne 0 ]; then echo 'ERROR: Failed to decrypt private key with SSL_KEY_PASSPHRASE'; exit 1; fi"
_write_certs = f'echo "{fullchain_b64}" | base64 -d > /tmp/server.crt && echo "{key_b64}" | base64 -d > /tmp/server.key'
_decrypt_key = f'export SSL_KEY_PASSPHRASE="{key_pass}" && openssl rsa -in /tmp/server.key -passin env:SSL_KEY_PASSPHRASE -out /tmp/server.key 2>/dev/null; if [ $? -ne 0 ]; then echo "ERROR: Failed to decrypt private key with SSL_KEY_PASSPHRASE"; exit 1; fi'
else:
key_b64 = base64.b64encode(ca_chain["server_key"].encode()).decode()
_write_certs = f"echo '{fullchain_b64}' | base64 -d > /tmp/server.crt && echo '{key_b64}' | base64 -d > /tmp/server.key"
_write_certs = f'echo "{fullchain_b64}" | base64 -d > /tmp/server.crt && echo "{key_b64}" | base64 -d > /tmp/server.key'
_decrypt_key = ""
_protocol = "https"
@@ -537,6 +537,119 @@ def superset_container(request, superset_db_url, superset_secret_key, superset_a
if use_tls and _write_certs:
_decrypt_prefix = f"{_decrypt_key} && " if _decrypt_key else ""
web_cmd = f"sh -c 'python -m pip install psycopg2-binary -q && {_write_certs} && {_decrypt_prefix}{_bootstrap_config}{_export_config} && {_superset_run}'"
else:
web_cmd = f"sh -c 'python -m pip install psycopg2-binary -q && {_bootstrap_config}{_export_config} && {_superset_run}'"
superset.with_command(web_cmd)
superset.with_exposed_ports(8088)
superset.start()
# Store protocol on the container object for superset_url fixture
superset._ss_protocol = _protocol
# Wait for the /health endpoint to respond
host = superset.get_container_host_ip()
port = superset.get_exposed_port(8088)
health_url = f"{_protocol}://{host}:{port}/health"
deadline = time.time() + 90
last_error = None
while time.time() < deadline:
try:
# verify=False in TLS mode because the custom CA hasn't been installed yet
resp = requests.get(health_url, timeout=3, verify=False)
if resp.status_code == 200:
break
except requests.RequestException as e:
last_error = e
time.sleep(2)
else:
logs = superset.get_logs()
superset.stop()
raise TimeoutError(f"Superset did not become healthy within 90s. Last error: {last_error}\nContainer logs:\n{logs}")
yield superset
superset.stop()
# #endregion superset_container
# #region superset_url [C:1] [TYPE Fixture]
# @BRIEF Function-scoped — base URL of the running Superset container (auto-detects http/https).
@pytest.fixture
def superset_url(superset_container):
host = superset_container.get_container_host_ip()
port = superset_container.get_exposed_port(8088)
protocol = getattr(superset_container, "_ss_protocol", "http")
return f"{protocol}://{host}:{port}"
# #endregion superset_url
# #region superset_admin_headers [C:2] [TYPE Fixture]
# @BRIEF Function-scoped — authenticated session headers for Superset admin API calls.
@pytest.fixture
def superset_admin_headers(superset_url, superset_admin_password):
"""Log in as admin and return headers with CSRF token and session cookie."""
session = requests.Session()
# Step 1: Get CSRF token from login page
login_url = f"{superset_url}/login/"
resp = session.get(login_url, timeout=10)
resp.raise_for_status()
csrf_token = None
if "csrf_token" in resp.text:
import re
match = re.search(r'csrf_token"\s*:\s*"([^"]+)"', resp.text)
if match:
csrf_token = match.group(1)
# Step 2: Submit login form
resp = session.post(
login_url,
data={
"username": "admin",
"password": superset_admin_password,
"csrf_token": csrf_token or "",
},
headers={"Referer": login_url},
timeout=10,
)
resp.raise_for_status()
return session
# #endregion superset_admin_headers
# ── JWT-based Superset Fixtures ────────────────────────────────────
# ADR-0012: после установки psycopg2 + superset_config.py + Docker bridge IP,
# JWT-авторизация (/api/v1/security/login) работает в тестовом контейнере.
# #region superset_jwt_headers [C:2] [TYPE Fixture]
# @BRIEF Function-scoped — JWT Bearer token headers for Superset REST API.
# @POST Returns dict with Authorization: Bearer <access_token>.
@pytest.fixture
def superset_jwt_headers(superset_url, superset_admin_password):
"""Obtain JWT access token from the running Superset container."""
resp = requests.post(
f"{superset_url}/api/v1/security/login",
json={
"username": "admin",
"password": superset_admin_password,
"provider": "db",
},
timeout=10,
)
resp.raise_for_status()
data = resp.json()
access_token = data["access_token"]

View File

@@ -0,0 +1,312 @@
# #region TestBackendContainer [C:3] [TYPE Module] [SEMANTICS test,integration,backend,container,docker,entrypoint,certs]
# @BRIEF Integration tests for the superset-tools-backend Docker image:
# verifies entrypoint + certs.sh work correctly inside the container.
# @RELATION BINDS_TO -> [docker/backend.Dockerfile]
# @RELATION BINDS_TO -> [docker/certs.sh]
# @RELATION BINDS_TO -> [docker/backend.entrypoint.sh]
# @RELATION CALLS -> [TestSupersetTlsCustomCA]
# @RATIONALE In 0.5.0 release, backend entrypoint crashed with
# "/app/entrypoint.sh: line 264: /app/certs.sh: No such file or directory"
# because docker/backend.Dockerfile was missing COPY for certs.sh.
# Existing testcontainers tests only exercise apache/superset containers,
# not the superset-tools backend container. These tests close that gap.
# @TEST_CONTRACT BackendImageCertPaths ->
# /app/certs.sh exists and is readable
# source /app/certs.sh defines install_all_certs, install_ca_to_nss, normalize_cert
# /app/entrypoint.sh exists and is executable
# @TEST_CONTRACT BackendContainerSmoke ->
# docker compose up backend starts successfully
# /api/health/summary responds 200
# Container logs do NOT contain "certs.sh: No such file or directory"
# Container logs do NOT contain "Traceback" or unrecoverable startup error
# @REQUIRES Docker daemon running. --run-integration flag.
"""Integration tests for superset-tools-backend Docker image.
Two test suites:
1. **TestBackendImageCertsSh** — fast (no DB, no compose).
Builds the backend Docker image, runs a container with --entrypoint bash,
and verifies certs.sh is present, sourceable, and all certificate
installation functions are defined.
2. **TestBackendContainerSmoke** — full stack (needs PostgreSQL).
Starts PostgreSQL via testcontainers, deploys backend via docker compose,
waits for the health endpoint, and checks container logs for startup errors.
"""
import os
import re
import subprocess
import time
from pathlib import Path
import pytest
# ── Project paths ────────────────────────────────────────────────────────
PROJECT_ROOT = Path(__file__).resolve().parents[3]
BACKEND_IMAGE = "superset-tools-backend:test-integration"
# ── Fixtures ──────────────────────────────────────────────────────────────
@pytest.fixture(scope="module")
def _backend_image():
"""Build superset-tools-backend:test-integration once per module."""
# Only rebuild if the Dockerfile changed (simple mtime check)
dockerfile = PROJECT_ROOT / "docker" / "backend.Dockerfile"
image_marker = PROJECT_ROOT / "backend" / ".image-built"
if not image_marker.exists() or image_marker.stat().st_mtime < dockerfile.stat().st_mtime:
subprocess.run(
["docker", "build", "-f", str(dockerfile), "-t", BACKEND_IMAGE, str(PROJECT_ROOT)],
check=True,
capture_output=True,
text=True,
timeout=300,
)
image_marker.touch()
return BACKEND_IMAGE
# ══════════════════════════════════════════════════════════════════════════
# TEST 1: Fast — verify certs.sh presence and basic functionality
# ══════════════════════════════════════════════════════════════════════════
class TestBackendImageCertPaths:
"""Fast checks: certs.sh exists, is sourceable, defines all functions.
These tests run in ~10s without PostgreSQL or docker compose.
They catch the 'certs.sh: No such file or directory' failure mode.
"""
@pytest.fixture(autouse=True)
def _image(self, _backend_image):
pass
# ── helpers ─────────────────────────────────────────────────────
@staticmethod
def _run_bash(command: str) -> subprocess.CompletedProcess:
return subprocess.run(
["docker", "run", "--rm", "--entrypoint", "bash", BACKEND_IMAGE, "-c", command],
capture_output=True,
text=True,
timeout=30,
)
# ── tests ───────────────────────────────────────────────────────
# #region test_certs_sh_exists [C:1] [TYPE Function] [SEMANTICS test,integration,certs]
# @BRIEF /app/certs.sh must exist and be readable.
def test_certs_sh_exists(self):
r = self._run_bash("test -f /app/certs.sh && test -r /app/certs.sh")
assert r.returncode == 0, f"File exists: {r.returncode == 0}, stderr: {r.stderr[:200]}"
# #endregion test_certs_sh_exists
# #region test_entrypoint_sh_exists [C:1] [TYPE Function] [SEMANTICS test,integration,entrypoint]
# @BRIEF /app/entrypoint.sh must exist and be executable.
def test_entrypoint_sh_exists(self):
r = self._run_bash("test -f /app/entrypoint.sh && test -x /app/entrypoint.sh")
assert r.returncode == 0, f"File: /app/entrypoint.sh, stderr: {r.stderr[:200]}"
# #endregion test_entrypoint_sh_exists
# #region test_certs_sh_source_defines_functions [C:2] [TYPE Function] [SEMANTICS test,integration,certs]
# @BRIEF sourcing /app/certs.sh defines install_all_certs, install_ca_to_nss, normalize_cert.
def test_certs_sh_source_defines_functions(self):
r = self._run_bash("source /app/certs.sh && declare -F install_all_certs >/dev/null 2>&1 && echo -n 'install_all_certs '; declare -F install_ca_to_nss >/dev/null 2>&1 && echo -n 'install_ca_to_nss '; declare -F normalize_cert >/dev/null 2>&1 && echo -n 'normalize_cert'")
assert "install_all_certs" in r.stdout, f"install_all_certs not defined:\n{r.stderr}"
assert "install_ca_to_nss" in r.stdout, f"install_ca_to_nss not defined:\n{r.stderr}"
assert "normalize_cert" in r.stdout, f"normalize_cert not defined:\n{r.stderr}"
# #endregion test_certs_sh_source_defines_functions
# #region test_entrypoint_sources_certs_sh_ok [C:2] [TYPE Function] [SEMANTICS test,integration,entrypoint]
# @BRIEF Running the real entrypoint script (without DB) must not crash on source certs.sh.
# We pass a short DB wait timeout so it fails fast instead of retrying 30 times.
# The certs.sh sourcing happens BEFORE the DB wait loop.
def test_entrypoint_sources_certs_sh_ok(self):
r = subprocess.run(
[
"docker",
"run",
"--rm",
"-e",
"POSTGRES_HOST=does-not-exist",
"-e",
"POSTGRES_USER=test",
"-e",
"POSTGRES_PASSWORD=test",
"-e",
"POSTGRES_DB=test",
# Reduce wait time — if certs.sh source succeeds, the error
# will be about DB connection, not certs.sh
"--entrypoint",
"bash",
BACKEND_IMAGE,
"-c",
"timeout 5 /app/entrypoint.sh 2>&1 || true",
],
capture_output=True,
text=True,
timeout=15,
)
# The certs.sh source comes BEFORE DB wait.
# If certs.sh is missing, the output MUST contain "No such file or directory".
# If certs.sh loads OK, the output should show DB connection error OR cert install.
assert re.search(r"No such file or directory.*certs\.sh|certs\.sh.*No such file or directory", r.stdout + r.stderr) is None, f"certs.sh not found! Entrypoint tried to source missing file:\nSTDOUT:\n{r.stdout[:2000]}\nSTDERR:\n{r.stderr[:2000]}"
# #endregion test_entrypoint_sources_certs_sh_ok
# ══════════════════════════════════════════════════════════════════════════
# TEST 2: Full stack — docker compose up backend + health check
# ══════════════════════════════════════════════════════════════════════════
@pytest.mark.skipif(
not os.environ.get("E2E_BACKEND_CONTAINER"),
reason="Skipped by default — set E2E_BACKEND_CONTAINER=1 to run full-stack test",
)
@pytest.mark.usefixtures("_backend_image")
class TestBackendContainerSmoke:
"""Full-stack smoke test: deploy backend container via docker compose
with PostgreSQL, wait for health endpoint, verify container logs.
Requires: set E2E_BACKEND_CONTAINER=1 (takes 30-60s, needs PostgreSQL).
"""
# #region test_backend_health [C:3] [TYPE Function] [SEMANTICS test,integration,smoke]
# @BRIEF docker run (host network) backend with testcontainers PostgreSQL,
# wait for health, check container logs for startup errors.
# @PRE Docker daemon running. Port 8000 must be free on the host.
# @POST Backend health endpoint responds 200. Container logs show no certs.sh errors.
# @SIDE_EFFECT Starts/stops PostgreSQL and backend Docker containers.
def test_backend_health(self):
import time as time_module
import urllib.request
import urllib.error
# ── Use host network mode so backend can reach PG on localhost ──
from testcontainers.postgres import PostgresContainer
pg = PostgresContainer(
"postgres:16-alpine",
dbname="ss_tools",
username="postgres",
password="postgres",
)
pg.start()
pg_port = pg.get_exposed_port(5432)
# ── Start backend container (host network) ──
container_name = f"ss-tools-integration-backend-{os.urandom(4).hex()}"
backend_port = "8000"
db_url = f"postgresql://postgres:postgres@127.0.0.1:{pg_port}/ss_tools"
run_cmd = [
"docker",
"run",
"-d",
"--name",
container_name,
"--network",
"host",
"-e",
f"DATABASE_URL={db_url}",
"-e",
"INITIAL_ADMIN_CREATE=true",
"-e",
"INITIAL_ADMIN_USERNAME=admin",
"-e",
"INITIAL_ADMIN_PASSWORD=admin123",
"-e",
"INITIAL_ADMIN_EMAIL=admin@example.com",
"-e",
"ENABLE_BELIEF_STATE_LOGGING=true",
"-e",
"TASK_LOG_LEVEL=ERROR",
"-e",
"CERTS_PATH=/opt/certs",
BACKEND_IMAGE,
]
r = subprocess.run(run_cmd, capture_output=True, text=True, timeout=30)
if r.returncode != 0:
pg.stop()
pytest.fail(f"docker run failed:\nSTDERR: {r.stderr[:2000]}")
container_id = r.stdout.strip()
container_removed = False
def _cleanup(rm_container=True):
nonlocal container_removed
if rm_container and not container_removed:
subprocess.run(["docker", "rm", "-f", container_id], capture_output=True, timeout=10)
container_removed = True
try:
pg.stop()
except Exception:
pass
try:
# ── Wait for health ──
health_url = f"http://127.0.0.1:{backend_port}/api/health/summary"
deadline = time_module.time() + 60
last_error = ""
while time_module.time() < deadline:
try:
hr = subprocess.run(
["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", "--connect-timeout", "2", "--max-time", "4", health_url],
capture_output=True,
text=True,
timeout=10,
)
http_code = hr.stdout.strip()
if http_code in ("200", "401", "403"):
break
last_error = f"HTTP {http_code}"
except subprocess.TimeoutExpired:
last_error = "curl timeout"
except Exception as e:
last_error = str(e)
time_module.sleep(2)
else:
# Timeout — dump container logs
logs_r = subprocess.run(
["docker", "logs", container_id, "--tail", "60"],
capture_output=True,
text=True,
timeout=10,
)
full_log = logs_r.stdout[:5000] + logs_r.stderr[:2000]
_cleanup(rm_container=True)
pytest.fail(f"Backend health check failed after 60s.\nLast error: {last_error}\nContainer logs:\n{full_log}")
# ── Check container logs for startup errors ──
logs_r = subprocess.run(
["docker", "logs", container_id],
capture_output=True,
text=True,
timeout=10,
)
combined_logs = logs_r.stdout + logs_r.stderr
if re.search(r"No such file or directory.*certs\.sh|certs\.sh.*No such file", combined_logs):
_cleanup(rm_container=True)
pytest.fail(f"certs.sh not found! Entrypoint crashed:\n{combined_logs[:3000]}")
if re.search(r"Traceback|Error.*certs\.sh", combined_logs):
_cleanup(rm_container=True)
pytest.fail(f"Entrypoint crash detected:\n{combined_logs[:3000]}")
finally:
_cleanup(rm_container=True)
# #endregion test_backend_health
# #endregion TestBackendContainer

View File

@@ -51,7 +51,8 @@ COPY backend/ /app/backend/
# Frontend build (from stage 1)
COPY --from=frontend-builder /app/frontend/build /app/frontend/build
# Entrypoint
# Entrypoint + certs.sh
COPY docker/certs.sh /app/certs.sh
COPY docker/backend.entrypoint.sh /app/entrypoint.sh
RUN chmod +x /app/entrypoint.sh

View File

@@ -31,7 +31,8 @@ RUN python -m playwright install --with-deps chromium
# Исходный код
COPY backend/ /app/backend/
# Копируем entrypoint — установит корп. сертификаты, Playwright и сделает bootstrap admin
# Копируем shared certs.sh и entrypoint — установит корп. сертификаты, Playwright и сделает bootstrap admin
COPY docker/certs.sh /app/certs.sh
COPY docker/backend.entrypoint.sh /app/entrypoint.sh
RUN chmod +x /app/entrypoint.sh

View File

@@ -16,254 +16,18 @@ set -euo pipefail
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [backend/src/scripts/create_admin.py]
# @RELATION DEPENDS_ON -> [backend/src/scripts/init_auth_db.py]
# @INVARIANT Если CERTS_PATH пуст или нет .crt файлов — install_certificates не падает.
# @INVARIANT Если CERTS_PATH пуст или нет сертификатов — certs.sh:install_all_certs не падает.
# @INVARIANT Существующий admin аккаунт НЕ перезаписывается при перезапуске контейнера
# (create_admin.py идемпотентен).
# @INVARIANT Повторный запуск install_llm_ca_certs не дублирует сертификаты в bundle.
# @INVARIANT Повторный запуск install_ca_to_nss не дублирует сертификаты в NSS DB.
# @INVARIANT Повторный запуск certs.sh:install_all_certs не дублирует сертификаты в bundle.
# @INVARIANT Повторный запуск certs.sh:install_ca_to_nss не дублирует сертификаты в NSS DB.
# @INVARIANT wait_for_db exit(1) если БД недоступна после DB_WAIT_ATTEMPTS попыток —
# контейнер не продолжит запуск с недоступной БД.
# @INVARIANT Alembic миграции выполняются ТОЛЬКО в entrypoint, НЕ в app.py lifespan.
# #endregion docker.backend.entrypoint
# ======================================================================
# install_certificates — корпоративные CA-сертификаты
# ======================================================================
# #region docker.backend.entrypoint.install_certificates [C:3] [TYPE Function]
# @BRIEF Устанавливает корпоративные .crt сертификаты из CERTS_PATH в системное хранилище Debian.
# @PRE CERTS_PATH указывает на директорию (может быть пуста или не существовать).
# В системе установлен пакет ca-certificates.
# @POST .crt файлы скопированы в /usr/local/share/ca-certificates/custom/ и
# добавлены в системное хранилище через update-ca-certificates.
# @SIDE_EFFECT Запускает update-ca-certificates — модифицирует /etc/ssl/certs/.
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [ca-certificates (system package)]
# @EXAMPLE:
# CERTS_PATH=/opt/certs docker run ... # монтируем ./certs в /opt/certs
# В ./certs кладём: my-corporate-ca.crt, another-ca.crt
# entrypoint установит оба в доверенные.
install_certificates() {
local certs_dir="${CERTS_PATH:-/opt/certs}"
if [ ! -d "$certs_dir" ]; then
echo "[entrypoint] CERTS_PATH=${certs_dir} не существует — пропускаем установку сертификатов"
return 0
fi
local cert_count
cert_count="$(find "$certs_dir" -maxdepth 1 \( -name '*.crt' -o -name '*.pem' \) 2>/dev/null | wc -l)"
if [ "$cert_count" -eq 0 ]; then
echo "[entrypoint] Нет .crt/.pem файлов в ${certs_dir} — пропускаем установку сертификатов"
return 0
fi
local target="/usr/local/share/ca-certificates/custom"
mkdir -p "$target"
echo "[entrypoint] Устанавливаем корпоративные сертификаты из ${certs_dir}..."
local copied=0
# nullglob: если файлов нет, цикл не выполняется
shopt -s nullglob
for cert in "$certs_dir"/*.crt "$certs_dir"/*.pem; do
cp -v "$cert" "$target/" 2>&1 | sed 's/^/[entrypoint] /'
copied=$((copied + 1))
done
shopt -u nullglob
if [ "$copied" -eq 0 ]; then
echo "[entrypoint] Нет файлов для копирования — пропускаем"
return 0
fi
echo "[entrypoint] Обновляем системное хранилище CA-сертификатов..."
update-ca-certificates --fresh 2>&1 | sed 's/^/[entrypoint] /'
echo "[entrypoint] ✅ Установлено ${copied} корпоративных сертификатов"
}
# #endregion docker.backend.entrypoint.install_certificates
# ======================================================================
# install_llm_ca_certs — скачка и установка CA-сертификатов по URL
# ======================================================================
# #region docker.backend.entrypoint.install_llm_ca_certs [C:4] [TYPE Function]
# @BRIEF Скачивает PEM/DER-сертификаты из LLM_CA_CERT_URLS, конвертирует DER→PEM,
# устанавливает в системное хранилище, создаёт хеш-симлинки и при необходимости
# добавляет в ca-certificates.crt напрямую (fallback для OpenSSL 3-совместимости).
# @RATIONALE DER→PEM конвертация необходима, т.к. update-ca-certificates ожидает
# PEM-формат (-----BEGIN CERTIFICATE-----). Корпоративные PKI часто отдают
# сертификаты в DER (бинарном) формате через HTTP.
# @RATIONALE Хеш-симлинки создаются с поддержкой коллизий (.0, .1, .2...), т.к.
# update-ca-certificates в Debian Bookworm иногда пропускает создание ссылок
# для сертификатов из поддиректорий. Fingerprint (SHA256) используется для
# детекции дубликатов вместо имени файла — защита от ложных срабатываний grep.
# @RATIONALE Fallback cat в ca-certificates.crt гарантирует что OpenSSL 3.x
# увидит сертификаты даже если update-ca-certificates их пропустит.
# Перед добавлением проверяется fingerprint — при повторном запуске
# сертификат не дублируется.
# @REJECTED Вариант "только контейнерный volume с .crt файлами" отвергнут —
# требует ручного копирования сертификатов на хост перед запуском.
# URL-based подход позволяет централизованно управлять сертификатами через PKI.
# @REJECTED Использование grep по имени файла/серийному номеру для детекции
# дубликатов отвергнуто — имя файла может совпадать с содержимым PEM,
# а серийный номер не уникален при ротации CA. Fingerprint (SHA256)
# является криптографически уникальным идентификатором.
# @PRE LLM_CA_CERT_URLS — строка с URL через пробел; curl и openssl установлены.
# @POST Сертификаты доступны в /etc/ssl/certs/ca-certificates.crt.
# Хеш-симлинки созданы для каждого сертификата (с поддержкой коллизий).
# @SIDE_EFFECT Скачивает файлы из внешних URL. Модифицирует /etc/ssl/certs/.
# @LAYER Infrastructure
# @EXAMPLE:
# LLM_CA_CERT_URLS="http://pki.company.com/root.crt http://pki.company.com/intermediate.crt"
install_llm_ca_certs() {
local urls="${LLM_CA_CERT_URLS:-}"
if [ -z "$urls" ]; then
echo "[entrypoint] LLM_CA_CERT_URLS не задан — пропускаем скачку сертификатов"
return 0
fi
local work_dir="/tmp/llm-ca-certs"
local target_dir="/usr/local/share/ca-certificates/llm"
mkdir -p "$work_dir" "$target_dir"
echo "[entrypoint] === Скачивание и установка LLM CA-сертификатов ==="
local index=0
for url in $urls; do
url="$(echo "$url" | xargs)" # trim
[ -z "$url" ] && continue
index=$((index + 1))
# Определяем имя файла из URL, или генерируем
local filename
filename="$(basename "$url" | sed 's/[?#].*//')"
if [ -z "$filename" ] || [ "$filename" = "." ] || [ "$filename" = ".." ]; then
filename="llm-ca-${index}.crt"
fi
local raw_file="${work_dir}/${filename}"
local pem_file="${target_dir}/$(echo "$filename" | sed 's/\.[^.]*$//').crt"
echo "[entrypoint] [${index}] Скачивание: ${url}"
# Скачиваем с retry 3 раза
local attempt=0
local success=0
while [ $attempt -lt 3 ]; do
attempt=$((attempt + 1))
# Сначала пробуем с verify, если не получилось — с --insecure
# (chicken-and-egg: PKI сервер может использовать тот же CA, который мы скачиваем)
if curl -sS --connect-timeout 10 --max-time 30 -o "$raw_file" "$url" 2>/dev/null; then
success=1
break
fi
# TLS verify failed? Пробуем без проверки (корп. PKI bootstrap)
if [ $attempt -eq 1 ]; then
echo "[entrypoint] ↻ TLS verify failed, retrying with --insecure..."
if curl -sS -k --connect-timeout 10 --max-time 30 -o "$raw_file" "$url" 2>/dev/null; then
echo "[entrypoint] ✓ Скачано с --insecure (проверка fingerprint в конце)"
success=1
break
fi
fi
echo "[entrypoint] ⚠ Попытка ${attempt}/3 не удалась, повтор через 2с..."
sleep 2
done
if [ "$success" -eq 0 ]; then
echo "[entrypoint] ❌ Не удалось скачать: ${url} — пропускаем"
continue
fi
# Определяем формат: PEM или DER
if openssl x509 -in "$raw_file" -inform PEM -noout 2>/dev/null; then
echo "[entrypoint] ✓ Формат: PEM"
cp "$raw_file" "$pem_file"
elif openssl x509 -in "$raw_file" -inform DER -noout 2>/dev/null; then
echo "[entrypoint] ↻ Формат: DER — конвертируем в PEM"
if ! openssl x509 -in "$raw_file" -inform DER -out "$pem_file" -outform PEM 2>&1; then
echo "[entrypoint] ❌ Ошибка конвертации DER→PEM — пропускаем"
rm -f "$raw_file" "${pem_file}"
continue
fi
else
echo "[entrypoint] ❌ Неизвестный формат (не PEM и не DER) — пропускаем"
# Выводим первые 20 байт для диагностики
local first_bytes
first_bytes="$(head -c 20 "$raw_file" 2>/dev/null | cat -v)"
echo "[entrypoint] Первые 20 байт: ${first_bytes}"
rm -f "$raw_file"
continue
fi
chmod 644 "$pem_file"
echo "[entrypoint] ✅ Установлен: $(basename "$pem_file")"
rm -f "$raw_file"
done
if [ -z "$(ls -A "$target_dir" 2>/dev/null)" ]; then
echo "[entrypoint] Нет скачанных сертификатов — пропускаем"
rm -rf "$work_dir"
return 0
fi
# Обновляем системное хранилище
echo "[entrypoint] --- Обновление системного хранилища CA-сертификатов ---"
update-ca-certificates --fresh 2>&1 | sed 's/^/[entrypoint] /'
# Проверяем, попали ли сертификаты в бандл, и создаём хеш-симлинки
echo "[entrypoint] --- Валидация установки сертификатов ---"
for cert_file in "$target_dir"/*.crt; do
[ -f "$cert_file" ] || continue
local cert_name
cert_name="$(basename "$cert_file" .crt)"
local cert_subject
cert_subject="$(openssl x509 -in "$cert_file" -noout -subject 2>/dev/null | head -1 || echo "unknown")"
local cert_fingerprint
cert_fingerprint="$(openssl x509 -in "$cert_file" -noout -fingerprint -sha256 2>/dev/null | sed 's/.*=//' || echo "")"
# Проверяем есть ли в ca-certificates.crt по fingerprint (криптографически уникален)
if [ -n "$cert_fingerprint" ] && \
grep -qF "$cert_fingerprint" /etc/ssl/certs/ca-certificates.crt 2>/dev/null; then
echo "[entrypoint] ✅ ${cert_name} — в ca-certificates.crt (${cert_subject})"
else
echo "[entrypoint] ⚠ ${cert_name}НЕ в ca-certificates.crt. Добавляем напрямую..."
cat "$cert_file" >> /etc/ssl/certs/ca-certificates.crt
echo "[entrypoint] Добавлен напрямую в ca-certificates.crt"
fi
# Создаём хеш-симлинку с поддержкой коллизий (.0, .1, .2...)
local hash_val
hash_val="$(openssl x509 -in "$cert_file" -noout -hash 2>/dev/null || true)"
if [ -n "$hash_val" ]; then
local suffix=0
local link_path="/etc/ssl/certs/${hash_val}.${suffix}"
# Ищем первый свободный suffix или symlink уже указывающий на наш файл
while [ -L "$link_path" ]; do
if [ "$(readlink "$link_path")" = "$cert_file" ]; then
break 2 # уже есть, ничего не делаем
fi
suffix=$((suffix + 1))
link_path="/etc/ssl/certs/${hash_val}.${suffix}"
done
if [ ! -L "$link_path" ]; then
ln -sf "$cert_file" "$link_path"
echo "[entrypoint] 🔗 Создана симлинка: ${hash_val}.${suffix}${cert_name}"
fi
fi
done
# Проверяем итоговое количество
local total_in_bundle
total_in_bundle="$(awk -v cmd='openssl x509 -noout -subject' '/BEGIN/{close(cmd)};{print | cmd}' \
< /etc/ssl/certs/ca-certificates.crt 2>/dev/null | wc -l || echo "0")"
echo "[entrypoint] ✅ Всего сертификатов в bundle: ${total_in_bundle}"
rm -rf "$work_dir"
echo "[entrypoint] === Установка LLM CA-сертификатов завершена ==="
}
# #endregion docker.backend.entrypoint.install_llm_ca_certs
# NOTE: install_certificates and install_llm_ca_certs removed — replaced by docker/certs.sh:install_all_certs.
# Centralized CERTS_PATH is the only certificate source. LLM_CA_CERT_URLS env var is removed (refs ADR-0009).
# ======================================================================
# bootstrap_admin — создание admin пользователя
@@ -323,12 +87,13 @@ bootstrap_admin() {
# для CA trust, отдельно от системного OpenSSL bundle. update-ca-certificates
# добавляет сертификаты в /etc/ssl/certs/, но Chromium их не видит.
# certutil -A -t "C,," импортирует PEM как доверенный CA для TLS.
# @RATIONALE NSS импорт выполняется после install_llm_ca_certs (скачивает .pem)
# и update-ca-certificates (устанавливает в систему). Этот порядок
# @RATIONALE NSS импорт выполняется после certs.sh:install_all_certs
# (устанавливает .crt в /usr/local/share/ca-certificates/custom/)
# и update-ca-certificates (добавляет в систему). Этот порядок
# гарантирует, что PEM-файлы уже есть на диске до создания NSS DB.
# @RATIONALE Для дедупликации используется SHA256 fingerprint сертификата,
# а не nickname — nickname может совпадать для файлов из разных директорий.
# Nickname формируется как "llm-имя" или "custom-имя" для избежания коллизий.
# Nickname формируется как "custom-имя" (все сертификаты из CERTS_PATH).
# @REJECTED Использование Playwright --ignore-certificate-errors отвергнуто —
# отключает ВСЕ SSL проверки глобально, а не только для корп. CA.
# @REJECTED Nickname-only дедупликация отвергнута — при разных файлах
@@ -347,9 +112,8 @@ install_ca_to_nss() {
local nss_db_dir="${HOME:-/root}/.pki/nssdb"
local nss_db="sql:${nss_db_dir}"
# CA certs могут быть в llm/ или custom/
# All CA certs are installed via docker/certs.sh into custom/ dir
local target_dirs=(
"/usr/local/share/ca-certificates/llm"
"/usr/local/share/ca-certificates/custom"
)

View File

@@ -1,13 +1,17 @@
# [DEF:ADR-0009:ADR]
# @STATUS ACTIVE
# @PURPOSE Define the strategy for corporate SSL certificate management across all LLM HTTP clients in superset-tools (httpx, requests, openai), covering system CA store, NSS database for Chromium/Playwright, and the LLM_SSL_VERIFY escape hatch.
# @PURPOSE Define the strategy for corporate SSL certificate management across all HTTP clients in superset-tools (httpx, requests, openai), covering system CA store, NSS database for Chromium/Playwright. Centralized CERTS_PATH contract replaces per-client SSL toggles. LLM_SSL_VERIFY and LLM_CA_CERT_URLS removed in 0.2.x.
# @RELATION DEPENDS_ON -> [ADR-0001:ADR]
# @RELATION DEPENDS_ON -> [ADR-0004:ADR]
# @RELATION CALLS -> [docker/backend.entrypoint.sh]
# @RELATION CALLS -> [docker/certs.sh]
# @RELATION CALLS -> [docker/frontend.entrypoint.sh]
# @RELATION CALLS -> [docker/agent.entrypoint.sh]
# @RELATION CALLS -> [backend/src/core/ssl.py]
# @RELATION CALLS -> [backend/src/plugins/llm_analysis/service.py]
# @RELATION CALLS -> [backend/src/plugins/translate/_llm_http.py]
# @RELATION CALLS -> [backend/src/plugins/translate/preview_llm_client.py]
# @RELATION CALLS -> [backend/src/plugins/translate/_llm_async_http.py]
# @RELATION CALLS -> [scripts/check_llm_certs.py]
# @RELATION CALLS -> [scripts/diag_container.py]
# @RATIONALE superset-tools operates in corporate environments with internal Certificate Authorities.
# Three separate HTTP stacks are used: httpx (AsyncOpenAI in llm_analysis plugin),
# requests (sync translate plugin), and Playwright Chromium (dashboard screenshots).
@@ -66,12 +70,11 @@ Diagnostic matrix (verified on production server 2026-05-28):
### Layer 1: System CA Store (OpenSSL)
- Entrypoint `install_certificates()` copies `.crt` files from `CERTS_PATH` volume mount
- Entrypoint `install_llm_ca_certs()` downloads PEM/DER certificates from `LLM_CA_CERT_URLS`
- DER is auto-converted to PEM (`openssl x509 -inform DER -outform PEM`)
- Certificates are saved with **`.crt` extension** (`.pem` is silently ignored by `update-ca-certificates`)
- Shared `docker/certs.sh:install_all_certs()` copies `.crt`/`.pem`/`.cer`/`.der` files from `CERTS_PATH` volume mount
- DER is auto-converted to PEM via `normalize_cert()` (`openssl x509 -inform DER -outform PEM`)
- Server/private files (`server.crt`, `server.key`, `*.key`, `*.p12`, `*.pfx`) are excluded from CA import
- Certificates are saved with **`.crt` extension** to `/usr/local/share/ca-certificates/custom/`
- `update-ca-certificates --fresh` adds them to `/etc/ssl/certs/ca-certificates.crt`
- **Fallback**: if `update-ca-certificates` misses certs, they are appended to `ca-certificates.crt` with SHA256 fingerprint dedup
- Hash symlinks created with collision support: `.0`, `.1`, `.2` suffixes
### Layer 2: Python HTTP Clients
@@ -94,27 +97,32 @@ intermediate CA in OpenSSL 3.x. `verify=<capath_dir>` works correctly.
- Trust attributes: `"C,,"` (trusted CA for TLS server certs)
- Fixes `ERR_CERT_AUTHORITY_INVALID` in Playwright dashboard screenshots
### Layer 4: LLM_SSL_VERIFY Escape Hatch
### Layer 4: Centralized CERTS_PATH Trust Contract (0.2.x)
- Env var `LLM_SSL_VERIFY=false` disables SSL verification entirely
- Accepted values for false: `false`, `0`, `no`, `off` (case-insensitive)
- Default: enabled (returns capath-based SSLContext or path)
- Used by all three HTTP client implementations
- **WARNING**: `verify=False` is for DIAGNOSTIC USE ONLY. Never leave in production.
- **REMOVED**: `LLM_SSL_VERIFY` — TLS verification is always enabled; no env escape hatch exists.
- **REMOVED**: `LLM_CA_CERT_URLS` — certificate download by URL is no longer supported.
- **Replaced by**: `CERTS_PATH` volume mount (`./certs:/opt/certs:ro`) as the single certificate input.
- All trust anchors come from mounted `CERTS_PATH` directory (`.crt`, `.pem`, `.cer`, `.der` files).
- Shared `docker/certs.sh` handles installation across all containers (backend, frontend, agent).
- Centralized Python helper `backend/src/core/ssl.py` provides `system_ssl_context()` / `httpx_verify()`.
- No code path disables TLS verification — the `verify=False` escape hatch is permanently removed.
- **Migration**: Place corporate CA files in `./certs/`, remove `LLM_SSL_VERIFY`/`LLM_CA_CERT_URLS` from `.env`.
## Key Files
| File | Role |
|------|------|
| `docker/backend.Dockerfile` | Installs `libnss3-tools` (certutil), Playwright Chromium |
| `docker/backend.entrypoint.sh` | `install_certificates`, `install_llm_ca_certs`, `install_ca_to_nss` |
| `plugins/llm_analysis/service.py` | `LLMClient._get_ssl_verify()` `ssl.create_default_context(capath=...)` |
| `plugins/translate/_llm_http.py` | `_get_verify()``"/etc/ssl/certs/"` |
| `plugins/translate/preview_llm_client.py` | `_get_verify()``"/etc/ssl/certs/"` |
| `scripts/check_llm_certs.py` | Full diagnostic: openssl, httpx, requests, NSS |
| `docker-compose.yml` | Passes `LLM_SSL_VERIFY`, `LLM_CA_CERT_URLS` |
| `docker-compose.enterprise-clean.yml` | Same as above |
| `.env.enterprise-clean` | Default values for both env vars |
| `docker/certs.sh` | Shared cert installer: `install_all_certs`, `install_ca_to_nss`, `normalize_cert` |
| `docker/backend.entrypoint.sh` | Sources `certs.sh`, calls `install_all_certs` + `install_ca_to_nss` |
| `docker/frontend.entrypoint.sh` | Installs CA certs from `CERTS_PATH` into Alpine store |
| `docker/agent.entrypoint.sh` | Sources `certs.sh`, installs certs for agent container |
| `backend/src/core/ssl.py` | Centralized SSL helper: `system_ssl_context()`, `httpx_verify()`, `describe_context()` |
| `plugins/llm_analysis/service.py` | `LLMClient._get_ssl_verify()` → delegates to `core.ssl.httpx_verify()` |
| `plugins/translate/_llm_async_http.py` | `_get_verify()` → delegates to `core.ssl.httpx_verify()` |
| `scripts/check_llm_certs.py` | Full diagnostic: openssl, httpx, requests, NSS, centralized SSL |
| `docker-compose.yml` | Mounts `${CERTS_PATH:-./certs}:/opt/certs:ro` (LLM env vars removed) |
| `docker-compose.enterprise-clean.yml` | Same — `CERTS_PATH` mount only |
## How to Test Certificate Installation
@@ -148,8 +156,8 @@ openssl x509 -in /usr/local/share/ca-certificates/custom/RUSAL_ROOT.crt -noout -
# 2. Verify full chain
openssl verify -CAfile /usr/local/share/ca-certificates/custom/RUSAL_ROOT.crt \
-untrusted /usr/local/share/ca-certificates/llm/UC_RUSAL_Policy_CA.crt \
/usr/local/share/ca-certificates/llm/UC_RUSAL_RGM_Issuing_CA.crt
-untrusted /usr/local/share/ca-certificates/custom/UC_RUSAL_Policy_CA.crt \
/usr/local/share/ca-certificates/custom/UC_RUSAL_RGM_Issuing_CA.crt
# → OK
# 3. Connect with capath
@@ -180,10 +188,11 @@ Corporate PKI servers (`pki.rusal.com`) often serve certificates in DER (binary)
`update-ca-certificates` requires PEM. Auto-detection and conversion
(`openssl x509 -inform DER -outform PEM`) is essential.
### Finding 5: Chicken-and-egg TLS bootstrap
### Finding 5: Chicken-and-egg TLS bootstrap (HISTORICAL — removed in 0.2.x)
Downloading a CA certificate from a PKI server that uses the same CA causes a TLS
verification loop. Entrypoint uses `curl --insecure` as fallback when initial `curl`
fails with TLS error.
verification loop. Formerly handled by `install_llm_ca_certs()` with `curl --insecure`.
**Removed in 0.2.x**: certificates must be placed in `CERTS_PATH` volume mount by operator,
eliminating the chicken-and-egg download problem.
### Finding 6: NSS DB format
Chromium uses NSS Shared DB in `~/.pki/nssdb/`. The `sql:` prefix is required for
@@ -213,11 +222,12 @@ Fix: pass `input=""` or `echo | openssl s_client ...`.
```bash
# On target server:
xz -dc superset-tools-backend.0.1.7.tar.xz | docker load
xz -dc superset-tools-frontend.0.1.7.tar.xz | docker load
xz -dc superset-tools-backend.0.2.x.tar.xz | docker load
xz -dc superset-tools-frontend.0.2.x.tar.xz | docker load
# Enable capath-based verification (remove LLM_SSL_VERIFY=false)
sed -i '/LLM_SSL_VERIFY=false/d' .env.enterprise-clean
# Place corporate CA files in ./certs (LLM_SSL_VERIFY and LLM_CA_CERT_URLS removed)
ls ./certs/
# RUSAL_ROOT.crt RGM_Issuing.crt etc.
docker compose -f docker-compose.enterprise-clean.yml \
--env-file .env.enterprise-clean down
@@ -239,5 +249,6 @@ docker compose -f docker-compose.enterprise-clean.yml \
| 0.1.5 | Initial SSL support: `LLM_SSL_VERIFY`, `_format_connection_error()`, CA download via URL, NSS import |
| 0.1.6 | QA fixes: fingerprint dedup, hash symlink collision, NSS collision, DER→PEM error handling, chicken-and-egg TLS, nullglob, translate plugin SSL |
| 0.1.7 | **capath instead of cafile**: OpenSSL 3.x cafile limitation discovered and fixed. `.crt` extension for downloaded certs. Diagnostic script rewritten. |
| 0.2.x | **Centralized SSL**: `LLM_SSL_VERIFY` and `LLM_CA_CERT_URLS` removed. Shared `docker/certs.sh` for all containers. `backend/src/core/ssl.py` central helper. `CERTS_PATH` volume mount is the only certificate input. Verify=False escape hatch permanently removed. |
# [/DEF:ADR-0009:ADR]

View File

@@ -34,10 +34,15 @@ results: dict = {}
# ── Helpers ──
def _run(cmd: list[str], timeout: int = 15, input_data: str = "") -> tuple[int, str, str]:
def _run(
cmd: list[str], timeout: int = 15, input_data: str = ""
) -> tuple[int, str, str]:
"""Run subprocess with optional stdin."""
try:
r = subprocess.run(cmd, input=input_data, capture_output=True, text=True, timeout=timeout)
r = subprocess.run(
cmd, input=input_data, capture_output=True, text=True, timeout=timeout
)
return r.returncode, r.stdout, r.stderr
except FileNotFoundError:
return -1, "", "command not found"
@@ -55,10 +60,18 @@ def _count_certs(path: str) -> int:
# ── Openssl tests ──
def test_openssl(label: str, extra_args: list[str]) -> None:
"""Test openssl s_client with empty stdin to prevent hang."""
cmd = ["openssl", "s_client", "-connect", f"{host}:443",
"-servername", host, "-verify_return_error"] + extra_args
cmd = [
"openssl",
"s_client",
"-connect",
f"{host}:443",
"-servername",
host,
"-verify_return_error",
] + extra_args
rc, out, err = _run(cmd, timeout=20, input_data="")
for line in out.split("\n"):
if "Verify return code" in line:
@@ -72,9 +85,11 @@ def test_openssl(label: str, extra_args: list[str]) -> None:
# ── Python library tests ──
def test_httpx(key: str, verify) -> None:
try:
import httpx
r = httpx.get(TARGET_URL, verify=verify, timeout=10)
results[key] = "OK"
print(f" {key}: ✅ status={r.status_code}")
@@ -86,6 +101,7 @@ def test_httpx(key: str, verify) -> None:
def test_requests(key: str, verify) -> None:
try:
import requests as req
r = req.get(TARGET_URL, verify=verify, timeout=10)
results[key] = "OK"
print(f" {key}: ✅ status={r.status_code}")
@@ -100,11 +116,12 @@ def test_requests(key: str, verify) -> None:
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--target", default=os.environ.get("TARGET_URL", "https://lite.ai.rusal.com"))
parser.add_argument(
"--target", default=os.environ.get("TARGET_URL", "https://lite.ai.rusal.com")
)
args = parser.parse_args()
TARGET_URL = args.target
host = TARGET_URL.replace("https://", "").split("/")[0]
LLM_SSL_VERIFY = os.environ.get("LLM_SSL_VERIFY", "true")
# ── 1. System ──
print("\n=== 1. System ===")
@@ -117,7 +134,9 @@ if __name__ == "__main__":
v = getattr(mod, "__version__", "installed")
p = getattr(mod, "__file__", "")
if lib == "certifi":
print(f" {lib}={v} path={mod.where()} certs={_count_certs(mod.where())}")
print(
f" {lib}={v} path={mod.where()} certs={_count_certs(mod.where())}"
)
else:
print(f" {lib}={v}")
except ImportError:
@@ -136,11 +155,24 @@ if __name__ == "__main__":
continue
for f in sorted(d.iterdir()):
if f.suffix in (".pem", ".crt") and f.is_file():
rc, out, _ = _run(["openssl", "x509", "-in", str(f), "-noout",
"-subject", "-issuer"], timeout=3)
rc, out, _ = _run(
["openssl", "x509", "-in", str(f), "-noout", "-subject", "-issuer"],
timeout=3,
)
if rc != 0:
rc, out, _ = _run(["openssl", "x509", "-in", str(f), "-inform",
"DER", "-noout", "-subject"], timeout=3)
rc, out, _ = _run(
[
"openssl",
"x509",
"-in",
str(f),
"-inform",
"DER",
"-noout",
"-subject",
],
timeout=3,
)
subj = out.replace("subject=", "").strip() if rc == 0 else "UNREADABLE"
print(f" {f.parent.name}/{f.name} subj={subj[:60]}")
@@ -154,11 +186,21 @@ if __name__ == "__main__":
print("\n=== 4. Python httpx (LLMClient) ===")
try:
import httpx
test_httpx("httpx_True", verify=True)
try:
import ssl
ctx_cafile = ssl.create_default_context(cafile=str(CA_BUNDLE)) if CA_BUNDLE.exists() else ssl.create_default_context()
ctx_capath = ssl.create_default_context(capath=str(CA_DIR)) if CA_DIR.is_dir() else ssl.create_default_context()
ctx_cafile = (
ssl.create_default_context(cafile=str(CA_BUNDLE))
if CA_BUNDLE.exists()
else ssl.create_default_context()
)
ctx_capath = (
ssl.create_default_context(capath=str(CA_DIR))
if CA_DIR.is_dir()
else ssl.create_default_context()
)
test_httpx("httpx_cafile", verify=ctx_cafile)
test_httpx("httpx_capath", verify=ctx_capath)
except Exception as e:
@@ -171,6 +213,7 @@ if __name__ == "__main__":
print("\n=== 5. Python requests (translate plugin) ===")
try:
import requests as req
test_requests("requests_True", verify=True)
test_requests("requests_cafile", verify=str(CA_BUNDLE))
test_requests("requests_capath", verify=str(CA_DIR))
@@ -182,7 +225,9 @@ if __name__ == "__main__":
print("\n=== 6. NSS Chromium ===")
rc, out, err = _run(["certutil", "-L", "-d", NSS_DB], timeout=5)
if rc == 0:
lines = [l for l in out.split("\n") if l.strip() and "Certificate Nickname" not in l]
lines = [
l for l in out.split("\n") if l.strip() and "Certificate Nickname" not in l
]
print(f" DB={NSS_DB} certs={len(lines)}")
for l in lines:
if any(x in l.lower() for x in ("rusal", "llm-", "custom-")):
@@ -190,17 +235,38 @@ if __name__ == "__main__":
else:
print(f" NSS: {err.strip() or 'unavailable'}")
# ── 7. LLM_SSL_VERIFY env ──
print("\n=== 7. LLM_SSL_VERIFY ===")
print(f" LLM_SSL_VERIFY={LLM_SSL_VERIFY} disabled={LLM_SSL_VERIFY.strip().lower() in ('false','0','no','off')}")
# ── 7. Centralized SSL (core.ssl) ──
print("\n=== 7. Centralized SSL (core.ssl) ===")
try:
from src.core.ssl import system_ssl_context, describe_context
ctx = system_ssl_context()
print(f" {describe_context(ctx)}")
print(
" LLM_SSL_VERIFY and LLM_CA_CERT_URLS removed — centralized CERTS_PATH only."
)
except ImportError:
print(" src.core.ssl: UNAVAILABLE (backend not installed)")
# ── 8. JSON Summary ──
print("\n=== 8. JSON Summary ===")
summary = {k: results[k] for k in [
"openssl_default", "openssl_cafile", "openssl_capath",
"httpx_True", "httpx_cafile", "httpx_capath", "httpx_False",
"requests_True", "requests_cafile", "requests_capath", "requests_False",
] if k in results}
summary = {
k: results[k]
for k in [
"openssl_default",
"openssl_cafile",
"openssl_capath",
"httpx_True",
"httpx_cafile",
"httpx_capath",
"httpx_False",
"requests_True",
"requests_cafile",
"requests_capath",
"requests_False",
]
if k in results
}
# Diagnosis
diag = []
@@ -212,7 +278,9 @@ if __name__ == "__main__":
v = str(summary.get(k, ""))
diag.append(f"{k}={'OK' if v.startswith('OK') else 'FAIL'}")
if str(results.get("httpx_capath", "")).startswith("OK") and not str(results.get("httpx_cafile", "")).startswith("OK"):
if str(results.get("httpx_capath", "")).startswith("OK") and not str(
results.get("httpx_cafile", "")
).startswith("OK"):
diag.append("VERDICT=capath_works_cafile_fails_use_capath")
elif str(summary.get("httpx_False", "")).startswith("OK"):
diag.append("VERDICT=all_ssl_fails_verify_disabled_works")