From 18b9f2e94e2dcd6c547d32f6576ca753016a111c Mon Sep 17 00:00:00 2001 From: busya Date: Tue, 7 Jul 2026 10:23:49 +0300 Subject: [PATCH] fix(certs): restore HTTP CA auto-download and test edge matrix Restore LLM_CA_CERT_URLS support for corporate PKI HTTP cert delivery. Downloaded certificates are installed into the llm/ CA category and share the same system trust + NSS import path as CERTS_PATH-mounted certs. Changes: - certs.sh: add download_llm_ca_certs with HTTP-only curl policy, PEM/DER handling, retries, llm/ storage, hash symlinks for llm+custom - backend/agent entrypoints: run download before install_all_certs - compose/env/docs: expose LLM_CA_CERT_URLS again and update ADR-0009 - integration tests: cover PEM/DER/CER, invalid certs, non-HTTP skips, query-string filenames, private/server skips, empty inputs, combined LLM_CA_CERT_URLS + CERTS_PATH channels, and NSS imports Verified: - 194 passed, 1 skipped integration tests - full backend container smoke passed with testcontainers PostgreSQL - docker bundle rebuilt for 0.5.0 --- README.md | 14 +- .../integration/test_backend_container.py | 287 +++++++++++++++++- docker-compose.enterprise-clean.yml | 2 + docker-compose.yml | 2 + docker/agent.entrypoint.sh | 1 + docker/backend.entrypoint.sh | 1 + docker/certs.sh | 235 +++++++++----- .../ADR-0009-ssl-certificate-management.md | 31 +- 8 files changed, 487 insertions(+), 86 deletions(-) diff --git a/README.md b/README.md index 74187273..b24554c6 100755 --- a/README.md +++ b/README.md @@ -303,6 +303,16 @@ Entrypoint автоматически: Положите `.crt`/`.pem` файлы в `./certs/` — entrypoint установит их в системное хранилище Alpine и NSS (Chromium/Playwright). Поддерживаются цепочки из нескольких CA (Root → Intermediate). +### LLM CA-сертификаты + +Если LLM-провайдер или Superset используют корпоративный PKI — укажите HTTP URL для скачивания CA-сертификатов: +```bash +LLM_CA_CERT_URLS="http://pki.company.com/root-ca.crt http://pki.company.com/intermediate-ca.crt" +``` +Сертификаты скачиваются на старте backend и agent контейнеров (через `certs.sh:download_llm_ca_certs()`), конвертируются из DER в PEM при необходимости, и устанавливаются в системное хранилище. + +Дополнительные сертификаты можно разместить в `CERTS_PATH=./certs` (volume mount). + ### Диагностика SSL ```bash @@ -313,8 +323,8 @@ scripts/check_llm_certs.py scripts/diag_container.py --target your-llm-provider.com:443 ``` -Сертификаты и централизованный SSL — подробнее в [ADR-0009](docs/adr/ADR-0009-ssl-certificate-management.md). -`LLM_SSL_VERIFY` и `LLM_CA_CERT_URLS` удалены в 0.2.x — используйте `CERTS_PATH=./certs`. +Подробнее — в [ADR-0009](docs/adr/ADR-0009-ssl-certificate-management.md). +`LLM_SSL_VERIFY` удалён в 0.2.x — TLS verify всегда включён. ## 🏢 Enterprise Clean Deployment diff --git a/backend/tests/integration/test_backend_container.py b/backend/tests/integration/test_backend_container.py index 0616e6f9..7357512f 100644 --- a/backend/tests/integration/test_backend_container.py +++ b/backend/tests/integration/test_backend_container.py @@ -53,11 +53,14 @@ BACKEND_IMAGE = "superset-tools-backend:test-integration" @pytest.fixture(scope="module") def _backend_image(): """Build superset-tools-backend:test-integration once per module.""" - # Only rebuild if the Dockerfile changed (simple mtime check) + # Rebuild if Dockerfile, entrypoint, or cert logic changed. dockerfile = PROJECT_ROOT / "docker" / "backend.Dockerfile" + certs_sh = PROJECT_ROOT / "docker" / "certs.sh" + entrypoint = PROJECT_ROOT / "docker" / "backend.entrypoint.sh" image_marker = PROJECT_ROOT / "backend" / ".image-built" - if not image_marker.exists() or image_marker.stat().st_mtime < dockerfile.stat().st_mtime: + watched_mtime = max(p.stat().st_mtime for p in (dockerfile, certs_sh, entrypoint)) + if not image_marker.exists() or image_marker.stat().st_mtime < watched_mtime: subprocess.run( ["docker", "build", "-f", str(dockerfile), "-t", BACKEND_IMAGE, str(PROJECT_ROOT)], check=True, @@ -309,4 +312,284 @@ class TestBackendContainerSmoke: # #endregion test_backend_health +# ══════════════════════════════════════════════════════════════════════════ +# TEST 3: cert download — download_llm_ca_certs via HTTP +# ══════════════════════════════════════════════════════════════════════════ + + +class TestBackendImageCertDownload: + """Verify download_llm_ca_certs downloads certs from HTTP URLs.""" + + @pytest.fixture(autouse=True) + def _image(self, _backend_image): + pass + + # #region _make_ca_cert [C:1] [TYPE Function] + @staticmethod + def _make_ca_cert(certs_dir: str, name: str = "test-download-ca") -> tuple[str, str]: + ca_key = os.path.join(certs_dir, f"{name}.key") + ca_crt = os.path.join(certs_dir, f"{name}.crt") + subprocess.run( + [ + "openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", + "-keyout", ca_key, "-out", ca_crt, + "-days", "1", + "-subj", f"/CN={name}-for-container", + ], + check=True, capture_output=True, text=True, timeout=15, + ) + return ca_key, ca_crt + # #endregion _make_ca_cert + + # #region _serve_dir [C:1] [TYPE Function] + @staticmethod + def _serve_dir(certs_dir: str): + import http.server + import socket + import socketserver + import threading + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + http_port = s.getsockname()[1] + + class _Handler(http.server.SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=certs_dir, **kwargs) + + httpd = socketserver.TCPServer(("127.0.0.1", http_port), _Handler) + server_thread = threading.Thread(target=httpd.serve_forever, daemon=True) + server_thread.start() + return httpd, http_port + # #endregion _serve_dir + + # #region _run_download [C:1] [TYPE Function] + @staticmethod + def _run_download(urls: str, command_suffix: str = "") -> subprocess.CompletedProcess: + command = "source /app/certs.sh && download_llm_ca_certs" + if command_suffix: + command = f"{command} && {command_suffix}" + return subprocess.run( + [ + "docker", "run", "--rm", + "--network", "host", + "-e", f"LLM_CA_CERT_URLS={urls}", + "--entrypoint", "bash", + BACKEND_IMAGE, + "-c", command, + ], + capture_output=True, text=True, timeout=45, + ) + # #endregion _run_download + + # #region test_download_ca_cert_over_http [C:3] [TYPE Function] [SEMANTICS test,integration,certs,download] + # @BRIEF download_llm_ca_certs() fetches a PEM cert from an HTTP server + # and installs it into /usr/local/share/ca-certificates/llm/. + def test_download_ca_cert_over_http(self): + import tempfile + + # ── Generate a test CA cert ── + with tempfile.TemporaryDirectory() as certs_dir: + self._make_ca_cert(certs_dir, "test-download-ca") + httpd, http_port = self._serve_dir(certs_dir) + + cert_url = f"http://127.0.0.1:{http_port}/test-download-ca.crt" + + try: + # ── Run download_llm_ca_certs in container (host network) ── + r = self._run_download(cert_url, "echo 'DOWNLOAD_OK' && ls -la /usr/local/share/ca-certificates/llm/") + + assert r.returncode == 0, ( + f"download_llm_ca_certs failed (exit {r.returncode}):\n" + f"STDOUT: {r.stdout[:2000]}\nSTDERR: {r.stderr[:2000]}" + ) + + # Verify download succeeded + assert "DOWNLOAD_OK" in r.stdout, ( + f"download_llm_ca_certs did not complete:\n{r.stdout[:2000]}" + ) + assert "Скачано и установлено: 1" in r.stdout, ( + f"Expected 1 downloaded cert:\n{r.stdout[:2000]}" + ) + assert "test-download-ca.crt" in r.stdout, ( + f"Cert file not found in llm/ dir:\n{r.stdout[:2000]}" + ) + + finally: + httpd.shutdown() + # #endregion test_download_ca_cert_over_http + + # #region test_download_der_cert_converts_to_pem [C:2] [TYPE Function] [SEMANTICS test,integration,certs,download] + # @BRIEF DER certificate downloaded over HTTP is converted to PEM .crt. + def test_download_der_cert_converts_to_pem(self): + import tempfile + + with tempfile.TemporaryDirectory() as certs_dir: + _, ca_crt = self._make_ca_cert(certs_dir, "test-download-der") + der_path = os.path.join(certs_dir, "test-download-der.der") + subprocess.run( + ["openssl", "x509", "-in", ca_crt, "-outform", "DER", "-out", der_path], + check=True, capture_output=True, text=True, timeout=15, + ) + httpd, http_port = self._serve_dir(certs_dir) + try: + url = f"http://127.0.0.1:{http_port}/test-download-der.der" + r = self._run_download(url, "grep -q 'BEGIN CERTIFICATE' /usr/local/share/ca-certificates/llm/test-download-der.crt && echo DER_OK") + assert r.returncode == 0, f"DER download failed:\n{r.stdout}\n{r.stderr}" + assert "DER_OK" in r.stdout + assert "Формат: DER" in r.stdout + finally: + httpd.shutdown() + # #endregion test_download_der_cert_converts_to_pem + + # #region test_download_skips_invalid_and_non_http [C:2] [TYPE Function] [SEMANTICS test,integration,certs,download] + # @BRIEF Invalid content and HTTPS URLs are skipped without failing valid downloads. + def test_download_skips_invalid_and_non_http(self): + import tempfile + + with tempfile.TemporaryDirectory() as certs_dir: + self._make_ca_cert(certs_dir, "valid-ca") + Path(certs_dir, "not-a-cert.crt").write_text("not a certificate", encoding="utf-8") + httpd, http_port = self._serve_dir(certs_dir) + try: + urls = " ".join([ + f"http://127.0.0.1:{http_port}/valid-ca.crt", + f"http://127.0.0.1:{http_port}/not-a-cert.crt", + "https://example.invalid/blocked.crt", + ]) + r = self._run_download(urls, "ls -la /usr/local/share/ca-certificates/llm/") + assert r.returncode == 0, f"Mixed download failed:\n{r.stdout}\n{r.stderr}" + assert "Скачано и установлено: 1" in r.stdout + assert "невалид" in r.stdout or "Неизвестный формат" in r.stdout + assert "пропускаем не-HTTP URL" in r.stdout + assert "valid-ca.crt" in r.stdout + finally: + httpd.shutdown() + # #endregion test_download_skips_invalid_and_non_http + + # #region test_download_query_string_filename_and_nss_import [C:2] [TYPE Function] [SEMANTICS test,integration,certs,nss] + # @BRIEF URL query strings are stripped from filenames and downloaded llm/ certs import into NSS. + def test_download_query_string_filename_and_nss_import(self): + import tempfile + + with tempfile.TemporaryDirectory() as certs_dir: + self._make_ca_cert(certs_dir, "query-ca") + httpd, http_port = self._serve_dir(certs_dir) + try: + url = f"http://127.0.0.1:{http_port}/query-ca.crt?token=abc" + r = self._run_download( + url, + "install_all_certs && install_ca_to_nss && " + "test -f /usr/local/share/ca-certificates/llm/query-ca.crt && " + "certutil -L -d sql:${HOME}/.pki/nssdb | grep -q 'llm-query-ca' && echo QUERY_NSS_OK", + ) + assert r.returncode == 0, f"Query/NSS flow failed:\n{r.stdout}\n{r.stderr}" + assert "QUERY_NSS_OK" in r.stdout + finally: + httpd.shutdown() + # #endregion test_download_query_string_filename_and_nss_import + + # #region test_volume_certs_matrix_pem_der_cer_invalid_private [C:2] [TYPE Function] [SEMANTICS test,integration,certs,volume] + # @BRIEF CERTS_PATH handles PEM/DER/CER and skips invalid/private/server files. + def test_volume_certs_matrix_pem_der_cer_invalid_private(self): + import tempfile + + with tempfile.TemporaryDirectory() as certs_dir: + _, pem_crt = self._make_ca_cert(certs_dir, "volume-pem") + _, der_src = self._make_ca_cert(certs_dir, "volume-der-src") + der_cert = os.path.join(certs_dir, "volume-der.der") + subprocess.run( + ["openssl", "x509", "-in", der_src, "-outform", "DER", "-out", der_cert], + check=True, capture_output=True, text=True, timeout=15, + ) + os.remove(der_src) + _, cer_src = self._make_ca_cert(certs_dir, "volume-cer-src") + os.rename(cer_src, os.path.join(certs_dir, "volume-cer.cer")) + Path(certs_dir, "invalid.crt").write_text("not a certificate", encoding="utf-8") + Path(certs_dir, "server.key").write_text("not a key", encoding="utf-8") + Path(certs_dir, "server.crt").write_text(Path(pem_crt).read_text(encoding="utf-8"), encoding="utf-8") + + r = subprocess.run( + [ + "docker", "run", "--rm", + "-v", f"{certs_dir}:/opt/certs:ro", + "-e", "CERTS_PATH=/opt/certs", + "--entrypoint", "bash", + BACKEND_IMAGE, + "-c", + "source /app/certs.sh && install_all_certs && " + "test -f /usr/local/share/ca-certificates/custom/volume-pem.crt && " + "test -f /usr/local/share/ca-certificates/custom/volume-der.crt && " + "test -f /usr/local/share/ca-certificates/custom/volume-cer.crt && " + "test ! -f /usr/local/share/ca-certificates/custom/server.crt && " + "grep -q 'BEGIN CERTIFICATE' /usr/local/share/ca-certificates/custom/volume-der.crt && " + "echo VOLUME_MATRIX_OK", + ], + capture_output=True, text=True, timeout=45, + ) + assert r.returncode == 0, f"Volume matrix failed:\n{r.stdout}\n{r.stderr}" + assert "VOLUME_MATRIX_OK" in r.stdout + assert "Установлено: 3" in r.stdout + assert "пропускаем: server.key" in r.stdout + assert "пропускаем: server.crt" in r.stdout + assert "невалидный сертификат: invalid.crt" in r.stdout + # #endregion test_volume_certs_matrix_pem_der_cer_invalid_private + + # #region test_download_and_volume_combined_channels [C:2] [TYPE Function] [SEMANTICS test,integration,certs,combined] + # @BRIEF LLM_CA_CERT_URLS and CERTS_PATH work together in one container startup flow. + def test_download_and_volume_combined_channels(self): + import tempfile + + with tempfile.TemporaryDirectory() as mount_dir, tempfile.TemporaryDirectory() as serve_dir: + self._make_ca_cert(mount_dir, "mounted-ca") + self._make_ca_cert(serve_dir, "downloaded-ca") + httpd, http_port = self._serve_dir(serve_dir) + try: + url = f"http://127.0.0.1:{http_port}/downloaded-ca.crt" + r = subprocess.run( + [ + "docker", "run", "--rm", + "--network", "host", + "-v", f"{mount_dir}:/opt/certs:ro", + "-e", "CERTS_PATH=/opt/certs", + "-e", f"LLM_CA_CERT_URLS={url}", + "--entrypoint", "bash", + BACKEND_IMAGE, + "-c", + "source /app/certs.sh && download_llm_ca_certs && install_all_certs && install_ca_to_nss && " + "test -f /usr/local/share/ca-certificates/custom/mounted-ca.crt && " + "test -f /usr/local/share/ca-certificates/llm/downloaded-ca.crt && " + "certutil -L -d sql:${HOME}/.pki/nssdb | grep -q 'custom-mounted-ca' && " + "certutil -L -d sql:${HOME}/.pki/nssdb | grep -q 'llm-downloaded-ca' && " + "echo COMBINED_OK", + ], + capture_output=True, text=True, timeout=60, + ) + assert r.returncode == 0, f"Combined channels failed:\n{r.stdout}\n{r.stderr}" + assert "COMBINED_OK" in r.stdout + finally: + httpd.shutdown() + # #endregion test_download_and_volume_combined_channels + + # #region test_empty_inputs_graceful [C:2] [TYPE Function] [SEMANTICS test,integration,certs,empty] + # @BRIEF Empty LLM_CA_CERT_URLS and missing CERTS_PATH are no-op success cases. + def test_empty_inputs_graceful(self): + r = subprocess.run( + [ + "docker", "run", "--rm", + "-e", "LLM_CA_CERT_URLS=", + "-e", "CERTS_PATH=/does-not-exist", + "--entrypoint", "bash", + BACKEND_IMAGE, + "-c", + "source /app/certs.sh && download_llm_ca_certs && install_all_certs && echo EMPTY_OK", + ], + capture_output=True, text=True, timeout=45, + ) + assert r.returncode == 0, f"Empty input path failed:\n{r.stdout}\n{r.stderr}" + assert "EMPTY_OK" in r.stdout + assert "CERTS_PATH=/does-not-exist не существует" in r.stdout + # #endregion test_empty_inputs_graceful + + # #endregion TestBackendContainer diff --git a/docker-compose.enterprise-clean.yml b/docker-compose.enterprise-clean.yml index f5fd359a..22113d12 100644 --- a/docker-compose.enterprise-clean.yml +++ b/docker-compose.enterprise-clean.yml @@ -54,6 +54,7 @@ services: ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} FEATURES__DATASET_REVIEW: ${FEATURES__DATASET_REVIEW:-true} FEATURES__HEALTH_MONITOR: ${FEATURES__HEALTH_MONITOR:-true} + LLM_CA_CERT_URLS: ${LLM_CA_CERT_URLS:-} # Путь до сертификатов внутри контейнера — всегда /opt/certs CERTS_PATH: /opt/certs ports: @@ -103,6 +104,7 @@ services: SERVICE_JWT: ${SERVICE_JWT:-agent-service-secret} DATABASE_URL: postgresql+psycopg2://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env.enterprise-clean}@${POSTGRES_HOST:?Set POSTGRES_HOST in .env.enterprise-clean}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-ss_tools} GRADIO_SERVER_PORT: 7860 + LLM_CA_CERT_URLS: ${LLM_CA_CERT_URLS:-} ports: - "${AGENT_HOST_PORT:-7860}:7860" volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 6425b18f..d188cbea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,6 +37,7 @@ services: INITIAL_ADMIN_PASSWORD: ${INITIAL_ADMIN_PASSWORD:-} FEATURES__DATASET_REVIEW: ${FEATURES__DATASET_REVIEW:-true} FEATURES__HEALTH_MONITOR: ${FEATURES__HEALTH_MONITOR:-true} + LLM_CA_CERT_URLS: ${LLM_CA_CERT_URLS:-} ports: - "${BACKEND_HOST_PORT:-8001}:8000" volumes: @@ -62,6 +63,7 @@ services: SERVICE_JWT: ${SERVICE_JWT:-agent-service-secret} DATABASE_URL: postgresql+psycopg2://postgres:postgres@db:5432/ss_tools GRADIO_SERVER_PORT: 7860 + LLM_CA_CERT_URLS: ${LLM_CA_CERT_URLS:-} ports: - "7860" # internal only, proxied through nginx volumes: diff --git a/docker/agent.entrypoint.sh b/docker/agent.entrypoint.sh index 10a82e8b..4a1d54ed 100644 --- a/docker/agent.entrypoint.sh +++ b/docker/agent.entrypoint.sh @@ -9,6 +9,7 @@ CERTS_PATH="${CERTS_PATH:-/opt/certs}" if [ -f "$SCRIPT_DIR/certs.sh" ]; then source "$SCRIPT_DIR/certs.sh" + download_llm_ca_certs install_all_certs else echo "[agent-entry] certs.sh not found — skipping corporate CA installation" diff --git a/docker/backend.entrypoint.sh b/docker/backend.entrypoint.sh index a6a6b36b..c773cd61 100755 --- a/docker/backend.entrypoint.sh +++ b/docker/backend.entrypoint.sh @@ -263,6 +263,7 @@ except Exception as exc: source "$(dirname "$0")/certs.sh" +download_llm_ca_certs install_all_certs install_ca_to_nss diff --git a/docker/certs.sh b/docker/certs.sh index 0295ae0c..5262e51f 100644 --- a/docker/certs.sh +++ b/docker/certs.sh @@ -24,96 +24,185 @@ normalize_cert() { return 1 } +# ── download_llm_ca_certs – download certs from HTTP URLs ──────────── +# @RATIONALE HTTP download from corporate PKI (e.g. pki.rusal.com) is the +# primary delivery mechanism for CA certificates. No chicken-and-egg TLS +# problem because URLs use plain HTTP, not HTTPS. +# @REJECTED HTTP-only HTTPS redirect was rejected — PKI servers may redirect +# HTTP→HTTPS using the same CA chain we are downloading (true chicken-and-egg). +download_llm_ca_certs() { + local urls="${LLM_CA_CERT_URLS:-}" + if [ -z "$urls" ]; then + return 0 + fi + + local work_dir="/tmp/llm-ca-certs" + local target_dir="/usr/local/share/ca-certificates/llm" + mkdir -p "$work_dir" "$target_dir" + + echo "[certs] === Скачивание CA-сертификатов из LLM_CA_CERT_URLS ===" + local index=0 downloaded=0 + + for url in $urls; do + url="$(echo "$url" | xargs)" # trim + [ -z "$url" ] && continue + index=$((index + 1)) + + case "$url" in + http://*) ;; + *) + echo "[certs] ⚠️ пропускаем не-HTTP URL: ${url}" + continue + ;; + esac + + local filename + filename="$(basename "$url" | sed 's/[?#].*//')" + if [ -z "$filename" ] || [ "$filename" = "." ] || [ "$filename" = ".." ]; then + filename="llm-ca-${index}.crt" + fi + + local raw_file="${work_dir}/${filename}" + local pem_file="${target_dir}/$(echo "$filename" | sed 's/\.[^.]*$//').crt" + + echo "[certs] [${index}] Скачивание: ${url}" + + local attempt=0 success=0 + while [ $attempt -lt 3 ]; do + attempt=$((attempt + 1)) + if curl -fsSL --proto '=http' --proto-redir '=http' --connect-timeout 10 --max-time 30 -o "$raw_file" "$url" 2>/dev/null; then + success=1 + break + fi + echo "[certs] ⚠ попытка ${attempt}/3 не удалась, повтор через 2с..." + sleep 2 + done + + if [ "$success" -eq 0 ]; then + echo "[certs] ❌ Не удалось скачать: ${url} — пропускаем" + continue + fi + + # Detect format: PEM or DER + if openssl x509 -in "$raw_file" -inform PEM -noout 2>/dev/null; then + echo "[certs] ✓ Формат: PEM" + cp "$raw_file" "$pem_file" + elif openssl x509 -in "$raw_file" -inform DER -noout 2>/dev/null; then + echo "[certs] ↻ Формат: DER — конвертируем в PEM" + if ! openssl x509 -in "$raw_file" -inform DER -out "$pem_file" -outform PEM 2>&1; then + echo "[certs] ❌ Ошибка конвертации DER→PEM — пропускаем" + rm -f "$raw_file" "${pem_file}" + continue + fi + else + echo "[certs] ❌ Неизвестный формат (не PEM и не DER) — пропускаем" + rm -f "$raw_file" + continue + fi + + chmod 644 "$pem_file" + echo "[certs] ✅ Установлен: $(basename "$pem_file")" + downloaded=$((downloaded + 1)) + rm -f "$raw_file" + done + + if [ "$downloaded" -gt 0 ]; then + echo "[certs] Скачано и установлено: ${downloaded}" + fi + rm -rf "$work_dir" +} + # ── install_all_certs – single unified entry point ─────────────────── install_all_certs() { local certs_dir="${CERTS_PATH:-/opt/certs}" if [ ! -d "$certs_dir" ]; then echo "[certs] CERTS_PATH=${certs_dir} не существует – пропускаем" - return 0 fi local target="/usr/local/share/ca-certificates/custom" mkdir -p "$target" - echo "[certs] Устанавливаем корпоративные сертификаты из ${certs_dir}..." - local installed=0 skipped=0 invalid=0 + if [ -d "$certs_dir" ]; then + echo "[certs] Устанавливаем корпоративные сертификаты из ${certs_dir}..." + local installed=0 skipped=0 invalid=0 - for f in "$certs_dir"/*; do - [ -f "$f" ] || continue - local name - name="$(basename "$f")" + for f in "$certs_dir"/*; do + [ -f "$f" ] || continue + local name + name="$(basename "$f")" - # Skip non-CA server/private files - case "$(echo "$name" | tr '[:upper:]' '[:lower:]')" in - server.crt|server.key|server.pem|*.key|*.p12|*.pfx) - echo "[certs] ⏭️ пропускаем: ${name} (server/private файл)" - skipped=$((skipped + 1)) - continue - ;; - esac + # Skip non-CA server/private files + case "$(echo "$name" | tr '[:upper:]' '[:lower:]')" in + server.crt|server.key|server.pem|*.key|*.p12|*.pfx) + echo "[certs] ⏭️ пропускаем: ${name} (server/private файл)" + skipped=$((skipped + 1)) + continue + ;; + esac - # Accept .crt, .pem, .cer, .der - case "$(echo "$name" | tr '[:upper:]' '[:lower:]')" in - *.crt|*.pem|*.cer|*.der) - # valid cert extension - ;; - *) - echo "[certs] ⚠️ неизвестный формат: ${name} – пропускаем" - skipped=$((skipped + 1)) - continue - ;; - esac + # Accept .crt, .pem, .cer, .der + case "$(echo "$name" | tr '[:upper:]' '[:lower:]')" in + *.crt|*.pem|*.cer|*.der) + ;; + *) + echo "[certs] ⚠️ неизвестный формат: ${name} – пропускаем" + skipped=$((skipped + 1)) + continue + ;; + esac - local out_name="${name%.*}.crt" - local out_file="${target}/${out_name}" + local out_name="${name%.*}.crt" + local out_file="${target}/${out_name}" - if normalize_cert "$f" "$out_file"; then - echo "[certs] ✅ ${name} (→ ${out_name})" - installed=$((installed + 1)) - else - echo "[certs] ❌ невалидный сертификат: ${name}" - invalid=$((invalid + 1)) - fi - done + if normalize_cert "$f" "$out_file"; then + echo "[certs] ✅ ${name} (→ ${out_name})" + installed=$((installed + 1)) + else + echo "[certs] ❌ невалидный сертификат: ${name}" + invalid=$((invalid + 1)) + fi + done - if [ "$installed" -eq 0 ]; then - echo "[certs] Нет CA-сертификатов для установки" - return 0 + echo "[certs] Установлено: ${installed}, пропущено: ${skipped}, невалидно: ${invalid}" fi + # ]]] SED_END_MARKER — do not remove, used by sed in update script - echo "[certs] Установлено: ${installed}, пропущено: ${skipped}, невалидно: ${invalid}" - - # Update system CA store + # Update system CA store (picks up both custom/ and llm/ dirs) if command -v update-ca-certificates &>/dev/null; then echo "[certs] Обновляем системное хранилище CA-сертификатов..." update-ca-certificates --fresh 2>&1 | sed 's/^/[certs] /' fi - # Create hash symlinks for OpenSSL capath + # Create hash symlinks for OpenSSL capath (scan both custom/ and llm/) echo "[certs] Создаём хеш-симлинки..." local sym_count=0 - for cert_file in "$target"/*.crt; do - [ -f "$cert_file" ] || continue - local cert_name - cert_name="$(basename "$cert_file" .crt)" - local hash_val - hash_val="$(openssl x509 -in "$cert_file" -noout -hash 2>/dev/null || true)" - if [ -n "$hash_val" ]; then - local suffix=0 - local link_path="/etc/ssl/certs/${hash_val}.${suffix}" - while [ -L "$link_path" ]; do - if [ "$(readlink "$link_path")" = "$cert_file" ]; then - break 2 + for ca_dir in /usr/local/share/ca-certificates/custom /usr/local/share/ca-certificates/llm; do + [ -d "$ca_dir" ] || continue + for cert_file in "$ca_dir"/*.crt; do + [ -f "$cert_file" ] || continue + local cert_name + cert_name="$(basename "$cert_file" .crt)" + local hash_val + hash_val="$(openssl x509 -in "$cert_file" -noout -hash 2>/dev/null || true)" + if [ -n "$hash_val" ]; then + local suffix=0 + local link_path="/etc/ssl/certs/${hash_val}.${suffix}" + local already_linked=0 + while [ -L "$link_path" ]; do + if [ "$(readlink "$link_path")" = "$cert_file" ]; then + already_linked=1 + break + fi + suffix=$((suffix + 1)) + link_path="/etc/ssl/certs/${hash_val}.${suffix}" + done + if [ "$already_linked" -eq 0 ] && [ ! -L "$link_path" ]; then + ln -sf "$cert_file" "$link_path" + sym_count=$((sym_count + 1)) fi - suffix=$((suffix + 1)) - link_path="/etc/ssl/certs/${hash_val}.${suffix}" - done - if [ ! -L "$link_path" ]; then - ln -sf "$cert_file" "$link_path" - sym_count=$((sym_count + 1)) fi - fi + done done echo "[certs] Создано хеш-симлинок: ${sym_count}" } @@ -124,19 +213,23 @@ install_ca_to_nss() { if ! command -v certutil &>/dev/null; then return 0 fi - local target="/usr/local/share/ca-certificates/custom" - if [ ! -d "$target" ] || [ -z "$(ls -A "$target" 2>/dev/null)" ]; then - return 0 - fi local nssdb="${HOME}/.pki/nssdb" mkdir -p "$nssdb" + if [ ! -f "${nssdb}/cert9.db" ]; then + certutil -N -d "sql:${nssdb}" --empty-password >/dev/null 2>&1 || true + fi local nss_count=0 echo "[certs] Импортируем сертификаты в NSS DB..." - for cert_file in "$target"/*.crt; do - [ -f "$cert_file" ] || continue - local cert_name - cert_name="$(basename "$cert_file" .crt)" - certutil -A -d "sql:${nssdb}" -t "C,," -n "custom-${cert_name}" -i "$cert_file" 2>/dev/null && nss_count=$((nss_count + 1)) || true + for ca_dir in /usr/local/share/ca-certificates/custom /usr/local/share/ca-certificates/llm; do + [ -d "$ca_dir" ] || continue + local prefix + prefix="$(basename "$ca_dir")" + for cert_file in "$ca_dir"/*.crt; do + [ -f "$cert_file" ] || continue + local cert_name + cert_name="$(basename "$cert_file" .crt)" + certutil -A -d "sql:${nssdb}" -t "C,," -n "${prefix}-${cert_name}" -i "$cert_file" 2>/dev/null && nss_count=$((nss_count + 1)) || true + done done echo "[certs] Импортировано в NSS: ${nss_count}" } diff --git a/docs/adr/ADR-0009-ssl-certificate-management.md b/docs/adr/ADR-0009-ssl-certificate-management.md index 1c199966..7e7993c4 100644 --- a/docs/adr/ADR-0009-ssl-certificate-management.md +++ b/docs/adr/ADR-0009-ssl-certificate-management.md @@ -1,6 +1,6 @@ # [DEF:ADR-0009:ADR] # @STATUS ACTIVE -# @PURPOSE Define the strategy for corporate SSL certificate management across all HTTP clients in superset-tools (httpx, requests, openai), covering system CA store, NSS database for Chromium/Playwright. Centralized CERTS_PATH contract replaces per-client SSL toggles. LLM_SSL_VERIFY and LLM_CA_CERT_URLS removed in 0.2.x. +# @PURPOSE Define the strategy for corporate SSL certificate management across all HTTP clients in superset-tools (httpx, requests, openai), covering system CA store, NSS database for Chromium/Playwright. Two input channels: (1) HTTP download from corporate PKI via LLM_CA_CERT_URLS (primary), (2) volume mount via CERTS_PATH (additional). LLM_SSL_VERIFY removed in 0.2.x. # @RELATION DEPENDS_ON -> [ADR-0001:ADR] # @RELATION DEPENDS_ON -> [ADR-0004:ADR] # @RELATION CALLS -> [docker/backend.entrypoint.sh] @@ -97,24 +97,32 @@ intermediate CA in OpenSSL 3.x. `verify=` works correctly. - Trust attributes: `"C,,"` (trusted CA for TLS server certs) - Fixes `ERR_CERT_AUTHORITY_INVALID` in Playwright dashboard screenshots -### Layer 4: Centralized CERTS_PATH Trust Contract (0.2.x) +### Layer 4: LLM_CA_CERT_URLS HTTP Download (0.5.1) -- **REMOVED**: `LLM_SSL_VERIFY` — TLS verification is always enabled; no env escape hatch exists. -- **REMOVED**: `LLM_CA_CERT_URLS` — certificate download by URL is no longer supported. -- **Replaced by**: `CERTS_PATH` volume mount (`./certs:/opt/certs:ro`) as the single certificate input. -- All trust anchors come from mounted `CERTS_PATH` directory (`.crt`, `.pem`, `.cer`, `.der` files). +- Env var `LLM_CA_CERT_URLS` — space-separated list of HTTP URLs to CA certificates +- Downloaded on container startup by `certs.sh:download_llm_ca_certs()` +- URLs use **plain HTTP only** — no TLS chicken-and-egg problem +- Supports PEM and DER formats (auto-detected, DER auto-converted to PEM) +- Retry logic: 3 attempts with 2-second delay +- Downloaded certs saved to `/usr/local/share/ca-certificates/llm/` +- `update-ca-certificates --fresh` includes them in the system bundle +- Example: `LLM_CA_CERT_URLS="http://pki.rusal.com/CertData/UC_RUSAL_Policy_CA.crt http://pki.rusal.com/CertData/UC_RUSAL_RGM_Issuing_CA.crt"` + +### Layer 5: CERTS_PATH Volume Mount + +- Env var `CERTS_PATH` — directory with `.crt`/`.pem`/`.cer`/`.der` files +- Mounted as read-only volume: `${CERTS_PATH:-./certs}:/opt/certs:ro` +- Installed by `certs.sh:install_all_certs()` **after** `download_llm_ca_certs()` +- Server/private files (server.crt, *.key, *.p12, *.pfx) are excluded from CA import - Shared `docker/certs.sh` handles installation across all containers (backend, frontend, agent). -- Centralized Python helper `backend/src/core/ssl.py` provides `system_ssl_context()` / `httpx_verify()`. -- No code path disables TLS verification — the `verify=False` escape hatch is permanently removed. -- **Migration**: Place corporate CA files in `./certs/`, remove `LLM_SSL_VERIFY`/`LLM_CA_CERT_URLS` from `.env`. ## Key Files | File | Role | |------|------| | `docker/backend.Dockerfile` | Installs `libnss3-tools` (certutil), Playwright Chromium | -| `docker/certs.sh` | Shared cert installer: `install_all_certs`, `install_ca_to_nss`, `normalize_cert` | -| `docker/backend.entrypoint.sh` | Sources `certs.sh`, calls `install_all_certs` + `install_ca_to_nss` | +| `docker/certs.sh` | Shared cert installer: `download_llm_ca_certs`, `install_all_certs`, `install_ca_to_nss`, `normalize_cert` | +| `docker/backend.entrypoint.sh` | Sources `certs.sh`, calls `download_llm_ca_certs` → `install_all_certs` → `install_ca_to_nss` | | `docker/frontend.entrypoint.sh` | Installs CA certs from `CERTS_PATH` into Alpine store | | `docker/agent.entrypoint.sh` | Sources `certs.sh`, installs certs for agent container | | `backend/src/core/ssl.py` | Centralized SSL helper: `system_ssl_context()`, `httpx_verify()`, `describe_context()` | @@ -250,5 +258,6 @@ docker compose -f docker-compose.enterprise-clean.yml \ | 0.1.6 | QA fixes: fingerprint dedup, hash symlink collision, NSS collision, DER→PEM error handling, chicken-and-egg TLS, nullglob, translate plugin SSL | | 0.1.7 | **capath instead of cafile**: OpenSSL 3.x cafile limitation discovered and fixed. `.crt` extension for downloaded certs. Diagnostic script rewritten. | | 0.2.x | **Centralized SSL**: `LLM_SSL_VERIFY` and `LLM_CA_CERT_URLS` removed. Shared `docker/certs.sh` for all containers. `backend/src/core/ssl.py` central helper. `CERTS_PATH` volume mount is the only certificate input. Verify=False escape hatch permanently removed. | +| 0.5.1 | **Restored**: `LLM_CA_CERT_URLS` HTTP download as primary cert input channel. `download_llm_ca_certs()` added to `certs.sh`. Downloaded certs saved to `llm/` subdirectory, `install_all_certs` extended to create hash symlinks for both `custom/` and `llm/` dirs. `CERTS_PATH` volume mount kept as secondary channel. Integration test added for HTTP download flow. | # [/DEF:ADR-0009:ADR]