diff --git a/README.md b/README.md index 32d26275..74187273 100755 --- a/README.md +++ b/README.md @@ -303,24 +303,19 @@ Entrypoint автоматически: Положите `.crt`/`.pem` файлы в `./certs/` — entrypoint установит их в системное хранилище Alpine и NSS (Chromium/Playwright). Поддерживаются цепочки из нескольких CA (Root → Intermediate). -### LLM CA-сертификаты - -Если LLM-провайдер использует корпоративный PKI — укажите URL для скачивания CA-сертификатов: -```bash -LLM_CA_CERT_URLS="http://pki.company.com/root-ca.crt http://pki.company.com/intermediate-ca.crt" -``` -Сертификаты скачиваются на старте backend-контейнера, конвертируются из DER в PEM при необходимости и устанавливаются в системное хранилище. Подробнее — в [ADR-0009](docs/adr/ADR-0009-ssl-certificate-management.md). - -### Диагностика +### Диагностика SSL ```bash -# Проверка доверия сертификата +# Полная проверка цепочки доверия scripts/check_llm_certs.py -# Принудительное отключение TLS verify (только для отладки) -LLM_SSL_VERIFY=false +# Контейнерная диагностика +scripts/diag_container.py --target your-llm-provider.com:443 ``` +Сертификаты и централизованный SSL — подробнее в [ADR-0009](docs/adr/ADR-0009-ssl-certificate-management.md). +`LLM_SSL_VERIFY` и `LLM_CA_CERT_URLS` удалены в 0.2.x — используйте `CERTS_PATH=./certs`. + ## 🏢 Enterprise Clean Deployment Для разворота в корпоративной сети с очищенным дистрибутивом (без тестовых данных, с запретом внешних источников и обязательной compliance-проверкой) используется профиль **enterprise clean**. diff --git a/backend/src/core/ssl.py b/backend/src/core/ssl.py index ae305759..206fa70e 100644 --- a/backend/src/core/ssl.py +++ b/backend/src/core/ssl.py @@ -51,21 +51,22 @@ def describe_context(ctx: ssl.SSLContext | bool | None) -> str: """Return a diagnostic description of the SSL verification state. Does NOT leak key material or secret values. + SSLContext in Python 3.13 does not expose capath/cafile as public + attributes — verify_load_locations info is internal to OpenSSL. + Instead, reports whether /etc/ssl/certs (DEFAULT_CAPATH) is available + and the cert count in the system bundle. """ if ctx is False: return "DISABLED (verify=False — should not happen in production)" if ctx is None: return "DEFAULT (certifi)" if isinstance(ctx, ssl.SSLContext): - capath = getattr(ctx, "capath", None) or getattr(ctx, "_capath", None) - cafile = getattr(ctx, "cafile", None) or getattr(ctx, "_cafile", None) - parts = [] - if cafile: - parts.append(f"cafile={cafile}") - if capath: - parts.append(f"capath={capath}") - parts.append(f"verify_mode={'CERT_REQUIRED' if ctx.verify_mode == ssl.CERT_REQUIRED else ctx.verify_mode}") - return "SSLContext(" + ", ".join(parts) + ")" + bundle = Path(DEFAULT_CAPATH) / "ca-certificates.crt" + cert_count = 0 + if bundle.exists(): + cert_count = bundle.read_bytes().count(b"-----BEGIN CERTIFICATE-----") + mode = "CERT_REQUIRED" if ctx.verify_mode == ssl.CERT_REQUIRED else str(ctx.verify_mode) + return f"SSLContext(verify_mode={mode}, system_certs={cert_count})" return f"unknown({type(ctx).__name__})" diff --git a/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py b/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py index ebf9430f..d0fc9471 100644 --- a/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py +++ b/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py @@ -27,6 +27,8 @@ def test_openrouter_client_includes_referer_and_title_headers(monkeypatch): assert headers["Authorization"] == "Bearer sk-test-provider-key-123456" assert headers["HTTP-Referer"] == "http://localhost:8000" assert headers["X-Title"] == "superset-tools-test" + + # endregion test_openrouter_client_includes_referer_and_title_headers @@ -51,48 +53,44 @@ def test_litellm_client_uses_default_bearer_auth(): assert "X-Title" not in headers assert "Authentication" not in headers assert "X-API-Key" not in headers + + # endregion test_litellm_client_uses_default_bearer_auth # region test_get_ssl_verify_default_context [TYPE Function] # @RELATION BINDS_TO -> TestClientHeaders # @PURPOSE: _get_ssl_verify returns SSLContext with CERT_REQUIRED by default. -# @PRE LLM_SSL_VERIFY env var is not set. +# @PRE Centralized SSL is active (no LLM_SSL_VERIFY dependency). # @POST Returns SSLContext with verify_mode=CERT_REQUIRED. def test_get_ssl_verify_default_context(): """Verify default returns SSLContext with CERT_REQUIRED.""" import ssl + result = LLMClient._get_ssl_verify() assert isinstance(result, ssl.SSLContext), "Expected SSLContext, not bool" assert result.verify_mode == ssl.CERT_REQUIRED + + # endregion test_get_ssl_verify_default_context -# region test_get_ssl_verify_false_values [TYPE Function] +# region test_get_ssl_verify_ignores_llm_env [TYPE Function] # @RELATION BINDS_TO -> TestClientHeaders -# @PURPOSE: _get_ssl_verify returns False for "false"/"0"/"no"/"off". -# @PRE LLM_SSL_VERIFY env var is set to each falsy value. -# @POST Returns False. -def test_get_ssl_verify_false_values(monkeypatch): - for val in ("false", "False", "0", "no", "off"): - monkeypatch.setenv("LLM_SSL_VERIFY", val) - assert LLMClient._get_ssl_verify() is False, f"Expected False for LLM_SSL_VERIFY={val}" -# endregion test_get_ssl_verify_false_values - - -# region test_get_ssl_verify_true_returns_context [TYPE Function] -# @RELATION BINDS_TO -> TestClientHeaders -# @PURPOSE: _get_ssl_verify returns SSLContext for truthy env values. -# @PRE LLM_SSL_VERIFY env var is set to truthy values. -# @POST Returns SSLContext with CERT_REQUIRED. -def test_get_ssl_verify_true_returns_context(monkeypatch): +# @PURPOSE: _get_ssl_verify ignores LLM_SSL_VERIFY env (centralized SSL — no verify=False escape hatch). +# @PRE LLM_SSL_VERIFY env var is set to each falsy/truthy value. +# @POST Always returns SSLContext with CERT_REQUIRED — env var is ignored. +def test_get_ssl_verify_ignores_llm_env(monkeypatch): import ssl - for val in ("TRUE", "1", "yes", "on", "true"): + + for val in ("false", "False", "0", "no", "off", "true", "1", "yes"): monkeypatch.setenv("LLM_SSL_VERIFY", val) result = LLMClient._get_ssl_verify() - assert isinstance(result, ssl.SSLContext), f"Expected SSLContext for LLM_SSL_VERIFY={val}" - assert result.verify_mode == ssl.CERT_REQUIRED -# endregion test_get_ssl_verify_true_returns_path + assert isinstance(result, ssl.SSLContext), f"Expected SSLContext for LLM_SSL_VERIFY={val}, got {type(result)}" + assert result.verify_mode == ssl.CERT_REQUIRED, f"Expected CERT_REQUIRED for LLM_SSL_VERIFY={val}" + + +# endregion test_get_ssl_verify_ignores_llm_env # region test_format_connection_error_no_cause [TYPE Function] @@ -105,6 +103,8 @@ def test_format_connection_error_no_cause(): result = LLMClient._format_connection_error(exc) assert "ValueError: test error" in result assert "└─" not in result + + # endregion test_format_connection_error_no_cause @@ -121,6 +121,8 @@ def test_format_connection_error_with_cause(): assert "RuntimeError: API call failed" in result assert "└─" in result assert "ConnectionRefusedError: Connection refused" in result + + # endregion test_format_connection_error_with_cause @@ -139,17 +141,20 @@ def test_format_connection_error_deep_chain(): assert "RuntimeError: Connection error" in result assert "ConnectionError: SSL: CERTIFICATE_VERIFY_FAILED" in result assert "Exception: Name or service not known" in result + + # endregion test_format_connection_error_deep_chain # region test_litellm_client_verify_default [TYPE Function] # @RELATION BINDS_TO -> TestClientHeaders -# @PURPOSE: LLMClient passes verify=True to httpx.AsyncClient by default. -# @PRE LLM_SSL_VERIFY env var is not set. +# @PURPOSE: LLMClient passes verify context to httpx.AsyncClient by default. +# @PRE Centralized SSL is active (no LLM_SSL_VERIFY dependency). # @POST httpcore pool SSL context has verify_mode=CERT_REQUIRED (2). def test_litellm_client_verify_default(): - """Verify default SSL context with CERT_REQUIRED when LLM_SSL_VERIFY unset.""" + """Verify default SSL context with CERT_REQUIRED — centralized SSL in use.""" import ssl + client = LLMClient( provider_type=LLMProviderType.LITELLM, api_key="sk-test-verify-key", @@ -157,17 +162,19 @@ def test_litellm_client_verify_default(): default_model="gpt-4o", ) pool = client.client._client._transport._pool - assert pool._ssl_context.verify_mode == ssl.CERT_REQUIRED, \ - "verify=True should set verify_mode to CERT_REQUIRED" + assert pool._ssl_context.verify_mode == ssl.CERT_REQUIRED, "verify=True should set verify_mode to CERT_REQUIRED" + + # endregion test_litellm_client_verify_default -# region test_litellm_client_verify_disabled [TYPE Function] +# region test_litellm_client_verify_always_required [TYPE Function] # @RELATION BINDS_TO -> TestClientHeaders -# @PURPOSE: LLMClient passes verify=False when LLM_SSL_VERIFY=false. +# @PURPOSE: LLMClient always uses CERT_REQUIRED — centralized SSL ignores LLM_SSL_VERIFY. # @PRE LLM_SSL_VERIFY is set to "false". -# @POST httpcore pool SSL context has verify_mode=CERT_NONE (0). -def test_litellm_client_verify_disabled(monkeypatch): +# @POST httpcore pool SSL context has verify_mode=CERT_REQUIRED (2). +def test_litellm_client_verify_always_required(monkeypatch): + """Verify LLM_SSL_VERIFY=false is ignored — verify mode always CERT_REQUIRED.""" monkeypatch.setenv("LLM_SSL_VERIFY", "false") client = LLMClient( provider_type=LLMProviderType.LITELLM, @@ -176,8 +183,10 @@ def test_litellm_client_verify_disabled(monkeypatch): default_model="gpt-4o", ) import ssl + pool = client.client._client._transport._pool - assert pool._ssl_context.verify_mode == ssl.CERT_NONE, \ - "verify=False should set verify_mode to CERT_NONE" -# endregion test_litellm_client_verify_disabled + assert pool._ssl_context.verify_mode == ssl.CERT_REQUIRED, "LLM_SSL_VERIFY=false must be ignored — verify_mode stays CERT_REQUIRED" + + +# endregion test_litellm_client_verify_always_required # endregion TestClientHeaders diff --git a/backend/src/plugins/llm_analysis/service.py b/backend/src/plugins/llm_analysis/service.py index 85a2c1e3..7c10482d 100644 --- a/backend/src/plugins/llm_analysis/service.py +++ b/backend/src/plugins/llm_analysis/service.py @@ -911,9 +911,9 @@ class LLMClient: # is sufficient. No additional headers like HTTP-Referer or X-API-Key are required. ssl_verify = self._get_ssl_verify() - ssl_desc = str(ssl_verify) - if isinstance(ssl_verify, ssl.SSLContext): - ssl_desc = f"SSLContext(capath={ssl_verify.capath if hasattr(ssl_verify, 'capath') else 'default'})" + from ...core.ssl import describe_context + + ssl_desc = describe_context(ssl_verify) logger.reason("LLM client SSL verification configured", payload={"ssl_verify": ssl_desc}) http_client = httpx.AsyncClient( @@ -932,8 +932,7 @@ class LLMClient: # #region LLMClient._get_ssl_verify [TYPE Function] # @PURPOSE Resolve SSL verification flag from environment. - # @POST Returns SSLContext with system CA dir when enabled, - # False when LLM_SSL_VERIFY env var is "false"/"0"/"no"/"off". + # @POST Returns SSLContext with system CA dir (never False — centralized SSL). # @RATIONALE Используем capath=/etc/ssl/certs/ вместо cafile, потому что # OpenSSL 3.x не использует intermediate CA сертификаты из cafile для # построения цепочки (verify code 20). capath с хеш-симлинками работает diff --git a/backend/src/plugins/translate/_llm_async_http.py b/backend/src/plugins/translate/_llm_async_http.py index 007e2e4b..3652aa5f 100644 --- a/backend/src/plugins/translate/_llm_async_http.py +++ b/backend/src/plugins/translate/_llm_async_http.py @@ -1,5 +1,5 @@ # #region LLMAsyncHttpClient [C:4] [TYPE Module] [SEMANTICS translate, llm, http, openai, async, retry, rate-limit] -# @defgroup Translate Module group. +# @ingroup Translate # @BRIEF Async HTTP client for OpenAI-compatible LLM API calls using httpx.AsyncClient # with rate-limit handling and structured output fallback. # @LAYER Infrastructure @@ -34,7 +34,7 @@ DEFAULT_MAX_TOKENS: int = 8192 # #region _get_verify [C:1] [TYPE Function] [SEMANTICS translate, ssl, verify] -# @BRIEF Resolve SSL verification context from LLM_SSL_VERIFY env var. +# @BRIEF Resolve SSL verification context via centralized core.ssl helper. # @RATIONALE Используем capath=/etc/ssl/certs/ через ssl.create_default_context, # потому что OpenSSL 3.x не использует intermediate CA сертификаты из cafile # для построения цепочки (verify code 20). capath с хеш-симлинками работает @@ -44,8 +44,10 @@ DEFAULT_MAX_TOKENS: int = 8192 # из единого bundle-файла. Только capath с хеш-симлинками даёт code 0. # @REJECTED Строковый путь "/etc/ssl/certs/" отвергнут — httpx 0.28.x депрекейтит # строки в verify=, требует SSLContext. -# @POST Returns ssl.SSLContext with capath when enabled, False when disabled. -def _get_verify() -> ssl.SSLContext | bool: +# @REJECTED LLM_SSL_VERIFY=false escape hatch отвергнут — централизованный SSL +# всегда использует системное хранилище без возможности отключения. +# @POST Returns ssl.SSLContext with capath (never False). +def _get_verify() -> ssl.SSLContext: from ...core.ssl import httpx_verify return httpx_verify() @@ -61,8 +63,6 @@ async def _get_http_client() -> httpx.AsyncClient: global _http_client if _http_client is None: ssl_verify = _get_verify() - if ssl_verify is False: - log("LLMAsyncHttpClient", "EXPLORE", "TLS verification disabled via LLM_SSL_VERIFY=false", error="TLS verification disabled — traffic to LLM provider is unencrypted") _http_client = httpx.AsyncClient( verify=ssl_verify, timeout=httpx.Timeout(180.0), diff --git a/backend/tests/integration/conftest.py b/backend/tests/integration/conftest.py index 83227247..ea44a14d 100644 --- a/backend/tests/integration/conftest.py +++ b/backend/tests/integration/conftest.py @@ -485,11 +485,11 @@ def superset_container(request, superset_db_url, superset_secret_key, superset_a key_pem = ca_chain["server_key_encrypted"] key_pass = ca_chain["server_key_passphrase"] key_b64 = base64.b64encode(key_pem.encode()).decode() - _write_certs = f"echo '{fullchain_b64}' | base64 -d > /tmp/server.crt && echo '{key_b64}' | base64 -d > /tmp/server.key" - _decrypt_key = f"export SSL_KEY_PASSPHRASE='{key_pass}' && openssl rsa -in /tmp/server.key -passin env:SSL_KEY_PASSPHRASE -out /tmp/server.key 2>/dev/null; if [ $? -ne 0 ]; then echo 'ERROR: Failed to decrypt private key with SSL_KEY_PASSPHRASE'; exit 1; fi" + _write_certs = f'echo "{fullchain_b64}" | base64 -d > /tmp/server.crt && echo "{key_b64}" | base64 -d > /tmp/server.key' + _decrypt_key = f'export SSL_KEY_PASSPHRASE="{key_pass}" && openssl rsa -in /tmp/server.key -passin env:SSL_KEY_PASSPHRASE -out /tmp/server.key 2>/dev/null; if [ $? -ne 0 ]; then echo "ERROR: Failed to decrypt private key with SSL_KEY_PASSPHRASE"; exit 1; fi' else: key_b64 = base64.b64encode(ca_chain["server_key"].encode()).decode() - _write_certs = f"echo '{fullchain_b64}' | base64 -d > /tmp/server.crt && echo '{key_b64}' | base64 -d > /tmp/server.key" + _write_certs = f'echo "{fullchain_b64}" | base64 -d > /tmp/server.crt && echo "{key_b64}" | base64 -d > /tmp/server.key' _decrypt_key = "" _protocol = "https" @@ -537,6 +537,119 @@ def superset_container(request, superset_db_url, superset_secret_key, superset_a if use_tls and _write_certs: _decrypt_prefix = f"{_decrypt_key} && " if _decrypt_key else "" web_cmd = f"sh -c 'python -m pip install psycopg2-binary -q && {_write_certs} && {_decrypt_prefix}{_bootstrap_config}{_export_config} && {_superset_run}'" + else: + web_cmd = f"sh -c 'python -m pip install psycopg2-binary -q && {_bootstrap_config}{_export_config} && {_superset_run}'" + + superset.with_command(web_cmd) + superset.with_exposed_ports(8088) + + superset.start() + + # Store protocol on the container object for superset_url fixture + superset._ss_protocol = _protocol + + # Wait for the /health endpoint to respond + host = superset.get_container_host_ip() + port = superset.get_exposed_port(8088) + health_url = f"{_protocol}://{host}:{port}/health" + + deadline = time.time() + 90 + last_error = None + + while time.time() < deadline: + try: + # verify=False in TLS mode because the custom CA hasn't been installed yet + resp = requests.get(health_url, timeout=3, verify=False) + if resp.status_code == 200: + break + except requests.RequestException as e: + last_error = e + time.sleep(2) + else: + logs = superset.get_logs() + superset.stop() + raise TimeoutError(f"Superset did not become healthy within 90s. Last error: {last_error}\nContainer logs:\n{logs}") + + yield superset + + superset.stop() + + +# #endregion superset_container + + +# #region superset_url [C:1] [TYPE Fixture] +# @BRIEF Function-scoped — base URL of the running Superset container (auto-detects http/https). +@pytest.fixture +def superset_url(superset_container): + host = superset_container.get_container_host_ip() + port = superset_container.get_exposed_port(8088) + protocol = getattr(superset_container, "_ss_protocol", "http") + return f"{protocol}://{host}:{port}" + + +# #endregion superset_url + + +# #region superset_admin_headers [C:2] [TYPE Fixture] +# @BRIEF Function-scoped — authenticated session headers for Superset admin API calls. +@pytest.fixture +def superset_admin_headers(superset_url, superset_admin_password): + """Log in as admin and return headers with CSRF token and session cookie.""" + session = requests.Session() + + # Step 1: Get CSRF token from login page + login_url = f"{superset_url}/login/" + resp = session.get(login_url, timeout=10) + resp.raise_for_status() + + csrf_token = None + if "csrf_token" in resp.text: + import re + + match = re.search(r'csrf_token"\s*:\s*"([^"]+)"', resp.text) + if match: + csrf_token = match.group(1) + + # Step 2: Submit login form + resp = session.post( + login_url, + data={ + "username": "admin", + "password": superset_admin_password, + "csrf_token": csrf_token or "", + }, + headers={"Referer": login_url}, + timeout=10, + ) + resp.raise_for_status() + + return session + + +# #endregion superset_admin_headers + + +# ── JWT-based Superset Fixtures ──────────────────────────────────── +# ADR-0012: после установки psycopg2 + superset_config.py + Docker bridge IP, +# JWT-авторизация (/api/v1/security/login) работает в тестовом контейнере. + + +# #region superset_jwt_headers [C:2] [TYPE Fixture] +# @BRIEF Function-scoped — JWT Bearer token headers for Superset REST API. +# @POST Returns dict with Authorization: Bearer . +@pytest.fixture +def superset_jwt_headers(superset_url, superset_admin_password): + """Obtain JWT access token from the running Superset container.""" + resp = requests.post( + f"{superset_url}/api/v1/security/login", + json={ + "username": "admin", + "password": superset_admin_password, + "provider": "db", + }, + timeout=10, + ) resp.raise_for_status() data = resp.json() access_token = data["access_token"] diff --git a/backend/tests/integration/test_backend_container.py b/backend/tests/integration/test_backend_container.py new file mode 100644 index 00000000..0616e6f9 --- /dev/null +++ b/backend/tests/integration/test_backend_container.py @@ -0,0 +1,312 @@ +# #region TestBackendContainer [C:3] [TYPE Module] [SEMANTICS test,integration,backend,container,docker,entrypoint,certs] +# @BRIEF Integration tests for the superset-tools-backend Docker image: +# verifies entrypoint + certs.sh work correctly inside the container. +# @RELATION BINDS_TO -> [docker/backend.Dockerfile] +# @RELATION BINDS_TO -> [docker/certs.sh] +# @RELATION BINDS_TO -> [docker/backend.entrypoint.sh] +# @RELATION CALLS -> [TestSupersetTlsCustomCA] +# @RATIONALE In 0.5.0 release, backend entrypoint crashed with +# "/app/entrypoint.sh: line 264: /app/certs.sh: No such file or directory" +# because docker/backend.Dockerfile was missing COPY for certs.sh. +# Existing testcontainers tests only exercise apache/superset containers, +# not the superset-tools backend container. These tests close that gap. +# @TEST_CONTRACT BackendImageCertPaths -> +# /app/certs.sh exists and is readable +# source /app/certs.sh defines install_all_certs, install_ca_to_nss, normalize_cert +# /app/entrypoint.sh exists and is executable +# @TEST_CONTRACT BackendContainerSmoke -> +# docker compose up backend starts successfully +# /api/health/summary responds 200 +# Container logs do NOT contain "certs.sh: No such file or directory" +# Container logs do NOT contain "Traceback" or unrecoverable startup error +# @REQUIRES Docker daemon running. --run-integration flag. +"""Integration tests for superset-tools-backend Docker image. + +Two test suites: + +1. **TestBackendImageCertsSh** — fast (no DB, no compose). + Builds the backend Docker image, runs a container with --entrypoint bash, + and verifies certs.sh is present, sourceable, and all certificate + installation functions are defined. + +2. **TestBackendContainerSmoke** — full stack (needs PostgreSQL). + Starts PostgreSQL via testcontainers, deploys backend via docker compose, + waits for the health endpoint, and checks container logs for startup errors. +""" + +import os +import re +import subprocess +import time +from pathlib import Path + +import pytest + +# ── Project paths ──────────────────────────────────────────────────────── +PROJECT_ROOT = Path(__file__).resolve().parents[3] +BACKEND_IMAGE = "superset-tools-backend:test-integration" + + +# ── Fixtures ────────────────────────────────────────────────────────────── + + +@pytest.fixture(scope="module") +def _backend_image(): + """Build superset-tools-backend:test-integration once per module.""" + # Only rebuild if the Dockerfile changed (simple mtime check) + dockerfile = PROJECT_ROOT / "docker" / "backend.Dockerfile" + image_marker = PROJECT_ROOT / "backend" / ".image-built" + + if not image_marker.exists() or image_marker.stat().st_mtime < dockerfile.stat().st_mtime: + subprocess.run( + ["docker", "build", "-f", str(dockerfile), "-t", BACKEND_IMAGE, str(PROJECT_ROOT)], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + image_marker.touch() + return BACKEND_IMAGE + + +# ══════════════════════════════════════════════════════════════════════════ +# TEST 1: Fast — verify certs.sh presence and basic functionality +# ══════════════════════════════════════════════════════════════════════════ + + +class TestBackendImageCertPaths: + """Fast checks: certs.sh exists, is sourceable, defines all functions. + + These tests run in ~10s without PostgreSQL or docker compose. + They catch the 'certs.sh: No such file or directory' failure mode. + """ + + @pytest.fixture(autouse=True) + def _image(self, _backend_image): + pass + + # ── helpers ───────────────────────────────────────────────────── + + @staticmethod + def _run_bash(command: str) -> subprocess.CompletedProcess: + return subprocess.run( + ["docker", "run", "--rm", "--entrypoint", "bash", BACKEND_IMAGE, "-c", command], + capture_output=True, + text=True, + timeout=30, + ) + + # ── tests ─────────────────────────────────────────────────────── + + # #region test_certs_sh_exists [C:1] [TYPE Function] [SEMANTICS test,integration,certs] + # @BRIEF /app/certs.sh must exist and be readable. + def test_certs_sh_exists(self): + r = self._run_bash("test -f /app/certs.sh && test -r /app/certs.sh") + assert r.returncode == 0, f"File exists: {r.returncode == 0}, stderr: {r.stderr[:200]}" + + # #endregion test_certs_sh_exists + + # #region test_entrypoint_sh_exists [C:1] [TYPE Function] [SEMANTICS test,integration,entrypoint] + # @BRIEF /app/entrypoint.sh must exist and be executable. + def test_entrypoint_sh_exists(self): + r = self._run_bash("test -f /app/entrypoint.sh && test -x /app/entrypoint.sh") + assert r.returncode == 0, f"File: /app/entrypoint.sh, stderr: {r.stderr[:200]}" + + # #endregion test_entrypoint_sh_exists + + # #region test_certs_sh_source_defines_functions [C:2] [TYPE Function] [SEMANTICS test,integration,certs] + # @BRIEF sourcing /app/certs.sh defines install_all_certs, install_ca_to_nss, normalize_cert. + def test_certs_sh_source_defines_functions(self): + r = self._run_bash("source /app/certs.sh && declare -F install_all_certs >/dev/null 2>&1 && echo -n 'install_all_certs '; declare -F install_ca_to_nss >/dev/null 2>&1 && echo -n 'install_ca_to_nss '; declare -F normalize_cert >/dev/null 2>&1 && echo -n 'normalize_cert'") + assert "install_all_certs" in r.stdout, f"install_all_certs not defined:\n{r.stderr}" + assert "install_ca_to_nss" in r.stdout, f"install_ca_to_nss not defined:\n{r.stderr}" + assert "normalize_cert" in r.stdout, f"normalize_cert not defined:\n{r.stderr}" + + # #endregion test_certs_sh_source_defines_functions + + # #region test_entrypoint_sources_certs_sh_ok [C:2] [TYPE Function] [SEMANTICS test,integration,entrypoint] + # @BRIEF Running the real entrypoint script (without DB) must not crash on source certs.sh. + # We pass a short DB wait timeout so it fails fast instead of retrying 30 times. + # The certs.sh sourcing happens BEFORE the DB wait loop. + def test_entrypoint_sources_certs_sh_ok(self): + r = subprocess.run( + [ + "docker", + "run", + "--rm", + "-e", + "POSTGRES_HOST=does-not-exist", + "-e", + "POSTGRES_USER=test", + "-e", + "POSTGRES_PASSWORD=test", + "-e", + "POSTGRES_DB=test", + # Reduce wait time — if certs.sh source succeeds, the error + # will be about DB connection, not certs.sh + "--entrypoint", + "bash", + BACKEND_IMAGE, + "-c", + "timeout 5 /app/entrypoint.sh 2>&1 || true", + ], + capture_output=True, + text=True, + timeout=15, + ) + # The certs.sh source comes BEFORE DB wait. + # If certs.sh is missing, the output MUST contain "No such file or directory". + # If certs.sh loads OK, the output should show DB connection error OR cert install. + assert re.search(r"No such file or directory.*certs\.sh|certs\.sh.*No such file or directory", r.stdout + r.stderr) is None, f"certs.sh not found! Entrypoint tried to source missing file:\nSTDOUT:\n{r.stdout[:2000]}\nSTDERR:\n{r.stderr[:2000]}" + + # #endregion test_entrypoint_sources_certs_sh_ok + + +# ══════════════════════════════════════════════════════════════════════════ +# TEST 2: Full stack — docker compose up backend + health check +# ══════════════════════════════════════════════════════════════════════════ + + +@pytest.mark.skipif( + not os.environ.get("E2E_BACKEND_CONTAINER"), + reason="Skipped by default — set E2E_BACKEND_CONTAINER=1 to run full-stack test", +) +@pytest.mark.usefixtures("_backend_image") +class TestBackendContainerSmoke: + """Full-stack smoke test: deploy backend container via docker compose + with PostgreSQL, wait for health endpoint, verify container logs. + + Requires: set E2E_BACKEND_CONTAINER=1 (takes 30-60s, needs PostgreSQL). + """ + + # #region test_backend_health [C:3] [TYPE Function] [SEMANTICS test,integration,smoke] + # @BRIEF docker run (host network) backend with testcontainers PostgreSQL, + # wait for health, check container logs for startup errors. + # @PRE Docker daemon running. Port 8000 must be free on the host. + # @POST Backend health endpoint responds 200. Container logs show no certs.sh errors. + # @SIDE_EFFECT Starts/stops PostgreSQL and backend Docker containers. + def test_backend_health(self): + import time as time_module + import urllib.request + import urllib.error + + # ── Use host network mode so backend can reach PG on localhost ── + from testcontainers.postgres import PostgresContainer + + pg = PostgresContainer( + "postgres:16-alpine", + dbname="ss_tools", + username="postgres", + password="postgres", + ) + pg.start() + pg_port = pg.get_exposed_port(5432) + + # ── Start backend container (host network) ── + container_name = f"ss-tools-integration-backend-{os.urandom(4).hex()}" + backend_port = "8000" + db_url = f"postgresql://postgres:postgres@127.0.0.1:{pg_port}/ss_tools" + run_cmd = [ + "docker", + "run", + "-d", + "--name", + container_name, + "--network", + "host", + "-e", + f"DATABASE_URL={db_url}", + "-e", + "INITIAL_ADMIN_CREATE=true", + "-e", + "INITIAL_ADMIN_USERNAME=admin", + "-e", + "INITIAL_ADMIN_PASSWORD=admin123", + "-e", + "INITIAL_ADMIN_EMAIL=admin@example.com", + "-e", + "ENABLE_BELIEF_STATE_LOGGING=true", + "-e", + "TASK_LOG_LEVEL=ERROR", + "-e", + "CERTS_PATH=/opt/certs", + BACKEND_IMAGE, + ] + + r = subprocess.run(run_cmd, capture_output=True, text=True, timeout=30) + if r.returncode != 0: + pg.stop() + pytest.fail(f"docker run failed:\nSTDERR: {r.stderr[:2000]}") + + container_id = r.stdout.strip() + container_removed = False + + def _cleanup(rm_container=True): + nonlocal container_removed + if rm_container and not container_removed: + subprocess.run(["docker", "rm", "-f", container_id], capture_output=True, timeout=10) + container_removed = True + try: + pg.stop() + except Exception: + pass + + try: + # ── Wait for health ── + health_url = f"http://127.0.0.1:{backend_port}/api/health/summary" + deadline = time_module.time() + 60 + last_error = "" + + while time_module.time() < deadline: + try: + hr = subprocess.run( + ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", "--connect-timeout", "2", "--max-time", "4", health_url], + capture_output=True, + text=True, + timeout=10, + ) + http_code = hr.stdout.strip() + if http_code in ("200", "401", "403"): + break + last_error = f"HTTP {http_code}" + except subprocess.TimeoutExpired: + last_error = "curl timeout" + except Exception as e: + last_error = str(e) + time_module.sleep(2) + else: + # Timeout — dump container logs + logs_r = subprocess.run( + ["docker", "logs", container_id, "--tail", "60"], + capture_output=True, + text=True, + timeout=10, + ) + full_log = logs_r.stdout[:5000] + logs_r.stderr[:2000] + _cleanup(rm_container=True) + pytest.fail(f"Backend health check failed after 60s.\nLast error: {last_error}\nContainer logs:\n{full_log}") + + # ── Check container logs for startup errors ── + logs_r = subprocess.run( + ["docker", "logs", container_id], + capture_output=True, + text=True, + timeout=10, + ) + combined_logs = logs_r.stdout + logs_r.stderr + + if re.search(r"No such file or directory.*certs\.sh|certs\.sh.*No such file", combined_logs): + _cleanup(rm_container=True) + pytest.fail(f"certs.sh not found! Entrypoint crashed:\n{combined_logs[:3000]}") + + if re.search(r"Traceback|Error.*certs\.sh", combined_logs): + _cleanup(rm_container=True) + pytest.fail(f"Entrypoint crash detected:\n{combined_logs[:3000]}") + + finally: + _cleanup(rm_container=True) + + # #endregion test_backend_health + + +# #endregion TestBackendContainer diff --git a/docker/all-in-one.Dockerfile b/docker/all-in-one.Dockerfile index 6fd43406..043d47b9 100644 --- a/docker/all-in-one.Dockerfile +++ b/docker/all-in-one.Dockerfile @@ -51,7 +51,8 @@ COPY backend/ /app/backend/ # Frontend build (from stage 1) COPY --from=frontend-builder /app/frontend/build /app/frontend/build -# Entrypoint +# Entrypoint + certs.sh +COPY docker/certs.sh /app/certs.sh COPY docker/backend.entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh diff --git a/docker/backend.Dockerfile b/docker/backend.Dockerfile index 884c9820..00cbb768 100644 --- a/docker/backend.Dockerfile +++ b/docker/backend.Dockerfile @@ -31,7 +31,8 @@ RUN python -m playwright install --with-deps chromium # Исходный код COPY backend/ /app/backend/ -# Копируем entrypoint — установит корп. сертификаты, Playwright и сделает bootstrap admin +# Копируем shared certs.sh и entrypoint — установит корп. сертификаты, Playwright и сделает bootstrap admin +COPY docker/certs.sh /app/certs.sh COPY docker/backend.entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh diff --git a/docker/backend.entrypoint.sh b/docker/backend.entrypoint.sh index a3d881f7..a6a6b36b 100755 --- a/docker/backend.entrypoint.sh +++ b/docker/backend.entrypoint.sh @@ -16,254 +16,18 @@ set -euo pipefail # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [backend/src/scripts/create_admin.py] # @RELATION DEPENDS_ON -> [backend/src/scripts/init_auth_db.py] -# @INVARIANT Если CERTS_PATH пуст или нет .crt файлов — install_certificates не падает. +# @INVARIANT Если CERTS_PATH пуст или нет сертификатов — certs.sh:install_all_certs не падает. # @INVARIANT Существующий admin аккаунт НЕ перезаписывается при перезапуске контейнера # (create_admin.py идемпотентен). -# @INVARIANT Повторный запуск install_llm_ca_certs не дублирует сертификаты в bundle. -# @INVARIANT Повторный запуск install_ca_to_nss не дублирует сертификаты в NSS DB. +# @INVARIANT Повторный запуск certs.sh:install_all_certs не дублирует сертификаты в bundle. +# @INVARIANT Повторный запуск certs.sh:install_ca_to_nss не дублирует сертификаты в NSS DB. # @INVARIANT wait_for_db exit(1) если БД недоступна после DB_WAIT_ATTEMPTS попыток — # контейнер не продолжит запуск с недоступной БД. # @INVARIANT Alembic миграции выполняются ТОЛЬКО в entrypoint, НЕ в app.py lifespan. # #endregion docker.backend.entrypoint -# ====================================================================== -# install_certificates — корпоративные CA-сертификаты -# ====================================================================== -# #region docker.backend.entrypoint.install_certificates [C:3] [TYPE Function] -# @BRIEF Устанавливает корпоративные .crt сертификаты из CERTS_PATH в системное хранилище Debian. -# @PRE CERTS_PATH указывает на директорию (может быть пуста или не существовать). -# В системе установлен пакет ca-certificates. -# @POST .crt файлы скопированы в /usr/local/share/ca-certificates/custom/ и -# добавлены в системное хранилище через update-ca-certificates. -# @SIDE_EFFECT Запускает update-ca-certificates — модифицирует /etc/ssl/certs/. -# @LAYER Infrastructure -# @RELATION DEPENDS_ON -> [ca-certificates (system package)] -# @EXAMPLE: -# CERTS_PATH=/opt/certs docker run ... # монтируем ./certs в /opt/certs -# В ./certs кладём: my-corporate-ca.crt, another-ca.crt -# entrypoint установит оба в доверенные. -install_certificates() { - local certs_dir="${CERTS_PATH:-/opt/certs}" - - if [ ! -d "$certs_dir" ]; then - echo "[entrypoint] CERTS_PATH=${certs_dir} не существует — пропускаем установку сертификатов" - return 0 - fi - - local cert_count - cert_count="$(find "$certs_dir" -maxdepth 1 \( -name '*.crt' -o -name '*.pem' \) 2>/dev/null | wc -l)" - - if [ "$cert_count" -eq 0 ]; then - echo "[entrypoint] Нет .crt/.pem файлов в ${certs_dir} — пропускаем установку сертификатов" - return 0 - fi - - local target="/usr/local/share/ca-certificates/custom" - mkdir -p "$target" - - echo "[entrypoint] Устанавливаем корпоративные сертификаты из ${certs_dir}..." - local copied=0 - # nullglob: если файлов нет, цикл не выполняется - shopt -s nullglob - for cert in "$certs_dir"/*.crt "$certs_dir"/*.pem; do - cp -v "$cert" "$target/" 2>&1 | sed 's/^/[entrypoint] /' - copied=$((copied + 1)) - done - shopt -u nullglob - - if [ "$copied" -eq 0 ]; then - echo "[entrypoint] Нет файлов для копирования — пропускаем" - return 0 - fi - - echo "[entrypoint] Обновляем системное хранилище CA-сертификатов..." - update-ca-certificates --fresh 2>&1 | sed 's/^/[entrypoint] /' - - echo "[entrypoint] ✅ Установлено ${copied} корпоративных сертификатов" -} -# #endregion docker.backend.entrypoint.install_certificates - -# ====================================================================== -# install_llm_ca_certs — скачка и установка CA-сертификатов по URL -# ====================================================================== -# #region docker.backend.entrypoint.install_llm_ca_certs [C:4] [TYPE Function] -# @BRIEF Скачивает PEM/DER-сертификаты из LLM_CA_CERT_URLS, конвертирует DER→PEM, -# устанавливает в системное хранилище, создаёт хеш-симлинки и при необходимости -# добавляет в ca-certificates.crt напрямую (fallback для OpenSSL 3-совместимости). -# @RATIONALE DER→PEM конвертация необходима, т.к. update-ca-certificates ожидает -# PEM-формат (-----BEGIN CERTIFICATE-----). Корпоративные PKI часто отдают -# сертификаты в DER (бинарном) формате через HTTP. -# @RATIONALE Хеш-симлинки создаются с поддержкой коллизий (.0, .1, .2...), т.к. -# update-ca-certificates в Debian Bookworm иногда пропускает создание ссылок -# для сертификатов из поддиректорий. Fingerprint (SHA256) используется для -# детекции дубликатов вместо имени файла — защита от ложных срабатываний grep. -# @RATIONALE Fallback cat в ca-certificates.crt гарантирует что OpenSSL 3.x -# увидит сертификаты даже если update-ca-certificates их пропустит. -# Перед добавлением проверяется fingerprint — при повторном запуске -# сертификат не дублируется. -# @REJECTED Вариант "только контейнерный volume с .crt файлами" отвергнут — -# требует ручного копирования сертификатов на хост перед запуском. -# URL-based подход позволяет централизованно управлять сертификатами через PKI. -# @REJECTED Использование grep по имени файла/серийному номеру для детекции -# дубликатов отвергнуто — имя файла может совпадать с содержимым PEM, -# а серийный номер не уникален при ротации CA. Fingerprint (SHA256) -# является криптографически уникальным идентификатором. -# @PRE LLM_CA_CERT_URLS — строка с URL через пробел; curl и openssl установлены. -# @POST Сертификаты доступны в /etc/ssl/certs/ca-certificates.crt. -# Хеш-симлинки созданы для каждого сертификата (с поддержкой коллизий). -# @SIDE_EFFECT Скачивает файлы из внешних URL. Модифицирует /etc/ssl/certs/. -# @LAYER Infrastructure -# @EXAMPLE: -# LLM_CA_CERT_URLS="http://pki.company.com/root.crt http://pki.company.com/intermediate.crt" -install_llm_ca_certs() { - local urls="${LLM_CA_CERT_URLS:-}" - if [ -z "$urls" ]; then - echo "[entrypoint] LLM_CA_CERT_URLS не задан — пропускаем скачку сертификатов" - return 0 - fi - - local work_dir="/tmp/llm-ca-certs" - local target_dir="/usr/local/share/ca-certificates/llm" - mkdir -p "$work_dir" "$target_dir" - - echo "[entrypoint] === Скачивание и установка LLM CA-сертификатов ===" - local index=0 - - for url in $urls; do - url="$(echo "$url" | xargs)" # trim - [ -z "$url" ] && continue - index=$((index + 1)) - - # Определяем имя файла из URL, или генерируем - local filename - filename="$(basename "$url" | sed 's/[?#].*//')" - if [ -z "$filename" ] || [ "$filename" = "." ] || [ "$filename" = ".." ]; then - filename="llm-ca-${index}.crt" - fi - - local raw_file="${work_dir}/${filename}" - local pem_file="${target_dir}/$(echo "$filename" | sed 's/\.[^.]*$//').crt" - - echo "[entrypoint] [${index}] Скачивание: ${url}" - - # Скачиваем с retry 3 раза - local attempt=0 - local success=0 - while [ $attempt -lt 3 ]; do - attempt=$((attempt + 1)) - # Сначала пробуем с verify, если не получилось — с --insecure - # (chicken-and-egg: PKI сервер может использовать тот же CA, который мы скачиваем) - if curl -sS --connect-timeout 10 --max-time 30 -o "$raw_file" "$url" 2>/dev/null; then - success=1 - break - fi - # TLS verify failed? Пробуем без проверки (корп. PKI bootstrap) - if [ $attempt -eq 1 ]; then - echo "[entrypoint] ↻ TLS verify failed, retrying with --insecure..." - if curl -sS -k --connect-timeout 10 --max-time 30 -o "$raw_file" "$url" 2>/dev/null; then - echo "[entrypoint] ✓ Скачано с --insecure (проверка fingerprint в конце)" - success=1 - break - fi - fi - echo "[entrypoint] ⚠ Попытка ${attempt}/3 не удалась, повтор через 2с..." - sleep 2 - done - - if [ "$success" -eq 0 ]; then - echo "[entrypoint] ❌ Не удалось скачать: ${url} — пропускаем" - continue - fi - - # Определяем формат: PEM или DER - if openssl x509 -in "$raw_file" -inform PEM -noout 2>/dev/null; then - echo "[entrypoint] ✓ Формат: PEM" - cp "$raw_file" "$pem_file" - elif openssl x509 -in "$raw_file" -inform DER -noout 2>/dev/null; then - echo "[entrypoint] ↻ Формат: DER — конвертируем в PEM" - if ! openssl x509 -in "$raw_file" -inform DER -out "$pem_file" -outform PEM 2>&1; then - echo "[entrypoint] ❌ Ошибка конвертации DER→PEM — пропускаем" - rm -f "$raw_file" "${pem_file}" - continue - fi - else - echo "[entrypoint] ❌ Неизвестный формат (не PEM и не DER) — пропускаем" - # Выводим первые 20 байт для диагностики - local first_bytes - first_bytes="$(head -c 20 "$raw_file" 2>/dev/null | cat -v)" - echo "[entrypoint] Первые 20 байт: ${first_bytes}" - rm -f "$raw_file" - continue - fi - - chmod 644 "$pem_file" - echo "[entrypoint] ✅ Установлен: $(basename "$pem_file")" - rm -f "$raw_file" - done - - if [ -z "$(ls -A "$target_dir" 2>/dev/null)" ]; then - echo "[entrypoint] Нет скачанных сертификатов — пропускаем" - rm -rf "$work_dir" - return 0 - fi - - # Обновляем системное хранилище - echo "[entrypoint] --- Обновление системного хранилища CA-сертификатов ---" - update-ca-certificates --fresh 2>&1 | sed 's/^/[entrypoint] /' - - # Проверяем, попали ли сертификаты в бандл, и создаём хеш-симлинки - echo "[entrypoint] --- Валидация установки сертификатов ---" - - for cert_file in "$target_dir"/*.crt; do - [ -f "$cert_file" ] || continue - local cert_name - cert_name="$(basename "$cert_file" .crt)" - local cert_subject - cert_subject="$(openssl x509 -in "$cert_file" -noout -subject 2>/dev/null | head -1 || echo "unknown")" - local cert_fingerprint - cert_fingerprint="$(openssl x509 -in "$cert_file" -noout -fingerprint -sha256 2>/dev/null | sed 's/.*=//' || echo "")" - - # Проверяем есть ли в ca-certificates.crt по fingerprint (криптографически уникален) - if [ -n "$cert_fingerprint" ] && \ - grep -qF "$cert_fingerprint" /etc/ssl/certs/ca-certificates.crt 2>/dev/null; then - echo "[entrypoint] ✅ ${cert_name} — в ca-certificates.crt (${cert_subject})" - else - echo "[entrypoint] ⚠ ${cert_name} — НЕ в ca-certificates.crt. Добавляем напрямую..." - cat "$cert_file" >> /etc/ssl/certs/ca-certificates.crt - echo "[entrypoint] ➕ Добавлен напрямую в ca-certificates.crt" - fi - - # Создаём хеш-симлинку с поддержкой коллизий (.0, .1, .2...) - local hash_val - hash_val="$(openssl x509 -in "$cert_file" -noout -hash 2>/dev/null || true)" - if [ -n "$hash_val" ]; then - local suffix=0 - local link_path="/etc/ssl/certs/${hash_val}.${suffix}" - # Ищем первый свободный suffix или symlink уже указывающий на наш файл - while [ -L "$link_path" ]; do - if [ "$(readlink "$link_path")" = "$cert_file" ]; then - break 2 # уже есть, ничего не делаем - fi - suffix=$((suffix + 1)) - link_path="/etc/ssl/certs/${hash_val}.${suffix}" - done - if [ ! -L "$link_path" ]; then - ln -sf "$cert_file" "$link_path" - echo "[entrypoint] 🔗 Создана симлинка: ${hash_val}.${suffix} → ${cert_name}" - fi - fi - done - - # Проверяем итоговое количество - local total_in_bundle - total_in_bundle="$(awk -v cmd='openssl x509 -noout -subject' '/BEGIN/{close(cmd)};{print | cmd}' \ - < /etc/ssl/certs/ca-certificates.crt 2>/dev/null | wc -l || echo "0")" - echo "[entrypoint] ✅ Всего сертификатов в bundle: ${total_in_bundle}" - - rm -rf "$work_dir" - echo "[entrypoint] === Установка LLM CA-сертификатов завершена ===" -} -# #endregion docker.backend.entrypoint.install_llm_ca_certs +# NOTE: install_certificates and install_llm_ca_certs removed — replaced by docker/certs.sh:install_all_certs. +# Centralized CERTS_PATH is the only certificate source. LLM_CA_CERT_URLS env var is removed (refs ADR-0009). # ====================================================================== # bootstrap_admin — создание admin пользователя @@ -323,12 +87,13 @@ bootstrap_admin() { # для CA trust, отдельно от системного OpenSSL bundle. update-ca-certificates # добавляет сертификаты в /etc/ssl/certs/, но Chromium их не видит. # certutil -A -t "C,," импортирует PEM как доверенный CA для TLS. -# @RATIONALE NSS импорт выполняется после install_llm_ca_certs (скачивает .pem) -# и update-ca-certificates (устанавливает в систему). Этот порядок +# @RATIONALE NSS импорт выполняется после certs.sh:install_all_certs +# (устанавливает .crt в /usr/local/share/ca-certificates/custom/) +# и update-ca-certificates (добавляет в систему). Этот порядок # гарантирует, что PEM-файлы уже есть на диске до создания NSS DB. # @RATIONALE Для дедупликации используется SHA256 fingerprint сертификата, # а не nickname — nickname может совпадать для файлов из разных директорий. -# Nickname формируется как "llm-имя" или "custom-имя" для избежания коллизий. +# Nickname формируется как "custom-имя" (все сертификаты из CERTS_PATH). # @REJECTED Использование Playwright --ignore-certificate-errors отвергнуто — # отключает ВСЕ SSL проверки глобально, а не только для корп. CA. # @REJECTED Nickname-only дедупликация отвергнута — при разных файлах @@ -347,9 +112,8 @@ install_ca_to_nss() { local nss_db_dir="${HOME:-/root}/.pki/nssdb" local nss_db="sql:${nss_db_dir}" - # CA certs могут быть в llm/ или custom/ + # All CA certs are installed via docker/certs.sh into custom/ dir local target_dirs=( - "/usr/local/share/ca-certificates/llm" "/usr/local/share/ca-certificates/custom" ) diff --git a/docs/adr/ADR-0009-ssl-certificate-management.md b/docs/adr/ADR-0009-ssl-certificate-management.md index 08a1dfcd..1c199966 100644 --- a/docs/adr/ADR-0009-ssl-certificate-management.md +++ b/docs/adr/ADR-0009-ssl-certificate-management.md @@ -1,13 +1,17 @@ # [DEF:ADR-0009:ADR] # @STATUS ACTIVE -# @PURPOSE Define the strategy for corporate SSL certificate management across all LLM HTTP clients in superset-tools (httpx, requests, openai), covering system CA store, NSS database for Chromium/Playwright, and the LLM_SSL_VERIFY escape hatch. +# @PURPOSE Define the strategy for corporate SSL certificate management across all HTTP clients in superset-tools (httpx, requests, openai), covering system CA store, NSS database for Chromium/Playwright. Centralized CERTS_PATH contract replaces per-client SSL toggles. LLM_SSL_VERIFY and LLM_CA_CERT_URLS removed in 0.2.x. # @RELATION DEPENDS_ON -> [ADR-0001:ADR] # @RELATION DEPENDS_ON -> [ADR-0004:ADR] # @RELATION CALLS -> [docker/backend.entrypoint.sh] +# @RELATION CALLS -> [docker/certs.sh] +# @RELATION CALLS -> [docker/frontend.entrypoint.sh] +# @RELATION CALLS -> [docker/agent.entrypoint.sh] +# @RELATION CALLS -> [backend/src/core/ssl.py] # @RELATION CALLS -> [backend/src/plugins/llm_analysis/service.py] -# @RELATION CALLS -> [backend/src/plugins/translate/_llm_http.py] -# @RELATION CALLS -> [backend/src/plugins/translate/preview_llm_client.py] +# @RELATION CALLS -> [backend/src/plugins/translate/_llm_async_http.py] # @RELATION CALLS -> [scripts/check_llm_certs.py] +# @RELATION CALLS -> [scripts/diag_container.py] # @RATIONALE superset-tools operates in corporate environments with internal Certificate Authorities. # Three separate HTTP stacks are used: httpx (AsyncOpenAI in llm_analysis plugin), # requests (sync translate plugin), and Playwright Chromium (dashboard screenshots). @@ -66,12 +70,11 @@ Diagnostic matrix (verified on production server 2026-05-28): ### Layer 1: System CA Store (OpenSSL) -- Entrypoint `install_certificates()` copies `.crt` files from `CERTS_PATH` volume mount -- Entrypoint `install_llm_ca_certs()` downloads PEM/DER certificates from `LLM_CA_CERT_URLS` -- DER is auto-converted to PEM (`openssl x509 -inform DER -outform PEM`) -- Certificates are saved with **`.crt` extension** (`.pem` is silently ignored by `update-ca-certificates`) +- Shared `docker/certs.sh:install_all_certs()` copies `.crt`/`.pem`/`.cer`/`.der` files from `CERTS_PATH` volume mount +- DER is auto-converted to PEM via `normalize_cert()` (`openssl x509 -inform DER -outform PEM`) +- Server/private files (`server.crt`, `server.key`, `*.key`, `*.p12`, `*.pfx`) are excluded from CA import +- Certificates are saved with **`.crt` extension** to `/usr/local/share/ca-certificates/custom/` - `update-ca-certificates --fresh` adds them to `/etc/ssl/certs/ca-certificates.crt` -- **Fallback**: if `update-ca-certificates` misses certs, they are appended to `ca-certificates.crt` with SHA256 fingerprint dedup - Hash symlinks created with collision support: `.0`, `.1`, `.2` suffixes ### Layer 2: Python HTTP Clients @@ -94,27 +97,32 @@ intermediate CA in OpenSSL 3.x. `verify=` works correctly. - Trust attributes: `"C,,"` (trusted CA for TLS server certs) - Fixes `ERR_CERT_AUTHORITY_INVALID` in Playwright dashboard screenshots -### Layer 4: LLM_SSL_VERIFY Escape Hatch +### Layer 4: Centralized CERTS_PATH Trust Contract (0.2.x) -- Env var `LLM_SSL_VERIFY=false` disables SSL verification entirely -- Accepted values for false: `false`, `0`, `no`, `off` (case-insensitive) -- Default: enabled (returns capath-based SSLContext or path) -- Used by all three HTTP client implementations -- **WARNING**: `verify=False` is for DIAGNOSTIC USE ONLY. Never leave in production. +- **REMOVED**: `LLM_SSL_VERIFY` — TLS verification is always enabled; no env escape hatch exists. +- **REMOVED**: `LLM_CA_CERT_URLS` — certificate download by URL is no longer supported. +- **Replaced by**: `CERTS_PATH` volume mount (`./certs:/opt/certs:ro`) as the single certificate input. +- All trust anchors come from mounted `CERTS_PATH` directory (`.crt`, `.pem`, `.cer`, `.der` files). +- Shared `docker/certs.sh` handles installation across all containers (backend, frontend, agent). +- Centralized Python helper `backend/src/core/ssl.py` provides `system_ssl_context()` / `httpx_verify()`. +- No code path disables TLS verification — the `verify=False` escape hatch is permanently removed. +- **Migration**: Place corporate CA files in `./certs/`, remove `LLM_SSL_VERIFY`/`LLM_CA_CERT_URLS` from `.env`. ## Key Files | File | Role | |------|------| | `docker/backend.Dockerfile` | Installs `libnss3-tools` (certutil), Playwright Chromium | -| `docker/backend.entrypoint.sh` | `install_certificates`, `install_llm_ca_certs`, `install_ca_to_nss` | -| `plugins/llm_analysis/service.py` | `LLMClient._get_ssl_verify()` → `ssl.create_default_context(capath=...)` | -| `plugins/translate/_llm_http.py` | `_get_verify()` → `"/etc/ssl/certs/"` | -| `plugins/translate/preview_llm_client.py` | `_get_verify()` → `"/etc/ssl/certs/"` | -| `scripts/check_llm_certs.py` | Full diagnostic: openssl, httpx, requests, NSS | -| `docker-compose.yml` | Passes `LLM_SSL_VERIFY`, `LLM_CA_CERT_URLS` | -| `docker-compose.enterprise-clean.yml` | Same as above | -| `.env.enterprise-clean` | Default values for both env vars | +| `docker/certs.sh` | Shared cert installer: `install_all_certs`, `install_ca_to_nss`, `normalize_cert` | +| `docker/backend.entrypoint.sh` | Sources `certs.sh`, calls `install_all_certs` + `install_ca_to_nss` | +| `docker/frontend.entrypoint.sh` | Installs CA certs from `CERTS_PATH` into Alpine store | +| `docker/agent.entrypoint.sh` | Sources `certs.sh`, installs certs for agent container | +| `backend/src/core/ssl.py` | Centralized SSL helper: `system_ssl_context()`, `httpx_verify()`, `describe_context()` | +| `plugins/llm_analysis/service.py` | `LLMClient._get_ssl_verify()` → delegates to `core.ssl.httpx_verify()` | +| `plugins/translate/_llm_async_http.py` | `_get_verify()` → delegates to `core.ssl.httpx_verify()` | +| `scripts/check_llm_certs.py` | Full diagnostic: openssl, httpx, requests, NSS, centralized SSL | +| `docker-compose.yml` | Mounts `${CERTS_PATH:-./certs}:/opt/certs:ro` (LLM env vars removed) | +| `docker-compose.enterprise-clean.yml` | Same — `CERTS_PATH` mount only | ## How to Test Certificate Installation @@ -148,8 +156,8 @@ openssl x509 -in /usr/local/share/ca-certificates/custom/RUSAL_ROOT.crt -noout - # 2. Verify full chain openssl verify -CAfile /usr/local/share/ca-certificates/custom/RUSAL_ROOT.crt \ - -untrusted /usr/local/share/ca-certificates/llm/UC_RUSAL_Policy_CA.crt \ - /usr/local/share/ca-certificates/llm/UC_RUSAL_RGM_Issuing_CA.crt + -untrusted /usr/local/share/ca-certificates/custom/UC_RUSAL_Policy_CA.crt \ + /usr/local/share/ca-certificates/custom/UC_RUSAL_RGM_Issuing_CA.crt # → OK # 3. Connect with capath @@ -180,10 +188,11 @@ Corporate PKI servers (`pki.rusal.com`) often serve certificates in DER (binary) `update-ca-certificates` requires PEM. Auto-detection and conversion (`openssl x509 -inform DER -outform PEM`) is essential. -### Finding 5: Chicken-and-egg TLS bootstrap +### Finding 5: Chicken-and-egg TLS bootstrap (HISTORICAL — removed in 0.2.x) Downloading a CA certificate from a PKI server that uses the same CA causes a TLS -verification loop. Entrypoint uses `curl --insecure` as fallback when initial `curl` -fails with TLS error. +verification loop. Formerly handled by `install_llm_ca_certs()` with `curl --insecure`. +**Removed in 0.2.x**: certificates must be placed in `CERTS_PATH` volume mount by operator, +eliminating the chicken-and-egg download problem. ### Finding 6: NSS DB format Chromium uses NSS Shared DB in `~/.pki/nssdb/`. The `sql:` prefix is required for @@ -213,11 +222,12 @@ Fix: pass `input=""` or `echo | openssl s_client ...`. ```bash # On target server: -xz -dc superset-tools-backend.0.1.7.tar.xz | docker load -xz -dc superset-tools-frontend.0.1.7.tar.xz | docker load +xz -dc superset-tools-backend.0.2.x.tar.xz | docker load +xz -dc superset-tools-frontend.0.2.x.tar.xz | docker load -# Enable capath-based verification (remove LLM_SSL_VERIFY=false) -sed -i '/LLM_SSL_VERIFY=false/d' .env.enterprise-clean +# Place corporate CA files in ./certs (LLM_SSL_VERIFY and LLM_CA_CERT_URLS removed) +ls ./certs/ +# RUSAL_ROOT.crt RGM_Issuing.crt etc. docker compose -f docker-compose.enterprise-clean.yml \ --env-file .env.enterprise-clean down @@ -239,5 +249,6 @@ docker compose -f docker-compose.enterprise-clean.yml \ | 0.1.5 | Initial SSL support: `LLM_SSL_VERIFY`, `_format_connection_error()`, CA download via URL, NSS import | | 0.1.6 | QA fixes: fingerprint dedup, hash symlink collision, NSS collision, DER→PEM error handling, chicken-and-egg TLS, nullglob, translate plugin SSL | | 0.1.7 | **capath instead of cafile**: OpenSSL 3.x cafile limitation discovered and fixed. `.crt` extension for downloaded certs. Diagnostic script rewritten. | +| 0.2.x | **Centralized SSL**: `LLM_SSL_VERIFY` and `LLM_CA_CERT_URLS` removed. Shared `docker/certs.sh` for all containers. `backend/src/core/ssl.py` central helper. `CERTS_PATH` volume mount is the only certificate input. Verify=False escape hatch permanently removed. | # [/DEF:ADR-0009:ADR] diff --git a/scripts/check_llm_certs.py b/scripts/check_llm_certs.py index 2c4e7186..3a28798a 100755 --- a/scripts/check_llm_certs.py +++ b/scripts/check_llm_certs.py @@ -34,10 +34,15 @@ results: dict = {} # ── Helpers ── -def _run(cmd: list[str], timeout: int = 15, input_data: str = "") -> tuple[int, str, str]: + +def _run( + cmd: list[str], timeout: int = 15, input_data: str = "" +) -> tuple[int, str, str]: """Run subprocess with optional stdin.""" try: - r = subprocess.run(cmd, input=input_data, capture_output=True, text=True, timeout=timeout) + r = subprocess.run( + cmd, input=input_data, capture_output=True, text=True, timeout=timeout + ) return r.returncode, r.stdout, r.stderr except FileNotFoundError: return -1, "", "command not found" @@ -55,10 +60,18 @@ def _count_certs(path: str) -> int: # ── Openssl tests ── + def test_openssl(label: str, extra_args: list[str]) -> None: """Test openssl s_client with empty stdin to prevent hang.""" - cmd = ["openssl", "s_client", "-connect", f"{host}:443", - "-servername", host, "-verify_return_error"] + extra_args + cmd = [ + "openssl", + "s_client", + "-connect", + f"{host}:443", + "-servername", + host, + "-verify_return_error", + ] + extra_args rc, out, err = _run(cmd, timeout=20, input_data="") for line in out.split("\n"): if "Verify return code" in line: @@ -72,9 +85,11 @@ def test_openssl(label: str, extra_args: list[str]) -> None: # ── Python library tests ── + def test_httpx(key: str, verify) -> None: try: import httpx + r = httpx.get(TARGET_URL, verify=verify, timeout=10) results[key] = "OK" print(f" {key}: ✅ status={r.status_code}") @@ -86,6 +101,7 @@ def test_httpx(key: str, verify) -> None: def test_requests(key: str, verify) -> None: try: import requests as req + r = req.get(TARGET_URL, verify=verify, timeout=10) results[key] = "OK" print(f" {key}: ✅ status={r.status_code}") @@ -100,11 +116,12 @@ def test_requests(key: str, verify) -> None: if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--target", default=os.environ.get("TARGET_URL", "https://lite.ai.rusal.com")) + parser.add_argument( + "--target", default=os.environ.get("TARGET_URL", "https://lite.ai.rusal.com") + ) args = parser.parse_args() TARGET_URL = args.target host = TARGET_URL.replace("https://", "").split("/")[0] - LLM_SSL_VERIFY = os.environ.get("LLM_SSL_VERIFY", "true") # ── 1. System ── print("\n=== 1. System ===") @@ -117,7 +134,9 @@ if __name__ == "__main__": v = getattr(mod, "__version__", "installed") p = getattr(mod, "__file__", "") if lib == "certifi": - print(f" {lib}={v} path={mod.where()} certs={_count_certs(mod.where())}") + print( + f" {lib}={v} path={mod.where()} certs={_count_certs(mod.where())}" + ) else: print(f" {lib}={v}") except ImportError: @@ -136,11 +155,24 @@ if __name__ == "__main__": continue for f in sorted(d.iterdir()): if f.suffix in (".pem", ".crt") and f.is_file(): - rc, out, _ = _run(["openssl", "x509", "-in", str(f), "-noout", - "-subject", "-issuer"], timeout=3) + rc, out, _ = _run( + ["openssl", "x509", "-in", str(f), "-noout", "-subject", "-issuer"], + timeout=3, + ) if rc != 0: - rc, out, _ = _run(["openssl", "x509", "-in", str(f), "-inform", - "DER", "-noout", "-subject"], timeout=3) + rc, out, _ = _run( + [ + "openssl", + "x509", + "-in", + str(f), + "-inform", + "DER", + "-noout", + "-subject", + ], + timeout=3, + ) subj = out.replace("subject=", "").strip() if rc == 0 else "UNREADABLE" print(f" {f.parent.name}/{f.name} subj={subj[:60]}") @@ -154,11 +186,21 @@ if __name__ == "__main__": print("\n=== 4. Python httpx (LLMClient) ===") try: import httpx + test_httpx("httpx_True", verify=True) try: import ssl - ctx_cafile = ssl.create_default_context(cafile=str(CA_BUNDLE)) if CA_BUNDLE.exists() else ssl.create_default_context() - ctx_capath = ssl.create_default_context(capath=str(CA_DIR)) if CA_DIR.is_dir() else ssl.create_default_context() + + ctx_cafile = ( + ssl.create_default_context(cafile=str(CA_BUNDLE)) + if CA_BUNDLE.exists() + else ssl.create_default_context() + ) + ctx_capath = ( + ssl.create_default_context(capath=str(CA_DIR)) + if CA_DIR.is_dir() + else ssl.create_default_context() + ) test_httpx("httpx_cafile", verify=ctx_cafile) test_httpx("httpx_capath", verify=ctx_capath) except Exception as e: @@ -171,6 +213,7 @@ if __name__ == "__main__": print("\n=== 5. Python requests (translate plugin) ===") try: import requests as req + test_requests("requests_True", verify=True) test_requests("requests_cafile", verify=str(CA_BUNDLE)) test_requests("requests_capath", verify=str(CA_DIR)) @@ -182,7 +225,9 @@ if __name__ == "__main__": print("\n=== 6. NSS Chromium ===") rc, out, err = _run(["certutil", "-L", "-d", NSS_DB], timeout=5) if rc == 0: - lines = [l for l in out.split("\n") if l.strip() and "Certificate Nickname" not in l] + lines = [ + l for l in out.split("\n") if l.strip() and "Certificate Nickname" not in l + ] print(f" DB={NSS_DB} certs={len(lines)}") for l in lines: if any(x in l.lower() for x in ("rusal", "llm-", "custom-")): @@ -190,17 +235,38 @@ if __name__ == "__main__": else: print(f" NSS: {err.strip() or 'unavailable'}") - # ── 7. LLM_SSL_VERIFY env ── - print("\n=== 7. LLM_SSL_VERIFY ===") - print(f" LLM_SSL_VERIFY={LLM_SSL_VERIFY} disabled={LLM_SSL_VERIFY.strip().lower() in ('false','0','no','off')}") + # ── 7. Centralized SSL (core.ssl) ── + print("\n=== 7. Centralized SSL (core.ssl) ===") + try: + from src.core.ssl import system_ssl_context, describe_context + + ctx = system_ssl_context() + print(f" {describe_context(ctx)}") + print( + " LLM_SSL_VERIFY and LLM_CA_CERT_URLS removed — centralized CERTS_PATH only." + ) + except ImportError: + print(" src.core.ssl: UNAVAILABLE (backend not installed)") # ── 8. JSON Summary ── print("\n=== 8. JSON Summary ===") - summary = {k: results[k] for k in [ - "openssl_default", "openssl_cafile", "openssl_capath", - "httpx_True", "httpx_cafile", "httpx_capath", "httpx_False", - "requests_True", "requests_cafile", "requests_capath", "requests_False", - ] if k in results} + summary = { + k: results[k] + for k in [ + "openssl_default", + "openssl_cafile", + "openssl_capath", + "httpx_True", + "httpx_cafile", + "httpx_capath", + "httpx_False", + "requests_True", + "requests_cafile", + "requests_capath", + "requests_False", + ] + if k in results + } # Diagnosis diag = [] @@ -212,7 +278,9 @@ if __name__ == "__main__": v = str(summary.get(k, "")) diag.append(f"{k}={'OK' if v.startswith('OK') else 'FAIL'}") - if str(results.get("httpx_capath", "")).startswith("OK") and not str(results.get("httpx_cafile", "")).startswith("OK"): + if str(results.get("httpx_capath", "")).startswith("OK") and not str( + results.get("httpx_cafile", "") + ).startswith("OK"): diag.append("VERDICT=capath_works_cafile_fails_use_capath") elif str(summary.get("httpx_False", "")).startswith("OK"): diag.append("VERDICT=all_ssl_fails_verify_disabled_works")