fix(certs): restore HTTP CA auto-download and test edge matrix

Restore LLM_CA_CERT_URLS support for corporate PKI HTTP cert delivery.
Downloaded certificates are installed into the llm/ CA category and share
the same system trust + NSS import path as CERTS_PATH-mounted certs.

Changes:
- certs.sh: add download_llm_ca_certs with HTTP-only curl policy,
  PEM/DER handling, retries, llm/ storage, hash symlinks for llm+custom
- backend/agent entrypoints: run download before install_all_certs
- compose/env/docs: expose LLM_CA_CERT_URLS again and update ADR-0009
- integration tests: cover PEM/DER/CER, invalid certs, non-HTTP skips,
  query-string filenames, private/server skips, empty inputs, combined
  LLM_CA_CERT_URLS + CERTS_PATH channels, and NSS imports

Verified:
- 194 passed, 1 skipped integration tests
- full backend container smoke passed with testcontainers PostgreSQL
- docker bundle rebuilt for 0.5.0

backend/tests/integration/test_backend_container.py

@@ -53,11 +53,14 @@ BACKEND_IMAGE = "superset-tools-backend:test-integration"
 @pytest.fixture(scope="module")
 def _backend_image():
     """Build superset-tools-backend:test-integration once per module."""
-    # Only rebuild if the Dockerfile changed (simple mtime check)
+    # Rebuild if Dockerfile, entrypoint, or cert logic changed.
     dockerfile = PROJECT_ROOT / "docker" / "backend.Dockerfile"
+    certs_sh = PROJECT_ROOT / "docker" / "certs.sh"
+    entrypoint = PROJECT_ROOT / "docker" / "backend.entrypoint.sh"
     image_marker = PROJECT_ROOT / "backend" / ".image-built"
 
-    if not image_marker.exists() or image_marker.stat().st_mtime < dockerfile.stat().st_mtime:
+    watched_mtime = max(p.stat().st_mtime for p in (dockerfile, certs_sh, entrypoint))
+    if not image_marker.exists() or image_marker.stat().st_mtime < watched_mtime:
         subprocess.run(
             ["docker", "build", "-f", str(dockerfile), "-t", BACKEND_IMAGE, str(PROJECT_ROOT)],
             check=True,
@@ -309,4 +312,284 @@ class TestBackendContainerSmoke:
     # #endregion test_backend_health
 
 
+# ══════════════════════════════════════════════════════════════════════════
+#  TEST 3: cert download — download_llm_ca_certs via HTTP
+# ══════════════════════════════════════════════════════════════════════════
+
+
+class TestBackendImageCertDownload:
+    """Verify download_llm_ca_certs downloads certs from HTTP URLs."""
+
+    @pytest.fixture(autouse=True)
+    def _image(self, _backend_image):
+        pass
+
+    # #region _make_ca_cert [C:1] [TYPE Function]
+    @staticmethod
+    def _make_ca_cert(certs_dir: str, name: str = "test-download-ca") -> tuple[str, str]:
+        ca_key = os.path.join(certs_dir, f"{name}.key")
+        ca_crt = os.path.join(certs_dir, f"{name}.crt")
+        subprocess.run(
+            [
+                "openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes",
+                "-keyout", ca_key, "-out", ca_crt,
+                "-days", "1",
+                "-subj", f"/CN={name}-for-container",
+            ],
+            check=True, capture_output=True, text=True, timeout=15,
+        )
+        return ca_key, ca_crt
+    # #endregion _make_ca_cert
+
+    # #region _serve_dir [C:1] [TYPE Function]
+    @staticmethod
+    def _serve_dir(certs_dir: str):
+        import http.server
+        import socket
+        import socketserver
+        import threading
+
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.bind(("127.0.0.1", 0))
+            http_port = s.getsockname()[1]
+
+        class _Handler(http.server.SimpleHTTPRequestHandler):
+            def __init__(self, *args, **kwargs):
+                super().__init__(*args, directory=certs_dir, **kwargs)
+
+        httpd = socketserver.TCPServer(("127.0.0.1", http_port), _Handler)
+        server_thread = threading.Thread(target=httpd.serve_forever, daemon=True)
+        server_thread.start()
+        return httpd, http_port
+    # #endregion _serve_dir
+
+    # #region _run_download [C:1] [TYPE Function]
+    @staticmethod
+    def _run_download(urls: str, command_suffix: str = "") -> subprocess.CompletedProcess:
+        command = "source /app/certs.sh && download_llm_ca_certs"
+        if command_suffix:
+            command = f"{command} && {command_suffix}"
+        return subprocess.run(
+            [
+                "docker", "run", "--rm",
+                "--network", "host",
+                "-e", f"LLM_CA_CERT_URLS={urls}",
+                "--entrypoint", "bash",
+                BACKEND_IMAGE,
+                "-c", command,
+            ],
+            capture_output=True, text=True, timeout=45,
+        )
+    # #endregion _run_download
+
+    # #region test_download_ca_cert_over_http [C:3] [TYPE Function] [SEMANTICS test,integration,certs,download]
+    # @BRIEF download_llm_ca_certs() fetches a PEM cert from an HTTP server
+    #   and installs it into /usr/local/share/ca-certificates/llm/.
+    def test_download_ca_cert_over_http(self):
+        import tempfile
+
+        # ── Generate a test CA cert ──
+        with tempfile.TemporaryDirectory() as certs_dir:
+            self._make_ca_cert(certs_dir, "test-download-ca")
+            httpd, http_port = self._serve_dir(certs_dir)
+
+            cert_url = f"http://127.0.0.1:{http_port}/test-download-ca.crt"
+
+            try:
+                # ── Run download_llm_ca_certs in container (host network) ──
+                r = self._run_download(cert_url, "echo 'DOWNLOAD_OK' && ls -la /usr/local/share/ca-certificates/llm/")
+
+                assert r.returncode == 0, (
+                    f"download_llm_ca_certs failed (exit {r.returncode}):\n"
+                    f"STDOUT: {r.stdout[:2000]}\nSTDERR: {r.stderr[:2000]}"
+                )
+
+                # Verify download succeeded
+                assert "DOWNLOAD_OK" in r.stdout, (
+                    f"download_llm_ca_certs did not complete:\n{r.stdout[:2000]}"
+                )
+                assert "Скачано и установлено: 1" in r.stdout, (
+                    f"Expected 1 downloaded cert:\n{r.stdout[:2000]}"
+                )
+                assert "test-download-ca.crt" in r.stdout, (
+                    f"Cert file not found in llm/ dir:\n{r.stdout[:2000]}"
+                )
+
+            finally:
+                httpd.shutdown()
+    # #endregion test_download_ca_cert_over_http
+
+    # #region test_download_der_cert_converts_to_pem [C:2] [TYPE Function] [SEMANTICS test,integration,certs,download]
+    # @BRIEF DER certificate downloaded over HTTP is converted to PEM .crt.
+    def test_download_der_cert_converts_to_pem(self):
+        import tempfile
+
+        with tempfile.TemporaryDirectory() as certs_dir:
+            _, ca_crt = self._make_ca_cert(certs_dir, "test-download-der")
+            der_path = os.path.join(certs_dir, "test-download-der.der")
+            subprocess.run(
+                ["openssl", "x509", "-in", ca_crt, "-outform", "DER", "-out", der_path],
+                check=True, capture_output=True, text=True, timeout=15,
+            )
+            httpd, http_port = self._serve_dir(certs_dir)
+            try:
+                url = f"http://127.0.0.1:{http_port}/test-download-der.der"
+                r = self._run_download(url, "grep -q 'BEGIN CERTIFICATE' /usr/local/share/ca-certificates/llm/test-download-der.crt && echo DER_OK")
+                assert r.returncode == 0, f"DER download failed:\n{r.stdout}\n{r.stderr}"
+                assert "DER_OK" in r.stdout
+                assert "Формат: DER" in r.stdout
+            finally:
+                httpd.shutdown()
+    # #endregion test_download_der_cert_converts_to_pem
+
+    # #region test_download_skips_invalid_and_non_http [C:2] [TYPE Function] [SEMANTICS test,integration,certs,download]
+    # @BRIEF Invalid content and HTTPS URLs are skipped without failing valid downloads.
+    def test_download_skips_invalid_and_non_http(self):
+        import tempfile
+
+        with tempfile.TemporaryDirectory() as certs_dir:
+            self._make_ca_cert(certs_dir, "valid-ca")
+            Path(certs_dir, "not-a-cert.crt").write_text("not a certificate", encoding="utf-8")
+            httpd, http_port = self._serve_dir(certs_dir)
+            try:
+                urls = " ".join([
+                    f"http://127.0.0.1:{http_port}/valid-ca.crt",
+                    f"http://127.0.0.1:{http_port}/not-a-cert.crt",
+                    "https://example.invalid/blocked.crt",
+                ])
+                r = self._run_download(urls, "ls -la /usr/local/share/ca-certificates/llm/")
+                assert r.returncode == 0, f"Mixed download failed:\n{r.stdout}\n{r.stderr}"
+                assert "Скачано и установлено: 1" in r.stdout
+                assert "невалид" in r.stdout or "Неизвестный формат" in r.stdout
+                assert "пропускаем не-HTTP URL" in r.stdout
+                assert "valid-ca.crt" in r.stdout
+            finally:
+                httpd.shutdown()
+    # #endregion test_download_skips_invalid_and_non_http
+
+    # #region test_download_query_string_filename_and_nss_import [C:2] [TYPE Function] [SEMANTICS test,integration,certs,nss]
+    # @BRIEF URL query strings are stripped from filenames and downloaded llm/ certs import into NSS.
+    def test_download_query_string_filename_and_nss_import(self):
+        import tempfile
+
+        with tempfile.TemporaryDirectory() as certs_dir:
+            self._make_ca_cert(certs_dir, "query-ca")
+            httpd, http_port = self._serve_dir(certs_dir)
+            try:
+                url = f"http://127.0.0.1:{http_port}/query-ca.crt?token=abc"
+                r = self._run_download(
+                    url,
+                    "install_all_certs && install_ca_to_nss && "
+                    "test -f /usr/local/share/ca-certificates/llm/query-ca.crt && "
+                    "certutil -L -d sql:${HOME}/.pki/nssdb | grep -q 'llm-query-ca' && echo QUERY_NSS_OK",
+                )
+                assert r.returncode == 0, f"Query/NSS flow failed:\n{r.stdout}\n{r.stderr}"
+                assert "QUERY_NSS_OK" in r.stdout
+            finally:
+                httpd.shutdown()
+    # #endregion test_download_query_string_filename_and_nss_import
+
+    # #region test_volume_certs_matrix_pem_der_cer_invalid_private [C:2] [TYPE Function] [SEMANTICS test,integration,certs,volume]
+    # @BRIEF CERTS_PATH handles PEM/DER/CER and skips invalid/private/server files.
+    def test_volume_certs_matrix_pem_der_cer_invalid_private(self):
+        import tempfile
+
+        with tempfile.TemporaryDirectory() as certs_dir:
+            _, pem_crt = self._make_ca_cert(certs_dir, "volume-pem")
+            _, der_src = self._make_ca_cert(certs_dir, "volume-der-src")
+            der_cert = os.path.join(certs_dir, "volume-der.der")
+            subprocess.run(
+                ["openssl", "x509", "-in", der_src, "-outform", "DER", "-out", der_cert],
+                check=True, capture_output=True, text=True, timeout=15,
+            )
+            os.remove(der_src)
+            _, cer_src = self._make_ca_cert(certs_dir, "volume-cer-src")
+            os.rename(cer_src, os.path.join(certs_dir, "volume-cer.cer"))
+            Path(certs_dir, "invalid.crt").write_text("not a certificate", encoding="utf-8")
+            Path(certs_dir, "server.key").write_text("not a key", encoding="utf-8")
+            Path(certs_dir, "server.crt").write_text(Path(pem_crt).read_text(encoding="utf-8"), encoding="utf-8")
+
+            r = subprocess.run(
+                [
+                    "docker", "run", "--rm",
+                    "-v", f"{certs_dir}:/opt/certs:ro",
+                    "-e", "CERTS_PATH=/opt/certs",
+                    "--entrypoint", "bash",
+                    BACKEND_IMAGE,
+                    "-c",
+                    "source /app/certs.sh && install_all_certs && "
+                    "test -f /usr/local/share/ca-certificates/custom/volume-pem.crt && "
+                    "test -f /usr/local/share/ca-certificates/custom/volume-der.crt && "
+                    "test -f /usr/local/share/ca-certificates/custom/volume-cer.crt && "
+                    "test ! -f /usr/local/share/ca-certificates/custom/server.crt && "
+                    "grep -q 'BEGIN CERTIFICATE' /usr/local/share/ca-certificates/custom/volume-der.crt && "
+                    "echo VOLUME_MATRIX_OK",
+                ],
+                capture_output=True, text=True, timeout=45,
+            )
+            assert r.returncode == 0, f"Volume matrix failed:\n{r.stdout}\n{r.stderr}"
+            assert "VOLUME_MATRIX_OK" in r.stdout
+            assert "Установлено: 3" in r.stdout
+            assert "пропускаем: server.key" in r.stdout
+            assert "пропускаем: server.crt" in r.stdout
+            assert "невалидный сертификат: invalid.crt" in r.stdout
+    # #endregion test_volume_certs_matrix_pem_der_cer_invalid_private
+
+    # #region test_download_and_volume_combined_channels [C:2] [TYPE Function] [SEMANTICS test,integration,certs,combined]
+    # @BRIEF LLM_CA_CERT_URLS and CERTS_PATH work together in one container startup flow.
+    def test_download_and_volume_combined_channels(self):
+        import tempfile
+
+        with tempfile.TemporaryDirectory() as mount_dir, tempfile.TemporaryDirectory() as serve_dir:
+            self._make_ca_cert(mount_dir, "mounted-ca")
+            self._make_ca_cert(serve_dir, "downloaded-ca")
+            httpd, http_port = self._serve_dir(serve_dir)
+            try:
+                url = f"http://127.0.0.1:{http_port}/downloaded-ca.crt"
+                r = subprocess.run(
+                    [
+                        "docker", "run", "--rm",
+                        "--network", "host",
+                        "-v", f"{mount_dir}:/opt/certs:ro",
+                        "-e", "CERTS_PATH=/opt/certs",
+                        "-e", f"LLM_CA_CERT_URLS={url}",
+                        "--entrypoint", "bash",
+                        BACKEND_IMAGE,
+                        "-c",
+                        "source /app/certs.sh && download_llm_ca_certs && install_all_certs && install_ca_to_nss && "
+                        "test -f /usr/local/share/ca-certificates/custom/mounted-ca.crt && "
+                        "test -f /usr/local/share/ca-certificates/llm/downloaded-ca.crt && "
+                        "certutil -L -d sql:${HOME}/.pki/nssdb | grep -q 'custom-mounted-ca' && "
+                        "certutil -L -d sql:${HOME}/.pki/nssdb | grep -q 'llm-downloaded-ca' && "
+                        "echo COMBINED_OK",
+                    ],
+                    capture_output=True, text=True, timeout=60,
+                )
+                assert r.returncode == 0, f"Combined channels failed:\n{r.stdout}\n{r.stderr}"
+                assert "COMBINED_OK" in r.stdout
+            finally:
+                httpd.shutdown()
+    # #endregion test_download_and_volume_combined_channels
+
+    # #region test_empty_inputs_graceful [C:2] [TYPE Function] [SEMANTICS test,integration,certs,empty]
+    # @BRIEF Empty LLM_CA_CERT_URLS and missing CERTS_PATH are no-op success cases.
+    def test_empty_inputs_graceful(self):
+        r = subprocess.run(
+            [
+                "docker", "run", "--rm",
+                "-e", "LLM_CA_CERT_URLS=",
+                "-e", "CERTS_PATH=/does-not-exist",
+                "--entrypoint", "bash",
+                BACKEND_IMAGE,
+                "-c",
+                "source /app/certs.sh && download_llm_ca_certs && install_all_certs && echo EMPTY_OK",
+            ],
+            capture_output=True, text=True, timeout=45,
+        )
+        assert r.returncode == 0, f"Empty input path failed:\n{r.stdout}\n{r.stderr}"
+        assert "EMPTY_OK" in r.stdout
+        assert "CERTS_PATH=/does-not-exist не существует" in r.stdout
+    # #endregion test_empty_inputs_graceful
+
+
 # #endregion TestBackendContainer