busya
27a20cbbeb
Core fix: all httpx.AsyncClient instances now use ssl.create_default_context() instead of bool verify=True, which uses certifi and ignores system CA store. This makes corporate CA certificates installed via update-ca-certificates visible to Python HTTP clients. Files: - async_network.py: AsyncAPIClient.__init__ converts True→SSLContext - client_registry.py: get_client converts bool→SSLContext before passing - notifications/providers.py: _get_http_client uses ssl.create_default_context() - services/git/_base.py: GitServiceBase uses ssl.create_default_context() - translate/_llm_async_http.py: _get_verify returns SSLContext (not string) Test: new integration test with 3-tier PKI (Root→Intermediate→Server), TLS-protected Superset container, and custom CA installation via update-ca-certificates or SSL_CERT_DIR fallback. - conftest.py: added ca_chain, install_custom_ca, superset_tls_env fixtures; parameterized superset_container for TLS mode - test_superset_tls_custom_ca.py: 8 tests (openssl -CApath, -CAfile certifi, httpx capath, httpx certifi, AsyncAPIClient, SupersetClient, fingerprint, verify=False)
301 lines
13 KiB
Python
301 lines
13 KiB
Python
# #region TestSupersetTlsCustomCA [C:3] [TYPE Module] [SEMANTICS test,superset,tls,ssl,integration,ca]
|
|
# @BRIEF Integration tests validating the full corporate SSL certificate chain:
|
|
# custom CA generation → system store installation → HTTPS via ss-tools HTTP clients.
|
|
# @RELATION BINDS_TO -> [AsyncAPIClient]
|
|
# @RELATION BINDS_TO -> [SupersetClientBase]
|
|
# @RELATION BINDS_TO -> [SupersetClientRegistryModule]
|
|
# @RELATION BINDS_TO -> [install_certificates (entrypoint)]
|
|
# @RELATION BINDS_TO -> [ADR-0009]
|
|
# @RATIONALE Validates that ssl.create_default_context() (used by our patched HTTP clients)
|
|
# trusts custom CA certificates installed via update-ca-certificates or SSL_CERT_DIR.
|
|
# Uses 3-tier PKI (Root → Intermediate → Server) to test the OpenSSL 3.x capath vs cafile
|
|
# limitation documented in ADR-0009 Finding 7.
|
|
# @TEST_CONTRACT InstallCustomCA ->
|
|
# update-ca-certificates (or SSL_CERT_DIR fallback) makes custom Root CA discoverable
|
|
# ssl.create_default_context(capath=...) builds chain successfully
|
|
# ssl.create_default_context(cafile=...) fails with intermediate CA (OpenSSL 3.x bug)
|
|
# AsyncAPIClient(verify_ssl=True) authenticates against TLS Superset
|
|
# SupersetClient(verify_ssl=True) provides full API access over HTTPS
|
|
# @REQUIRES Docker daemon running. --run-integration flag.
|
|
|
|
import pytest
|
|
import re
|
|
import ssl
|
|
import subprocess
|
|
from urllib.parse import urlparse
|
|
|
|
# ── Test Class ──────────────────────────────────────────────────────────
|
|
|
|
|
|
# #region TestCustomCATrustChain [C:3] [TYPE Class]
|
|
# @BRIEF Validate that custom CA certificates installed into the system store
|
|
# are trusted by openssl, httpx, and ss-tools patched HTTP clients.
|
|
@pytest.mark.parametrize("superset_container", [{"tls": True}], indirect=True)
|
|
class TestCustomCATrustChain:
|
|
"""Full corporate CA trust chain validation against TLS-protected Superset."""
|
|
|
|
# ── openssl tests ─────────────────────────────────────────────────
|
|
|
|
# #region test_openssl_verify_capath_ok [C:2] [TYPE Function]
|
|
# @BRIEF openssl s_client with -CApath succeeds (hash symlinks).
|
|
# @POST Verify return code: 0 (ok)
|
|
def test_openssl_verify_capath_ok(self, superset_url, install_custom_ca):
|
|
"""openssl s_client -CApath must return code 0 (full chain)."""
|
|
parsed = urlparse(superset_url)
|
|
host = parsed.hostname
|
|
port = parsed.port or 443
|
|
ca_path = install_custom_ca["ca_verify_path"]
|
|
|
|
result = subprocess.run(
|
|
[
|
|
"openssl", "s_client",
|
|
"-connect", f"{host}:{port}",
|
|
"-servername", host,
|
|
"-CApath", ca_path,
|
|
"-verify_return_error",
|
|
],
|
|
input="", capture_output=True, text=True, timeout=15,
|
|
)
|
|
|
|
# Parse Verify return code from output
|
|
match = re.search(r"Verify return code:\s*(\d+)", result.stderr + result.stdout)
|
|
assert match is not None, (
|
|
f"No Verify return code in openssl output.\n"
|
|
f"stdout: {result.stdout[:2000]}\nstderr: {result.stderr[:2000]}"
|
|
)
|
|
code = int(match.group(1))
|
|
assert code == 0, (
|
|
f"Expected verify code 0, got {code}.\n"
|
|
f"Full output:\n{result.stderr[:2000]}"
|
|
)
|
|
# #endregion test_openssl_verify_capath_ok
|
|
|
|
|
|
# #region test_openssl_verify_cafile_certifi_fails [C:2] [TYPE Function]
|
|
# @BRIEF openssl s_client -CAfile with certifi's bundle fails.
|
|
# This proves that certifi's CA bundle does NOT trust our custom Root CA.
|
|
# Before our patches, httpx used certifi by default (verify=True → certifi).
|
|
# After our patches, httpx uses ssl.create_default_context() (system CA store).
|
|
def test_openssl_verify_cafile_certifi_fails(self, superset_url):
|
|
"""openssl s_client -CAfile $(certifi) must return non-zero verify code.
|
|
This proves that our custom CA is NOT trusted by certifi, which is what
|
|
httpx used before our patches. After patches, we use system CA store."""
|
|
import certifi
|
|
parsed = urlparse(superset_url)
|
|
host = parsed.hostname
|
|
port = parsed.port or 443
|
|
|
|
result = subprocess.run(
|
|
[
|
|
"openssl", "s_client",
|
|
"-connect", f"{host}:{port}",
|
|
"-servername", host,
|
|
"-CAfile", certifi.where(),
|
|
"-verify_return_error",
|
|
],
|
|
input="", capture_output=True, text=True, timeout=15,
|
|
)
|
|
|
|
match = re.search(r"Verify return code:\s*(\d+)", result.stderr + result.stdout)
|
|
assert match is not None, (
|
|
f"No Verify return code in openssl output.\n"
|
|
f"stdout: {result.stdout[:2000]}\nstderr: {result.stderr[:2000]}"
|
|
)
|
|
code = int(match.group(1))
|
|
assert code != 0, (
|
|
"Expected non-zero verify code with certifi CA bundle, got 0.\n"
|
|
f"This means certifi somehow trusts our generated CA, which should not happen.\n"
|
|
f"Full output:\n{result.stderr[:2000]}"
|
|
)
|
|
# #endregion test_openssl_verify_cafile_certifi_fails
|
|
|
|
|
|
# ── httpx SSL context tests ──────────────────────────────────────
|
|
|
|
# #region test_httpx_capath_works [C:2] [TYPE Function]
|
|
# @BRIEF httpx with ssl.create_default_context(capath=...) succeeds.
|
|
def test_httpx_capath_works(self, superset_url, install_custom_ca):
|
|
"""ssl.create_default_context(capath=...) must successfully connect.
|
|
This is the production code path used by all patched HTTP clients."""
|
|
import asyncio
|
|
|
|
import httpx
|
|
|
|
ca_path = install_custom_ca["ca_verify_path"]
|
|
ctx = ssl.create_default_context(capath=ca_path)
|
|
|
|
async def _test():
|
|
async with httpx.AsyncClient(verify=ctx, timeout=10) as client:
|
|
resp = await client.get(f"{superset_url}/health")
|
|
assert resp.status_code == 200
|
|
assert resp.text.strip() == "OK"
|
|
|
|
asyncio.run(_test())
|
|
# #endregion test_httpx_capath_works
|
|
|
|
|
|
# #region test_httpx_certifi_fails [C:2] [TYPE Function]
|
|
# @BRIEF httpx with certifi's CA bundle must fail.
|
|
# This proves the old behavior (verify=True → certifi) would fail with
|
|
# corporate CA certificates. After our patches, we use system CA store
|
|
# via ssl.create_default_context(capath=...).
|
|
def test_httpx_certifi_fails(self, superset_url):
|
|
"""ssl.create_default_context(cafile=certifi.where()) must raise SSLError.
|
|
Proves that certifi does NOT trust our custom Root CA, which is why
|
|
our patches replacing verify=True with ssl.create_default_context()
|
|
are necessary."""
|
|
import asyncio
|
|
|
|
import certifi
|
|
import httpx
|
|
|
|
ctx = ssl.create_default_context(cafile=certifi.where())
|
|
|
|
async def _test():
|
|
async with httpx.AsyncClient(verify=ctx, timeout=10) as client:
|
|
with pytest.raises((ssl.SSLCertVerificationError, httpx.RemoteProtocolError)):
|
|
await client.get(f"{superset_url}/health")
|
|
|
|
asyncio.run(_test())
|
|
# #endregion test_httpx_certifi_fails
|
|
|
|
|
|
# ── ss-tools client tests ────────────────────────────────────────
|
|
|
|
# #region test_asyncapiclient_verify_true [C:2] [TYPE Function]
|
|
# @BRIEF AsyncAPIClient(verify_ssl=True) authenticates against TLS Superset.
|
|
# Validates our patched async_network.py code path.
|
|
def test_asyncapiclient_verify_true(self, superset_url, superset_admin_password, install_custom_ca):
|
|
"""AsyncAPIClient with verify_ssl=True must authenticate over HTTPS.
|
|
This validates the core SSL fix in async_network.py."""
|
|
# install_custom_ca is used for its side effect — installing CA into system store
|
|
assert install_custom_ca is not None
|
|
import asyncio
|
|
|
|
from src.core.utils.async_network import AsyncAPIClient
|
|
|
|
parsed = urlparse(superset_url)
|
|
base_url = f"{parsed.scheme}://{parsed.hostname}:{parsed.port}"
|
|
|
|
client = AsyncAPIClient(
|
|
config={
|
|
"base_url": base_url,
|
|
"auth": {
|
|
"username": "admin",
|
|
"password": superset_admin_password,
|
|
"provider": "db",
|
|
"refresh": "true",
|
|
},
|
|
},
|
|
verify_ssl=True,
|
|
timeout=30,
|
|
)
|
|
|
|
async def _test():
|
|
try:
|
|
tokens = await client.authenticate()
|
|
assert "access_token" in tokens, "No access_token in auth response"
|
|
assert "csrf_token" in tokens, "No csrf_token in auth response"
|
|
finally:
|
|
await client.aclose()
|
|
|
|
asyncio.run(_test())
|
|
# #endregion test_asyncapiclient_verify_true
|
|
|
|
|
|
# #region test_superset_client_full_auth [C:2] [TYPE Function]
|
|
# @BRIEF Full SupersetClient auth + API call over TLS with verify_ssl=True.
|
|
def test_superset_client_full_auth(self, superset_url, superset_admin_password, install_custom_ca):
|
|
"""SupersetClient with verify_ssl=True must authenticate and
|
|
execute API queries over HTTPS."""
|
|
# install_custom_ca is used for its side effect — installing CA into system store
|
|
assert install_custom_ca is not None
|
|
import asyncio
|
|
|
|
from src.core.config_models import Environment
|
|
from src.core.superset_client import SupersetClient
|
|
|
|
env = Environment(
|
|
id="test_tls",
|
|
name="Test TLS Superset",
|
|
url=superset_url,
|
|
username="admin",
|
|
password=superset_admin_password,
|
|
verify_ssl=True,
|
|
timeout=30,
|
|
)
|
|
|
|
async def _test():
|
|
client = SupersetClient(env)
|
|
try:
|
|
tokens = await client.authenticate()
|
|
assert "access_token" in tokens, "No access_token"
|
|
assert "csrf_token" in tokens, "No csrf_token"
|
|
|
|
# Fetch dashboards list via the authenticated client
|
|
result = await client.get_dashboards(
|
|
query={"page": 0, "page_size": 10}
|
|
)
|
|
|
|
assert isinstance(result, dict), f"Expected dict, got {type(result)}"
|
|
assert "result" in result, f"No 'result' key: {list(result.keys())}"
|
|
finally:
|
|
await client.aclose()
|
|
|
|
asyncio.run(_test())
|
|
# #endregion test_superset_client_full_auth
|
|
|
|
|
|
# ── Regression tests ─────────────────────────────────────────────
|
|
|
|
# #region test_certifi_bundle_notrust [C:1] [TYPE Function]
|
|
# @BRIEF Prove that certifi's CA bundle does NOT contain our custom Root CA.
|
|
# This validates WHY the patches are needed: before our fix, httpx used
|
|
# certifi by default, which would NOT trust corporate CA certificates.
|
|
def test_certifi_bundle_notrust(self, ca_chain):
|
|
"""certifi.where() bundle must NOT contain our custom Root CA.
|
|
This proves that default httpx.AsyncClient(verify=True) would reject
|
|
corporate certificates installed via update-ca-certificates.
|
|
Uses SHA256 fingerprint dedup, same as the entrypoint fallback logic."""
|
|
import certifi
|
|
|
|
# Load our Root CA and compute its SHA256 fingerprint
|
|
from cryptography import x509
|
|
from cryptography.hazmat.primitives import hashes
|
|
|
|
our_ca = x509.load_pem_x509_certificate(ca_chain["root_crt"].encode())
|
|
our_fingerprint = our_ca.fingerprint(hashes.SHA256()).hex()
|
|
|
|
# Load certifi's bundle and check if any cert matches our fingerprint
|
|
with open(certifi.where(), "rb") as f:
|
|
bundle_data = f.read()
|
|
|
|
assert our_fingerprint not in bundle_data.hex(), (
|
|
"Our custom Root CA was found in certifi bundle — "
|
|
"this should not happen since the CA was just generated at test time."
|
|
)
|
|
# #endregion test_certifi_bundle_notrust
|
|
|
|
|
|
# #region test_verify_false_still_works [C:1] [TYPE Function]
|
|
# @BRIEF verify_ssl=False must still work (regression test).
|
|
# The SSL patches must not break the disabled-verification path.
|
|
def test_verify_false_still_works(self, superset_url):
|
|
"""AsyncAPIClient(verify_ssl=False) must work over HTTPS
|
|
(regression — the LLM_SSL_VERIFY=false escape hatch must not break)."""
|
|
import asyncio
|
|
|
|
import httpx
|
|
|
|
async def _test():
|
|
async with httpx.AsyncClient(verify=False, timeout=10) as client:
|
|
resp = await client.get(f"{superset_url}/health")
|
|
assert resp.status_code == 200
|
|
assert resp.text.strip() == "OK"
|
|
|
|
asyncio.run(_test())
|
|
# #endregion test_verify_false_still_works
|
|
|
|
# #endregion TestCustomCATrustChain
|
|
# #endregion TestSupersetTlsCustomCA
|