fix(ssl): replace verify=True with ssl.create_default_context() for corporate CA support
Core fix: all httpx.AsyncClient instances now use ssl.create_default_context() instead of bool verify=True, which uses certifi and ignores system CA store. This makes corporate CA certificates installed via update-ca-certificates visible to Python HTTP clients. Files: - async_network.py: AsyncAPIClient.__init__ converts True→SSLContext - client_registry.py: get_client converts bool→SSLContext before passing - notifications/providers.py: _get_http_client uses ssl.create_default_context() - services/git/_base.py: GitServiceBase uses ssl.create_default_context() - translate/_llm_async_http.py: _get_verify returns SSLContext (not string) Test: new integration test with 3-tier PKI (Root→Intermediate→Server), TLS-protected Superset container, and custom CA installation via update-ca-certificates or SSL_CERT_DIR fallback. - conftest.py: added ca_chain, install_custom_ca, superset_tls_env fixtures; parameterized superset_container for TLS mode - test_superset_tls_custom_ca.py: 8 tests (openssl -CApath, -CAfile certifi, httpx capath, httpx certifi, AsyncAPIClient, SupersetClient, fingerprint, verify=False)
This commit is contained in:
@@ -220,6 +220,111 @@ def verify_postgres_features(session: Session) -> dict:
|
||||
# #endregion verify_postgres_features
|
||||
|
||||
|
||||
# ── TLS / PKI Fixtures ────────────────────────────────────────────────
|
||||
|
||||
|
||||
# #region ca_chain [C:2] [TYPE Fixture]
|
||||
# @BRIEF Session-scoped — generates 3-tier PKI (Root CA → Intermediate CA → Server cert).
|
||||
# @POST Returns dict with root_crt, intermediate_crt, server_crt, server_key, fullchain PEM strings.
|
||||
@pytest.fixture(scope="session")
|
||||
def ca_chain():
|
||||
"""Generate 3-tier PKI: Root CA → Intermediate CA → Server certificate (SAN: localhost)."""
|
||||
from cryptography import x509
|
||||
from cryptography.x509.oid import NameOID
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
import datetime
|
||||
|
||||
def _gen_key():
|
||||
return rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
|
||||
def _gen_cert_pem(subject_name, issuer_name, subject_key, issuer_key, is_ca=False, san_dns=None, san_ip=None):
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
builder = x509.CertificateBuilder()
|
||||
builder = builder.subject_name(x509.Name([
|
||||
x509.NameAttribute(NameOID.COMMON_NAME, subject_name),
|
||||
]))
|
||||
builder = builder.issuer_name(x509.Name([
|
||||
x509.NameAttribute(NameOID.COMMON_NAME, issuer_name),
|
||||
]))
|
||||
builder = builder.not_valid_before(now - datetime.timedelta(hours=1))
|
||||
builder = builder.not_valid_after(now + datetime.timedelta(days=365))
|
||||
builder = builder.serial_number(x509.random_serial_number())
|
||||
builder = builder.public_key(subject_key.public_key())
|
||||
|
||||
if is_ca:
|
||||
builder = builder.add_extension(
|
||||
x509.BasicConstraints(ca=True, path_length=None), critical=True,
|
||||
)
|
||||
builder = builder.add_extension(
|
||||
x509.KeyUsage(
|
||||
key_cert_sign=True, crl_sign=True,
|
||||
digital_signature=False, content_commitment=False,
|
||||
key_encipherment=False, data_encipherment=False,
|
||||
key_agreement=False, encipher_only=False, decipher_only=False,
|
||||
), critical=True,
|
||||
)
|
||||
else:
|
||||
# Server cert
|
||||
builder = builder.add_extension(
|
||||
x509.BasicConstraints(ca=False, path_length=None), critical=True,
|
||||
)
|
||||
sans: list[x509.GeneralName] = []
|
||||
if san_dns:
|
||||
sans.append(x509.DNSName(san_dns))
|
||||
if san_ip:
|
||||
sans.append(x509.IPAddress(san_ip))
|
||||
builder = builder.add_extension(
|
||||
x509.SubjectAlternativeName(sans), critical=False,
|
||||
)
|
||||
|
||||
cert = builder.sign(issuer_key, hashes.SHA256())
|
||||
return cert.public_bytes(serialization.Encoding.PEM).decode()
|
||||
|
||||
root_key = _gen_key()
|
||||
root_crt = _gen_cert_pem("Test Root CA", "Test Root CA", root_key, root_key, is_ca=True)
|
||||
|
||||
intermediate_key = _gen_key()
|
||||
intermediate_crt = _gen_cert_pem(
|
||||
"Test Intermediate CA", "Test Root CA",
|
||||
intermediate_key, root_key, is_ca=True,
|
||||
)
|
||||
|
||||
server_key = _gen_key()
|
||||
import ipaddress
|
||||
server_crt = _gen_cert_pem(
|
||||
"localhost", "Test Intermediate CA",
|
||||
server_key, intermediate_key, is_ca=False,
|
||||
san_dns="localhost", san_ip=ipaddress.IPv4Address("127.0.0.1"),
|
||||
)
|
||||
|
||||
# Full chain for server: server PEM + intermediate PEM (order: leaf first)
|
||||
fullchain = server_crt + "\n" + intermediate_crt
|
||||
|
||||
return {
|
||||
"root_crt": root_crt,
|
||||
"root_key": root_key.private_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
encryption_algorithm=serialization.NoEncryption(),
|
||||
).decode(),
|
||||
"intermediate_crt": intermediate_crt,
|
||||
"intermediate_key": intermediate_key.private_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
encryption_algorithm=serialization.NoEncryption(),
|
||||
).decode(),
|
||||
"server_crt": server_crt,
|
||||
"server_key": server_key.private_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
encryption_algorithm=serialization.NoEncryption(),
|
||||
).decode(),
|
||||
"fullchain": fullchain,
|
||||
}
|
||||
# #endregion ca_chain
|
||||
|
||||
|
||||
# ── Superset Integration Test Fixtures ────────────────────────────────
|
||||
|
||||
|
||||
@@ -278,23 +383,54 @@ def superset_admin_password():
|
||||
|
||||
# #region superset_container [C:3] [TYPE Fixture]
|
||||
# @BRIEF Session-scoped Apache Superset container — db upgrade → admin → init → web server.
|
||||
# Can be parametrized with {"tls": True} to enable HTTPS with custom 3-tier PKI.
|
||||
# @PRE Docker daemon is running. superset_db_url points to an empty Postgres database.
|
||||
# @POST Superset web server is listening on port 8088. Admin user created.
|
||||
# @POST Superset web server is listening on port 8088 (HTTP or HTTPS). Admin user created.
|
||||
# @SIDE_EFFECT Spins up a short-lived init container, then a long-lived web container.
|
||||
# In TLS mode: writes server cert + key into container via base64, runs with --cert/--key.
|
||||
# @RATIONALE Two-container approach (init + web) avoids race conditions where the web
|
||||
# server starts before migrations have completed. The init container runs `superset db
|
||||
# upgrade`, creates the admin user, and runs `superset init`, then exits. The web
|
||||
# container then starts against the fully initialized database.
|
||||
# TLS parametrization allows testing corporate SSL certificate chain with the
|
||||
# same Superset instance, using a dynamically generated 3-tier PKI.
|
||||
# @REJECTED Single container with sequential exec calls — cannot change entrypoint
|
||||
# from init to web server within the same container lifecycle.
|
||||
# @REJECTED Using `latest` tag — version drift breaks tests silently.
|
||||
@pytest.fixture(scope="session")
|
||||
def superset_container(superset_db_url, superset_secret_key, superset_admin_password):
|
||||
"""Start an Apache Superset instance for integration testing."""
|
||||
def superset_container(request, superset_db_url, superset_secret_key, superset_admin_password):
|
||||
"""Start an Apache Superset instance for integration testing.
|
||||
|
||||
Parametrize with indirect=True, {"tls": True} to enable HTTPS with custom CA chain.
|
||||
"""
|
||||
from testcontainers.core.container import DockerContainer
|
||||
import base64
|
||||
|
||||
superset_image = "apache/superset:4.1.2"
|
||||
|
||||
# Check if TLS was requested via parametrization
|
||||
tls_config = getattr(request, 'param', None)
|
||||
use_tls = tls_config is not None and tls_config.get('tls', False)
|
||||
|
||||
if use_tls:
|
||||
ca_chain = request.getfixturevalue('ca_chain')
|
||||
# Base64-encode certs to avoid shell escaping issues in heredoc
|
||||
fullchain_b64 = base64.b64encode(ca_chain["fullchain"].encode()).decode()
|
||||
key_b64 = base64.b64encode(ca_chain["server_key"].encode()).decode()
|
||||
_write_certs = (
|
||||
f"echo '{fullchain_b64}' | base64 -d > /tmp/server.crt && "
|
||||
f"echo '{key_b64}' | base64 -d > /tmp/server.key"
|
||||
)
|
||||
_protocol = "https"
|
||||
_superset_run = (
|
||||
f"superset run -h 0.0.0.0 -p 8088 "
|
||||
f"--cert /tmp/server.crt --key /tmp/server.key --with-threads"
|
||||
)
|
||||
else:
|
||||
_write_certs = ""
|
||||
_protocol = "http"
|
||||
_superset_run = "superset run -h 0.0.0.0 -p 8088 --with-threads"
|
||||
|
||||
# Superset ignores SQLALCHEMY_DATABASE_URI env var — we must create
|
||||
# superset_config.py and point SUPERSET_CONFIG_PATH to it.
|
||||
_bootstrap_config = (
|
||||
@@ -311,6 +447,7 @@ def superset_container(superset_db_url, superset_secret_key, superset_admin_pass
|
||||
init.with_env("SUPERSET_DB_URI", superset_db_url)
|
||||
init.with_env("SUPERSET_LOAD_EXAMPLES", "no")
|
||||
init.with_env("FLASK_APP", "superset")
|
||||
# Note: init container does NOT need TLS certs — only runs db upgrade + init
|
||||
init.with_command(
|
||||
f"sh -c '"
|
||||
f"python -m pip install psycopg2-binary -q && "
|
||||
@@ -344,28 +481,46 @@ def superset_container(superset_db_url, superset_secret_key, superset_admin_pass
|
||||
superset.with_env("SUPERSET_DB_URI", superset_db_url)
|
||||
superset.with_env("SUPERSET_LOAD_EXAMPLES", "no")
|
||||
superset.with_env("FLASK_APP", "superset")
|
||||
superset.with_command(
|
||||
f"sh -c '"
|
||||
f"python -m pip install psycopg2-binary -q && "
|
||||
f"{_bootstrap_config}"
|
||||
f"{_export_config} && "
|
||||
f"superset run -h 0.0.0.0 -p 8088 --with-threads'"
|
||||
)
|
||||
|
||||
# Build web server command: optionally write certs, then start superset
|
||||
if use_tls and _write_certs:
|
||||
web_cmd = (
|
||||
f"sh -c '"
|
||||
f"python -m pip install psycopg2-binary -q && "
|
||||
f"{_write_certs} && "
|
||||
f"{_bootstrap_config}"
|
||||
f"{_export_config} && "
|
||||
f"{_superset_run}'"
|
||||
)
|
||||
else:
|
||||
web_cmd = (
|
||||
f"sh -c '"
|
||||
f"python -m pip install psycopg2-binary -q && "
|
||||
f"{_bootstrap_config}"
|
||||
f"{_export_config} && "
|
||||
f"{_superset_run}'"
|
||||
)
|
||||
|
||||
superset.with_command(web_cmd)
|
||||
superset.with_exposed_ports(8088)
|
||||
|
||||
superset.start()
|
||||
|
||||
# Store protocol on the container object for superset_url fixture
|
||||
superset._ss_protocol = _protocol
|
||||
|
||||
# Wait for the /health endpoint to respond
|
||||
host = superset.get_container_host_ip()
|
||||
port = superset.get_exposed_port(8088)
|
||||
health_url = f"http://{host}:{port}/health"
|
||||
health_url = f"{_protocol}://{host}:{port}/health"
|
||||
|
||||
deadline = time.time() + 90
|
||||
last_error = None
|
||||
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
resp = requests.get(health_url, timeout=3)
|
||||
# verify=False in TLS mode because the custom CA hasn't been installed yet
|
||||
resp = requests.get(health_url, timeout=3, verify=False)
|
||||
if resp.status_code == 200:
|
||||
break
|
||||
except requests.RequestException as e:
|
||||
@@ -386,12 +541,13 @@ def superset_container(superset_db_url, superset_secret_key, superset_admin_pass
|
||||
|
||||
|
||||
# #region superset_url [C:1] [TYPE Fixture]
|
||||
# @BRIEF Function-scoped — base URL of the running Superset container.
|
||||
# @BRIEF Function-scoped — base URL of the running Superset container (auto-detects http/https).
|
||||
@pytest.fixture
|
||||
def superset_url(superset_container):
|
||||
host = superset_container.get_container_host_ip()
|
||||
port = superset_container.get_exposed_port(8088)
|
||||
return f"http://{host}:{port}"
|
||||
protocol = getattr(superset_container, '_ss_protocol', 'http')
|
||||
return f"{protocol}://{host}:{port}"
|
||||
# #endregion superset_url
|
||||
|
||||
|
||||
@@ -491,4 +647,89 @@ async def superset_client(superset_env):
|
||||
# #endregion superset_client
|
||||
|
||||
|
||||
# ── TLS / Custom CA Fixtures ─────────────────────────────────────
|
||||
|
||||
|
||||
# #region install_custom_ca [C:2] [TYPE Fixture]
|
||||
# @BRIEF Function-scoped — installs Test Root CA into system trust store.
|
||||
# Tries update-ca-certificates (root required), falls back to SSL_CERT_DIR.
|
||||
# @POST Root CA accessible via ssl.create_default_context().
|
||||
# @SIDE_EFFECT Modifies /usr/local/share/ca-certificates/custom/ and runs update-ca-certificates.
|
||||
# On fallback: sets SSL_CERT_DIR env var, creates temp dir with hash symlinks.
|
||||
@pytest.fixture
|
||||
def install_custom_ca(ca_chain):
|
||||
"""Install the custom Root CA into the system trust store.
|
||||
|
||||
Strategy 1 (prod-like): write .crt → update-ca-certificates (requires root).
|
||||
Strategy 2 (fallback): SSL_CERT_DIR with hash symlinks.
|
||||
"""
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ca_pem = ca_chain["root_crt"]
|
||||
used_update_ca = False
|
||||
ca_verify_path = None
|
||||
cert_dir = None
|
||||
|
||||
# Strategy 1: update-ca-certificates (matches production install_certificates())
|
||||
try:
|
||||
cert_dir = Path("/usr/local/share/ca-certificates/custom")
|
||||
cert_dir.mkdir(parents=True, exist_ok=True)
|
||||
ca_path = cert_dir / "test_ss_tools_ca.crt"
|
||||
ca_path.write_text(ca_pem)
|
||||
subprocess.run(
|
||||
["update-ca-certificates", "--fresh"],
|
||||
check=True, timeout=30, capture_output=True,
|
||||
)
|
||||
ca_verify_path = "/etc/ssl/certs"
|
||||
used_update_ca = True
|
||||
except (subprocess.CalledProcessError, PermissionError, OSError):
|
||||
# Strategy 2: fallback — SSL_CERT_DIR
|
||||
cert_dir = Path(tempfile.mkdtemp(prefix="test_ca_"))
|
||||
ca_path = cert_dir / "test_ss_tools_ca.pem"
|
||||
ca_path.write_text(ca_pem)
|
||||
|
||||
# Create hash symlink for capath
|
||||
result = subprocess.run(
|
||||
["openssl", "x509", "-hash", "-noout"],
|
||||
input=ca_pem, capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
cert_hash = result.stdout.strip()
|
||||
symlink_path = cert_dir / f"{cert_hash}.0"
|
||||
if symlink_path.exists():
|
||||
symlink_path.unlink()
|
||||
symlink_path.symlink_to(ca_path.name)
|
||||
|
||||
os.environ["SSL_CERT_DIR"] = str(cert_dir)
|
||||
ca_verify_path = str(cert_dir)
|
||||
used_update_ca = False
|
||||
|
||||
yield {
|
||||
"ca_verify_path": ca_verify_path,
|
||||
"used_update_ca": used_update_ca,
|
||||
"ca_pem": ca_pem,
|
||||
}
|
||||
|
||||
# Cleanup
|
||||
if used_update_ca:
|
||||
ca_file = Path("/usr/local/share/ca-certificates/custom/test_ss_tools_ca.crt")
|
||||
if ca_file.exists():
|
||||
ca_file.unlink()
|
||||
try:
|
||||
subprocess.run(
|
||||
["update-ca-certificates", "--fresh"],
|
||||
timeout=30, capture_output=True,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
if "SSL_CERT_DIR" in os.environ:
|
||||
del os.environ["SSL_CERT_DIR"]
|
||||
if cert_dir and cert_dir.exists():
|
||||
import shutil
|
||||
shutil.rmtree(str(cert_dir), ignore_errors=True)
|
||||
# #endregion install_custom_ca
|
||||
|
||||
|
||||
# #endregion IntegrationTestConftest
|
||||
|
||||
Reference in New Issue
Block a user